def connect(redis_conn, key):
"""
Establishes connection with a node to:
1) Send version message
2) Receive version and verack message
3) Send getaddr message
4) Receive addr message containing list of peering nodes
Stores state and height for node in Redis.
"""
handshake_msgs = []
addr_msgs = []
redis_conn.hset(key, 'state', "") # Set Redis hash for a new node
(address, port, services) = key[5:].split("-", 2)
services = int(services)
height = redis_conn.get('height')
if height:
height = int(height)
proxy = None
if address.endswith(".onion"):
proxy = SETTINGS['tor_proxy']
conn = Connection((address, int(port)), (SETTINGS['source_address'], 0),
socket_timeout=SETTINGS['socket_timeout'],
proxy=proxy,
protocol_version=SETTINGS['protocol_version'],
to_services=services,
from_services=SETTINGS['services'],
user_agent=SETTINGS['user_agent'],
height=height,
relay=SETTINGS['relay'])
try:
logging.debug("Connecting to %s", conn.to_addr)
conn.open()
handshake_msgs = conn.handshake()
addr_msgs = conn.getaddr()
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("%s: %s", conn.to_addr, err)
finally:
conn.close()
gevent.sleep(0.3)
redis_pipe = redis_conn.pipeline()
if len(handshake_msgs) > 0:
version_msg = handshake_msgs[0]
from_services = version_msg.get('services', 0)
if from_services != services:
logging.debug("%s Expected %d, got %d for services", conn.to_addr,
services, from_services)
key = "node:{}-{}-{}".format(address, port, from_services)
height_key = "height:{}-{}-{}".format(address, port, from_services)
redis_pipe.setex(height_key, SETTINGS['max_age'],
version_msg.get('height', 0))
now = int(time.time())
peers = enumerate_node(redis_pipe, addr_msgs, now)
logging.debug("%s Peers: %d", conn.to_addr, peers)
redis_pipe.hset(key, 'state', "up")
redis_pipe.execute()
def connect(redis_conn, key):
"""
Establishes connection with a node to:
1) Send version message
2) Receive version and verack message
3) Send getaddr message
4) Receive addr message containing list of peering nodes
Stores state and height for node in Redis.
"""
handshake_msgs = []
addr_msgs = []
redis_conn.hset(key, 'state', "") # Set Redis hash for a new node
(address, port, services) = key[5:].split("-", 2)
height = redis_conn.get('height')
if height:
height = int(height)
conn = Connection((address, int(port)),
(SETTINGS['source_address'], 0),
socket_timeout=SETTINGS['socket_timeout'],
proxy=SETTINGS['proxy'],
protocol_version=SETTINGS['protocol_version'],
to_services=int(services),
from_services=SETTINGS['services'],
user_agent=SETTINGS['user_agent'],
height=height,
relay=SETTINGS['relay'])
try:
logging.debug("Connecting to %s", conn.to_addr)
conn.open()
handshake_msgs = conn.handshake()
addr_msgs = conn.getaddr()
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("%s: %s", conn.to_addr, err)
finally:
conn.close()
gevent.sleep(0.3)
redis_pipe = redis_conn.pipeline()
if len(handshake_msgs) > 0:
height_key = "height:{}-{}-{}".format(address, port, services)
redis_pipe.setex(height_key, SETTINGS['max_age'],
handshake_msgs[0].get('height', 0))
now = int(time.time())
peers = enumerate_node(redis_pipe, addr_msgs, now)
logging.debug("%s Peers: %d", conn.to_addr, peers)
redis_pipe.hset(key, 'state', "up")
redis_pipe.execute()
def _getaddr(self, node, port):
"""
Establishes connection with a node to:
1) Send version message
2) Receive version and verack message
3) Send getaddr message
4) Receive addr message containing list of adjacent nodes
"""
to_addr = (node, port)
conn = Connection(to_addr,
socket_timeout=SETTINGS['socket_timeout'],
user_agent=SETTINGS['user_agent'],
start_height=SETTINGS['start_height'])
error = ""
handshake_msgs = []
addr_msg = {}
try:
conn.open()
handshake_msgs = conn.handshake()
addr_msg = conn.getaddr()
except (ProtocolError, socket.error) as err:
error = str(err)
logging.debug("{}: {} dropped".format(err, to_addr))
finally:
conn.close()
nodes = None
degree = 0
# Record version information for the remote node
if len(handshake_msgs) > 0:
if not self.database.has_node(node, table="nodes_version"):
self.database.add_node_version(node, handshake_msgs[0])
nodes = []
if 'addr_list' in addr_msg:
nodes = self.get_nodes_from_addr_list(addr_msg['addr_list'])
degree = len(nodes)
# Cache the result in database for reuse in subsequent getaddr()
# calls for the same node.
self.database.add_node_getaddr(node, nodes, error, degree)
return nodes
def _getaddr(self, node, port):
"""
Establishes connection with a node to:
1) Send version message
2) Receive version and verack message
3) Send getaddr message
4) Receive addr message containing list of adjacent nodes
"""
to_addr = (node, port)
conn = Connection(to_addr, socket_timeout=SETTINGS['socket_timeout'],
user_agent=SETTINGS['user_agent'],
start_height=SETTINGS['start_height'])
error = ""
handshake_msgs = []
addr_msg = {}
try:
conn.open()
handshake_msgs = conn.handshake()
addr_msg = conn.getaddr()
except (ProtocolError, socket.error) as err:
error = str(err)
logging.debug("{}: {} dropped".format(err, to_addr))
finally:
conn.close()
nodes = None
degree = 0
# Record version information for the remote node
if len(handshake_msgs) > 0:
if not self.database.has_node(node, table="nodes_version"):
self.database.add_node_version(node, handshake_msgs[0])
nodes = []
if 'addr_list' in addr_msg:
nodes = self.get_nodes_from_addr_list(addr_msg['addr_list'])
degree = len(nodes)
# Cache the result in database for reuse in subsequent getaddr()
# calls for the same node.
self.database.add_node_getaddr(node, nodes, error, degree)
return nodes
def task():
"""
Assigned to a worker to retrieve (pop) a node from the reachable set and
attempt to establish and maintain connection with the node.
"""
node = REDIS_CONN.spop('reachable')
if node is None:
return
(address, port, services, height) = eval(node)
node = (address, port)
if REDIS_CONN.sadd('open', node) == 0:
logging.debug("Connection exists: %s", node)
return
handshake_msgs = []
conn = Connection(node, (SETTINGS['source_address'], 0),
socket_timeout=SETTINGS['socket_timeout'],
protocol_version=SETTINGS['protocol_version'],
to_services=services,
from_services=SETTINGS['services'],
user_agent=SETTINGS['user_agent'],
height=height,
relay=SETTINGS['relay'])
try:
conn.open()
handshake_msgs = conn.handshake()
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("Closing %s (%s)", node, err)
conn.close()
if len(handshake_msgs) == 0:
REDIS_CONN.srem('open', node)
return
Keepalive(conn=conn, version_msg=handshake_msgs[0]).keepalive()
conn.close()
REDIS_CONN.srem('open', node)
def task():
"""
Assigned to a worker to retrieve (pop) a node from the reachable set and
attempt to establish and maintain connection with the node.
"""
node = REDIS_CONN.spop('reachable')
if node is None:
return
(address, port, services, height) = eval(node)
node = (address, port)
if REDIS_CONN.sadd('open', node) == 0:
logging.debug("Connection exists: %s", node)
return
handshake_msgs = []
conn = Connection(node, (SETTINGS['source_address'], 0),
socket_timeout=SETTINGS['socket_timeout'],
protocol_version=SETTINGS['protocol_version'],
to_services=services,
from_services=SETTINGS['services'],
user_agent=SETTINGS['user_agent'],
height=height,
relay=SETTINGS['relay'])
try:
conn.open()
handshake_msgs = conn.handshake()
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("Closing %s (%s)", node, err)
conn.close()
if len(handshake_msgs) == 0:
REDIS_CONN.srem('open', node)
return
Keepalive(conn=conn, version_msg=handshake_msgs[0]).keepalive()
conn.close()
REDIS_CONN.srem('open', node)
def connect(redis_conn, key):
"""
Establishes connection with a node to:
1) Send version message
2) Receive version and verack message
3) Send getaddr message
4) Receive addr message containing list of peering nodes
Stores state and height for node in Redis.
"""
handshake_msgs = []
addr_msgs = []
redis_conn.set(key, "") # Set Redis key for a new node
(address, port, services) = key[5:].split("-", 2)
services = int(services)
height = redis_conn.get('height')
if height:
height = int(height)
proxy = None
if address.endswith(".onion"):
proxy = CONF['tor_proxy']
conn = Connection((address, int(port)), (CONF['source_address'], 0),
magic_number=CONF['magic_number'],
socket_timeout=CONF['socket_timeout'],
proxy=proxy,
protocol_version=CONF['protocol_version'],
to_services=services,
from_services=CONF['services'],
user_agent=CONF['user_agent'],
height=height,
relay=CONF['relay'])
print(address)
print(int(port))
print(CONF['magic_number'])
print(CONF['socket_timeout'])
print(proxy)
print(CONF['protocol_version'])
print(CONF['services'])
print(CONF['user_agent'])
print(height)
print(CONF['relay'])
try:
logging.debug("Connecting to %s", conn.to_addr)
conn.open()
handshake_msgs = conn.handshake()
print("OK")
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("%s: %s", conn.to_addr, err)
print("just die")
redis_pipe = redis_conn.pipeline()
if len(handshake_msgs) > 0:
try:
conn.getaddr(block=False)
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("%s: %s", conn.to_addr, err)
else:
addr_wait = 0
while addr_wait < CONF['socket_timeout']:
addr_wait += 1
gevent.sleep(0.3)
try:
msgs = conn.get_messages(commands=['addr'])
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("%s: %s", conn.to_addr, err)
break
if msgs and any([msg['count'] > 1 for msg in msgs]):
addr_msgs = msgs
break
version_msg = handshake_msgs[0]
from_services = version_msg.get('services', 0)
if from_services != services:
logging.debug("%s Expected %d, got %d for services", conn.to_addr,
services, from_services)
key = "node:{}-{}-{}".format(address, port, from_services)
height_key = "height:{}-{}-{}".format(address, port, from_services)
redis_pipe.setex(height_key, CONF['max_age'],
version_msg.get('height', 0))
now = int(time.time())
(peers, excluded) = enumerate_node(redis_pipe, addr_msgs, now)
logging.debug("%s Peers: %d (Excluded: %d)", conn.to_addr, peers,
excluded)
redis_pipe.set(key, "")
redis_pipe.sadd('up', key)
conn.close()
redis_pipe.execute()
def connect(network, address, port, to_services, network_data, user_agent=None, explicit_p2p=False, p2p_nodes=True,
from_services=None, keepalive=False, attempt=1):
now = datetime.datetime.utcnow()
results = {'network': network, 'address': address, 'port': port, 'timestamp': now, 'seen': 0, 'attempt': attempt}
try:
handshake_msgs = []
new_addrs = []
proxy = None
if address.endswith(".onion"):
proxy = CONF['tor_proxy']
conn = Connection((address, port),
(CONF['source_address'], 0),
magic_number=network_data['magic_number'],
socket_timeout=CONF['socket_timeout'],
proxy=proxy,
protocol_version=int(network_data['protocol_version']),
min_protocol_version=network_data['min_protocol_version'],
to_services=int(to_services),
from_services=int(from_services or network_data['services']),
user_agent=user_agent or CONF['user_agent'],
height=int(network_data['height']),
relay=CONF['relay'])
try:
conn.open()
except (ProtocolError, ConnectionError, socket.error) as err:
results['error'] = str(err)
logging.debug("connection failed %s %s", type(err), err)
else:
try:
handshake_msgs = conn.handshake()
assert handshake_msgs
results['seen'] = 1
results['height'] = int(handshake_msgs[0]['height'])
results['version'] = int(handshake_msgs[0]['version'])
results['user_agent'] = handshake_msgs[0]['user_agent'].decode()
results['services'] = int(handshake_msgs[0]['services'])
except (ProtocolError, ConnectionError, socket.error, AssertionError) as err:
results['error'] = str(err)
logging.debug("handshake failed %s", err)
msgs = []
if len(handshake_msgs) > 0 and (p2p_nodes or explicit_p2p):
getaddr = True
chance = CONF['getaddr_prop']
if chance < 1.0 and p2p_nodes and not explicit_p2p and not "--seed" in sys.argv:
if np.random.rand() > chance:
getaddr = False
if getaddr:
try:
conn.getaddr(block=False)
msgs = msgs + conn.get_messages(commands=[b"addr"])
time.sleep(5)
msgs = msgs + conn.get_messages(commands=[b"addr"])
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("getaddr failed %s", err)
if keepalive:
Keepalive(conn, 10).keepalive(addr=True if p2p_nodes else False)
for msg in msgs:
if msg['count'] > 1:
ts = now.timestamp()
for addr in msg['addr_list']:
if ts - addr['timestamp'] < 12 * 60 * 60: # within 6 hours
new_addrs.append(addr)
conn.close()
return results, new_addrs
except Exception as err:
logging.warning("unspecified connection error: %s", err)
return {}, []
def task():
"""
Assigned to a worker to retrieve (pop) a node from the reachable set and
attempt to establish and maintain connection with the node.
"""
node = REDIS_CONN.spop('reachable')
if node is None:
return
(address, port, services, height) = eval(node)
node = (address, port)
# Check if prefix has hit its limit
cidr_key = None
if ":" in address and CONF['ipv6_prefix'] < 128:
cidr = ip_to_network(address, CONF['ipv6_prefix'])
cidr_key = 'ping:cidr:{}'.format(cidr)
nodes = REDIS_CONN.incr(cidr_key)
logging.info("+CIDR %s: %d", cidr, nodes)
if nodes > CONF['nodes_per_ipv6_prefix']:
logging.info("CIDR limit reached: %s", cidr)
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
return
if REDIS_CONN.sadd('open', node) == 0:
logging.info("Connection exists: %s", node)
if cidr_key:
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
return
proxy = None
if address.endswith(".onion"):
proxy = random.choice(CONF['tor_proxies'])
handshake_msgs = []
conn = Connection(node,
(CONF['source_address'], 0),
magic_number=CONF['magic_number'],
socket_timeout=CONF['socket_timeout'],
proxy=proxy,
protocol_version=CONF['protocol_version'],
to_services=services,
from_services=CONF['services'],
user_agent=CONF['user_agent'],
height=height,
relay=CONF['relay'])
try:
logging.debug("Connecting to %s", conn.to_addr)
conn.open()
handshake_msgs = conn.handshake()
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("Closing %s (%s)", node, err)
conn.close()
if len(handshake_msgs) == 0:
if cidr_key:
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
REDIS_CONN.srem('open', node)
return
if address.endswith(".onion"):
# Map local port to .onion node
local_port = conn.socket.getsockname()[1]
logging.info("Local port %s: %d", conn.to_addr, local_port)
REDIS_CONN.set('onion:{}'.format(local_port), conn.to_addr)
Keepalive(conn=conn, version_msg=handshake_msgs[0]).keepalive()
conn.close()
if cidr_key:
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
REDIS_CONN.srem('open', node)
def task():
"""
Assigned to a worker to retrieve (pop) a node from the reachable set and
attempt to establish and maintain connection with the node.
"""
node = REDIS_CONN.spop('reachable')
if node is None:
return
(address, port, services, height) = eval(node)
node = (address, port)
# Check if prefix has hit its limit
cidr_key = None
if ":" in address and SETTINGS['ipv6_prefix'] < 128:
cidr = ip_to_network(address, SETTINGS['ipv6_prefix'])
cidr_key = 'ping:cidr:{}'.format(cidr)
nodes = REDIS_CONN.incr(cidr_key)
logging.info("+CIDR %s: %d", cidr, nodes)
if nodes > SETTINGS['nodes_per_ipv6_prefix']:
logging.info("CIDR limit reached: %s", cidr)
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
return
if REDIS_CONN.sadd('open', node) == 0:
logging.info("Connection exists: %s", node)
if cidr_key:
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
return
proxy = None
if address.endswith(".onion"):
proxy = SETTINGS['tor_proxy']
handshake_msgs = []
conn = Connection(node, (SETTINGS['source_address'], 0),
socket_timeout=SETTINGS['socket_timeout'],
proxy=proxy,
protocol_version=SETTINGS['protocol_version'],
to_services=services,
from_services=SETTINGS['services'],
user_agent=SETTINGS['user_agent'],
height=height,
relay=SETTINGS['relay'])
try:
logging.debug("Connecting to %s", conn.to_addr)
conn.open()
handshake_msgs = conn.handshake()
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("Closing %s (%s)", node, err)
conn.close()
if len(handshake_msgs) == 0:
if cidr_key:
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
REDIS_CONN.srem('open', node)
return
if address.endswith(".onion"):
# Map local port to .onion node
local_port = conn.socket.getsockname()[1]
logging.info("%s: 127.0.0.1:%d", conn.to_addr, local_port)
REDIS_CONN.set('onion:{}'.format(local_port), conn.to_addr)
Keepalive(conn=conn, version_msg=handshake_msgs[0]).keepalive()
conn.close()
if cidr_key:
nodes = REDIS_CONN.decr(cidr_key)
logging.info("-CIDR %s: %d", cidr, nodes)
REDIS_CONN.srem('open', node)
def connect(redis_conn, key):
"""
Establishes connection with a node to:
1) Send version message
2) Receive version and verack message
3) Send getaddr message
4) Receive addr message containing list of peering nodes
Stores state and height for node in Redis.
"""
handshake_msgs = []
addr_msgs = []
redis_conn.set(key, "") # Set Redis key for a new node
(address, port, services) = key[5:].split("-", 2)
services = int(services)
height = redis_conn.get('height')
if height:
height = int(height)
proxy = None
if address.endswith(".onion"):
proxy = CONF['tor_proxy']
conn = Connection((address, int(port)),
(CONF['source_address'], 0),
magic_number=CONF['magic_number'],
socket_timeout=CONF['socket_timeout'],
proxy=proxy,
protocol_version=CONF['protocol_version'],
to_services=services,
from_services=CONF['services'],
user_agent=CONF['user_agent'],
height=height,
relay=CONF['relay'])
try:
logging.debug("Connecting to %s", conn.to_addr)
conn.open()
handshake_msgs = conn.handshake()
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("%s: %s", conn.to_addr, err)
redis_pipe = redis_conn.pipeline()
if len(handshake_msgs) > 0:
try:
conn.getaddr(block=False)
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("%s: %s", conn.to_addr, err)
else:
addr_wait = 0
while addr_wait < CONF['socket_timeout']:
addr_wait += 1
gevent.sleep(0.3)
try:
msgs = conn.get_messages(commands=['addr'])
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("%s: %s", conn.to_addr, err)
break
if msgs and any([msg['count'] > 1 for msg in msgs]):
addr_msgs = msgs
break
version_msg = handshake_msgs[0]
from_services = version_msg.get('services', 0)
if from_services != services:
logging.debug("%s Expected %d, got %d for services", conn.to_addr,
services, from_services)
key = "node:{}-{}-{}".format(address, port, from_services)
height_key = "height:{}-{}-{}".format(address, port, from_services)
redis_pipe.setex(height_key, CONF['max_age'],
version_msg.get('height', 0))
now = int(time.time())
(peers, excluded) = enumerate_node(redis_pipe, addr_msgs, now)
logging.debug("%s Peers: %d (Excluded: %d)",
conn.to_addr, peers, excluded)
redis_pipe.set(key, "")
redis_pipe.sadd('up', key)
conn.close()
redis_pipe.execute()
