def task():
"""
Assigned to a worker to retrieve (pop) a node from the reachable set and
attempt to establish and maintain connection with the node.
"""
node = REDIS_CONN.spop('reachable')
(address, port, start_height) = eval(node)
handshake_msgs = []
connection = Connection((address, port),
socket_timeout=SETTINGS['socket_timeout'],
user_agent=SETTINGS['user_agent'],
start_height=start_height)
try:
connection.open()
handshake_msgs = connection.handshake()
except (ProtocolError, socket.error) as err:
logging.debug("Closing {} ({})".format(connection.to_addr, err))
connection.close()
if len(handshake_msgs) == 0:
return
for msgs in keepalive(connection, handshake_msgs[0]):
save_inv(connection.to_addr, msgs)
def task():
"""
Assigned to a worker to retrieve (pop) a node from the reachable set and
attempt to establish and maintain connection with the node.
"""
node = REDIS_CONN.spop('reachable')
if node is None:
return
(address, port, height) = eval(node)
handshake_msgs = []
connection = Connection((address, port),
socket_timeout=SETTINGS['socket_timeout'],
user_agent=SETTINGS['user_agent'],
height=height)
try:
connection.open()
handshake_msgs = connection.handshake()
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("Closing {} ({})".format(connection.to_addr, err))
connection.close()
if len(handshake_msgs) == 0:
return
keepalive(connection, handshake_msgs[0])
def keepalive(connection, version_msg):
"""
Periodically sends a ping message to the specified node to maintain open
connection. Open connections are tracked in open set with the associated
data stored in opendata set in Redis.
"""
node = connection.to_addr
version = version_msg.get('version', "")
user_agent = version_msg.get('user_agent', "")
start_height = version_msg.get('start_height', 0)
now = int(time.time())
data = node + (version, user_agent, now)
REDIS_CONN.sadd('open', node)
REDIS_CONN.sadd('opendata', data)
start_height_key = "start_height:{}-{}".format(node[0], node[1])
start_height_ttl = SETTINGS['keepalive'] * 2 # > keepalive
REDIS_CONN.setex(start_height_key, start_height_ttl, start_height)
while True:
gevent.sleep(SETTINGS['keepalive'])
REDIS_CONN.setex(start_height_key, start_height_ttl, start_height)
try:
connection.ping()
except socket.error as err:
logging.debug("Closing {} ({})".format(node, err))
break
connection.close()
REDIS_CONN.srem('open', node)
REDIS_CONN.srem('opendata', data)
def keepalive(connection, version_msg):
"""
Periodically sends a ping message to the specified node to maintain open
connection. Open connections are tracked in open set with the associated
data stored in opendata set in Redis.
"""
node = connection.to_addr
version = version_msg.get('version', "")
user_agent = version_msg.get('user_agent', "")
start_height = version_msg.get('start_height', 0)
now = int(time.time())
data = node + (version, user_agent, now)
REDIS_CONN.sadd('open', node)
REDIS_CONN.sadd('opendata', data)
start_height_key = "start_height:{}-{}".format(node[0], node[1])
start_height_ttl = SETTINGS['keepalive'] * 2 # > keepalive
REDIS_CONN.setex(start_height_key, start_height_ttl, start_height)
while True:
gevent.sleep(SETTINGS['keepalive'])
REDIS_CONN.setex(start_height_key, start_height_ttl, start_height)
try:
connection.ping()
except socket.error as err:
logging.debug("Closing {} ({})".format(node, err))
break
connection.close()
REDIS_CONN.srem('open', node)
REDIS_CONN.srem('opendata', data)
def keepalive(connection, version_msg):
"""
Periodically sends a ping message to the specified node to maintain open
connection. Open connections are tracked in open set with the associated
data stored in opendata set in Redis.
"""
node = connection.to_addr
version = version_msg.get('version', "")
user_agent = version_msg.get('user_agent', "")
now = int(time.time())
data = node + (version, user_agent, now)
REDIS_CONN.sadd('open', node)
REDIS_CONN.sadd('opendata', data)
redis_pipe = REDIS_CONN.pipeline()
last_ping = now
while True:
try:
wait = int(REDIS_CONN.get('elapsed'))
except TypeError as err:
wait = 60
if time.time() > last_ping + wait:
nonce = random.getrandbits(64)
try:
connection.ping(nonce=nonce)
except socket.error as err:
logging.debug("Closing {} ({})".format(node, err))
break
last_ping = time.time()
key = "ping:{}-{}:{}".format(node[0], node[1], nonce)
redis_pipe.lpush(key, int(last_ping * 1000)) # in ms
redis_pipe.expire(key, SETTINGS['ttl'])
redis_pipe.execute()
# Sink received messages to flush them off socket buffer
try:
connection.get_messages()
except socket.timeout as err:
pass
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("Closing {} ({})".format(node, err))
break
gevent.sleep(0.3)
connection.close()
REDIS_CONN.srem('open', node)
REDIS_CONN.srem('opendata', data)
def keepalive(connection, version_msg):
"""
Periodically sends a ping message to the specified node to maintain open
connection. Open connections are tracked in open set with the associated
data stored in opendata set in Redis.
"""
node = connection.to_addr
version = version_msg.get('version', "")
user_agent = version_msg.get('user_agent', "")
now = int(time.time())
data = node + (version, user_agent, now)
REDIS_CONN.sadd('open', node)
REDIS_CONN.sadd('opendata', data)
redis_pipe = REDIS_CONN.pipeline()
last_ping = now
while True:
try:
wait = int(REDIS_CONN.get('elapsed'))
except TypeError as err:
wait = 60
if time.time() > last_ping + wait:
nonce = random.getrandbits(64)
try:
connection.ping(nonce=nonce)
except socket.error as err:
logging.debug("Closing {} ({})".format(node, err))
break
last_ping = time.time()
key = "ping:{}-{}:{}".format(node[0], node[1], nonce)
redis_pipe.lpush(key, int(last_ping * 1000)) # in ms
redis_pipe.expire(key, SETTINGS['ttl'])
redis_pipe.execute()
# Sink received messages to flush them off socket buffer
try:
connection.get_messages()
except socket.timeout as err:
pass
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("Closing {} ({})".format(node, err))
break
gevent.sleep(0.3)
connection.close()
REDIS_CONN.srem('open', node)
REDIS_CONN.srem('opendata', data)
def keepalive(connection, version_msg):
"""
Periodically sends a ping message to the specified node to maintain open
connection. Open connections are tracked in open set with the associated
data stored in opendata set in Redis.
"""
node = connection.to_addr
version = version_msg.get('version', "")
user_agent = version_msg.get('user_agent', "")
now = int(time.time())
data = node + (version, user_agent, now)
REDIS_CONN.sadd('open', node)
REDIS_CONN.sadd('opendata', data)
last_ping = now
while True:
try:
ttl = int(REDIS_CONN.get('elapsed'))
except TypeError as err:
ttl = 60
if time.time() > last_ping + ttl:
try:
connection.ping()
except socket.error as err:
logging.debug("Closing {} ({})".format(node, err))
break
last_ping = time.time()
try:
msgs = connection.get_messages(commands=["ping"])
except socket.timeout as err:
pass
except (ProtocolError, socket.error) as err:
logging.debug("Closing {} ({})".format(node, err))
break
else:
for msg in msgs:
logging.debug("Pong {} ({})".format(node, msg['nonce']))
connection.pong(msg['nonce'])
gevent.sleep(0.3)
connection.close()
REDIS_CONN.srem('open', node)
REDIS_CONN.srem('opendata', data)
def connect(redis_conn, key):
"""
Establishes connection with a node to:
1) Send version message
2) Receive version and verack message
3) Send getaddr message
4) Receive addr message containing list of peering nodes
Stores state and height for node in Redis.
"""
handshake_msgs = []
addr_msgs = []
redis_conn.hset(key, 'state', "") # Set Redis hash for a new node
(address, port) = key[5:].split("-", 1)
height = redis_conn.get('height')
if height is None:
height = 0
else:
height = int(height)
connection = Connection((address, int(port)),
socket_timeout=SETTINGS['socket_timeout'],
user_agent=SETTINGS['user_agent'],
height=height)
try:
logging.debug("Connecting to {}".format(connection.to_addr))
connection.open()
handshake_msgs = connection.handshake()
addr_msgs = connection.getaddr()
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("{}: {}".format(connection.to_addr, err))
finally:
connection.close()
gevent.sleep(0.3)
redis_pipe = redis_conn.pipeline()
if len(handshake_msgs) > 0:
height_key = "height:{}-{}".format(address, port)
redis_pipe.setex(height_key, SETTINGS['max_age'],
handshake_msgs[0].get('height', 0))
now = int(time.time())
peers = enumerate_node(redis_pipe, addr_msgs, now)
logging.debug("{} Peers: {}".format(connection.to_addr, peers))
redis_pipe.hset(key, 'state', "up")
redis_pipe.execute()
def connect(redis_conn, key):
"""
Establishes connection with a node to:
1) Send version message
2) Receive version and verack message
3) Send getaddr message
4) Receive addr message containing list of peering nodes
Stores state and height for node in Redis.
"""
handshake_msgs = []
addr_msgs = []
redis_conn.hset(key, 'state', "") # Set Redis hash for a new node
(address, port) = key[5:].split("-", 1)
height = redis_conn.get('height')
if height is None:
height = 0
else:
height = int(height)
connection = Connection((address, int(port)),
socket_timeout=SETTINGS['socket_timeout'],
user_agent=SETTINGS['user_agent'],
height=height)
try:
logging.debug("Connecting to {}".format(connection.to_addr))
connection.open()
handshake_msgs = connection.handshake()
addr_msgs = connection.getaddr()
except (ProtocolError, ConnectionError, socket.error) as err:
logging.debug("{}: {}".format(connection.to_addr, err))
finally:
connection.close()
gevent.sleep(0.3)
redis_pipe = redis_conn.pipeline()
if len(handshake_msgs) > 0:
height_key = "height:{}-{}".format(address, port)
redis_pipe.setex(height_key, SETTINGS['max_age'],
handshake_msgs[0].get('height', 0))
now = int(time.time())
peers = enumerate_node(redis_pipe, addr_msgs, now)
logging.debug("{} Peers: {}".format(connection.to_addr, peers))
redis_pipe.hset(key, 'state', "up")
redis_pipe.execute()
def keepalive(connection, version_msg):
"""
Periodically sends a ping message to the specified node to maintain open
connection. Open connections are tracked in open set with the associated
data stored in opendata set in Redis.
"""
node = connection.to_addr
version = version_msg.get('version', "")
user_agent = version_msg.get('user_agent', "")
now = int(time.time())
data = node + (version, user_agent, now)
REDIS_CONN.sadd('open', node)
REDIS_CONN.sadd('opendata', data)
last_ping = now
while True:
try:
ttl = int(REDIS_CONN.get('elapsed'))
except TypeError as err:
ttl = 60
if time.time() > last_ping + ttl:
try:
connection.ping()
except socket.error as err:
logging.debug("Closing {} ({})".format(node, err))
break
last_ping = time.time()
# Sink received messages to flush them off socket buffer
try:
connection.get_messages()
except socket.timeout as err:
pass
except (ProtocolError, socket.error) as err:
logging.debug("Closing {} ({})".format(node, err))
break
gevent.sleep(0.3)
connection.close()
REDIS_CONN.srem('open', node)
REDIS_CONN.srem('opendata', data)
def keepalive(connection, version_msg):
"""
Periodically sends a ping message to the specified node to maintain open
connection and yields received inv messages. Open connections are tracked
in open set with the associated data stored in opendata set in Redis.
"""
node = connection.to_addr
version = version_msg.get('version', "")
user_agent = version_msg.get('user_agent', "")
start_height = version_msg.get('start_height', 0)
now = int(time.time())
data = node + (version, user_agent, now)
REDIS_CONN.sadd('open', node)
REDIS_CONN.sadd('opendata', data)
start_height_key = "start_height:{}-{}".format(node[0], node[1])
start_height_ttl = SETTINGS['keepalive'] * 2 # > keepalive
REDIS_CONN.setex(start_height_key, start_height_ttl, start_height)
last_ping = now
while True:
if time.time() > last_ping + SETTINGS['keepalive']:
REDIS_CONN.expire(start_height_key, start_height_ttl)
try:
logging.debug("Ping: {}".format(node))
connection.ping()
except socket.error as err:
logging.debug("Closing {} ({})".format(node, err))
break
last_ping = time.time()
try:
msgs = connection.get_messages(commands=["inv"])
except socket.timeout as err:
logging.debug("{}: {}".format(node, err))
except (ProtocolError, socket.error) as err:
logging.debug("Closing {} ({})".format(node, err))
break
else:
yield msgs
gevent.sleep()
connection.close()
REDIS_CONN.srem('open', node)
REDIS_CONN.srem('opendata', data)
def connect(redis_conn, key):
"""
Establishes connection with a node to:
1) Send version message
2) Receive version and verack message
3) Send getaddr message
4) Receive addr message containing list of peering nodes
Stores node in Redis.
"""
handshake_msgs = []
addr_msgs = []
redis_conn.hset(key, TAG_FIELD, "") # Set Redis hash for a new node
(address, port) = key[5:].split("-", 1)
start_height = int(redis_conn.get('start_height'))
connection = Connection((address, int(port)),
socket_timeout=SETTINGS['socket_timeout'],
user_agent=SETTINGS['user_agent'],
start_height=start_height)
try:
connection.open()
handshake_msgs = connection.handshake()
addr_msgs = connection.getaddr()
except (ProtocolError, socket.error) as err:
logging.debug("{}: {}".format(connection.to_addr, err))
finally:
connection.close()
redis_pipe = redis_conn.pipeline()
if len(handshake_msgs) > 0:
start_height_key = "start_height:{}-{}".format(address, port)
version_msg = handshake_msgs[0]
peers = enumerate_node(redis_pipe, start_height_key, version_msg,
addr_msgs)
logging.debug("[{}] Peers: {}".format(connection.to_addr, peers))
redis_pipe.hset(key, TAG_FIELD, GREEN)
redis_pipe.execute()
def connect(redis_conn, key):
"""
Establishes connection with a node to:
1) Send version message
2) Receive version and verack message
3) Send getaddr message
4) Receive addr message containing list of peering nodes
Stores node in Redis.
"""
handshake_msgs = []
addr_msg = {}
redis_conn.hset(key, TAG_FIELD, "") # Set Redis hash for a new node
(address, port) = key[5:].split("-", 1)
start_height = int(redis_conn.get('start_height'))
connection = Connection((address, int(port)),
socket_timeout=SETTINGS['socket_timeout'],
user_agent=SETTINGS['user_agent'],
start_height=start_height)
try:
connection.open()
handshake_msgs = connection.handshake()
addr_msg = connection.getaddr()
except ProtocolError as err:
logging.debug("{}".format(err))
except socket.error as err:
logging.debug("{}".format(err))
finally:
connection.close()
redis_pipe = redis_conn.pipeline()
if len(handshake_msgs) > 0:
addNode(handshake_msgs[0])
enumerate_node(redis_pipe, key, handshake_msgs[0], addr_msg)
redis_pipe.hset(key, TAG_FIELD, GREEN)
redis_pipe.execute()
def task():
"""
Assigned to a worker to retrieve (pop) a node from the reachable set and
attempt to establish and maintain connection with the node.
"""
node = REDIS_CONN.spop('reachable')
(address, port, start_height) = eval(node)
handshake_msgs = []
connection = Connection((address, port),
socket_timeout=SETTINGS['socket_timeout'],
user_agent=SETTINGS['user_agent'],
start_height=start_height)
try:
connection.open()
handshake_msgs = connection.handshake()
except ProtocolError as err:
connection.close()
except socket.error as err:
connection.close()
if len(handshake_msgs) > 0:
keepalive(connection, handshake_msgs[0])
def task():
"""
Assigned to a worker to retrieve (pop) a node from the reachable set and
attempt to establish and maintain connection with the node.
"""
node = REDIS_CONN.spop('reachable')
(address, port, start_height) = eval(node)
handshake_msgs = []
connection = Connection((address, port),
socket_timeout=SETTINGS['socket_timeout'],
user_agent=SETTINGS['user_agent'],
start_height=start_height)
try:
connection.open()
handshake_msgs = connection.handshake()
except ProtocolError as err:
connection.close()
except socket.error as err:
connection.close()
if len(handshake_msgs) > 0:
keepalive(connection, handshake_msgs[0])
