def _check_for_consecutive_disconnects(self, error_name):
now = datetime.datetime.utcnow().timestamp()
FIVE_MINUTES = 5 * 60
if self.address.last_connection != 0 and now - self.address.last_connection < FIVE_MINUTES:
self.leader.AddDeadAddress(self.address, reason=f"{self.prefix} second {error_name} within 5 minutes")
else:
self.address.last_connection = Address.Now()
def conn_setup(node: NeoNode):
with self.assertLogHandler('network', 10) as log:
# setup last_connection, to indicate we've lost connection before
node.address.last_connection = Address.Now(
) # returns a timestamp of utcnow()
# setup the heartbeat data to have last happened 25 seconds ago
# if we disconnect now we should get a premature disconnect
node.start_outstanding_data_request[
HEARTBEAT_BLOCKS] = Address.Now() - 25
# now lose the connection
node.connectionLost(failure.Failure(error.ConnectionLost()))
self.assertIn("Premature disconnect", log.output[-2])
self.assertIn(str(error.ConnectionLost()), log.output[-1])
self.assertIn(self.addr, self.leader.DEAD_ADDRS)
self.assertNotIn(self.addr, self.leader.KNOWN_ADDRS)
self.assertNotIn(self.addr, self.leader.Peers)
self.assertFalse(node.has_tasks_running())
def connectionLost(self, reason=None):
"""Callback handler from twisted when a connection was lost."""
try:
self.connected = False
self.stop_block_loop()
self.stop_peerinfo_loop()
self.stop_header_loop()
self.ReleaseBlockRequests()
self.leader.RemoveConnectedPeer(self)
time_expired = self.time_expired(HEARTBEAT_BLOCKS)
# some NEO-cli versions have a 30s timeout to receive block/consensus or tx messages. By default neo-python doesn't respond to these requests
if time_expired > 20:
self.address.last_connection = Address.Now()
self.leader.AddDeadAddress(
self.address, reason=f"{self.prefix} Premature disconnect")
if reason and reason.check(twisted_error.ConnectionDone):
# this might happen if they close our connection because they've reached max peers or something similar
logger.debug(
f"{self.prefix} disconnected normally with reason:{reason.value}"
)
self._check_for_consecutive_disconnects("connection done")
elif reason and reason.check(twisted_error.ConnectionLost):
# Can be due to a timeout. Only if this happened again within 5 minutes do we label the node as bad
# because then it clearly doesn't want to talk to us or we have a bad connection to them.
# Otherwise allow for the node to be queued again by NodeLeader.
logger.debug(
f"{self.prefix} disconnected with connectionlost reason: {reason.value}"
)
self._check_for_consecutive_disconnects("connection lost")
else:
logger.debug(
f"{self.prefix} disconnected with reason: {reason.value}")
except Exception as e:
logger.error("Error with connection lost: %s " % e)
def try_me(err):
err.check(error.ConnectionAborted)
if self.disconnect_deferred:
d, self.disconnect_deferred = self.disconnect_deferred, None # type: defer.Deferred
d.addErrback(try_me)
if len(d.callbacks) > 0:
d.callback(reason)
else:
print("connLost, disconnect_deferred cancelling!")
d.cancel()
def conn_setup(node: NeoNode):
# at this point we should have a fully connected node, so lets try to simulate a connection lost by the other side
with self.assertLogHandler('network', 10) as log:
# setup last_connection, to indicate we've lost connection before
node.address.last_connection = Address.Now(
) # returns a timestamp of utcnow()
# now lose the connection
node.connectionLost(failure.Failure(error.ConnectionLost()))
self.assertIn("second connection lost within 5 minutes",
log.output[-1])
self.assertIn(str(error.ConnectionLost()), log.output[-2])
self.assertIn(self.addr, self.leader.DEAD_ADDRS)
self.assertNotIn(self.addr, self.leader.KNOWN_ADDRS)
self.assertNotIn(self.addr, self.leader.Peers)
self.assertFalse(node.has_tasks_running())
def test_now_helper(self):
n = Address.Now()
delta = datetime.now().utcnow().timestamp() - n
self.assertTrue(delta < 2)