def processReceived(self, limit):
if limit <= 0:
return 0
num_processed = 0
for num_processed in range(limit):
if len(self.rxMsgs) == 0:
return num_processed
msg, ident = self.rxMsgs.popleft()
frm = self.remotesByKeys[ident].name \
if ident in self.remotesByKeys else ident
if self.handlePingPong(msg, frm, ident):
continue
if not self.onlyListener and ident not in self.remotesByKeys:
logger.warning('{} received message from unknown remote {}'
.format(self, z85_to_friendly(ident)))
continue
try:
msg = self.deserializeMsg(msg)
except Exception as e:
logger.error('Error {} while converting message {} '
'to JSON from {}'.format(e, msg, z85_to_friendly(ident)))
continue
msg = self.doProcessReceived(msg, frm, ident)
if msg:
self.msgHandler((msg, frm))
return num_processed + 1
def processReceived(self, limit):
if limit <= 0:
return 0
num_processed = 0
for num_processed in range(limit):
if len(self.rxMsgs) == 0:
return num_processed
msg, ident = self.rxMsgs.popleft()
frm = self.remotesByKeys[ident].name \
if ident in self.remotesByKeys else ident
if self.handlePingPong(msg, frm, ident):
continue
if not self.onlyListener and ident not in self.remotesByKeys:
logger.warning(
'{} received message from unknown remote {}'.format(
self, z85_to_friendly(ident)))
continue
try:
msg = self.deserializeMsg(msg)
except Exception as e:
logger.error('Error {} while converting message {} '
'to JSON from {}'.format(e, msg,
z85_to_friendly(ident)))
continue
# We have received non-ping-pong message from some remote, we can clean this counter
if OP_FIELD_NAME not in msg or msg[OP_FIELD_NAME] != BATCH:
self.remote_ping_stats[z85_to_friendly(frm)] = 0
msg = self.doProcessReceived(msg, frm, ident)
if msg:
self.msgHandler((msg, frm))
return num_processed + 1
def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False):
remote = self.remotes.get(uid)
err_str = None
if not remote:
logger.debug("Remote {} does not exist!".format(z85_to_friendly(uid)))
return False, err_str
socket = remote.socket
if not socket:
logger.debug('{} has uninitialised socket '
'for remote {}'.format(self, z85_to_friendly(uid)))
return False, err_str
try:
if not serialized:
msg = self.prepare_to_send(msg)
logger.trace('{} transmitting message {} to {} by socket {} {}'
.format(self, msg, z85_to_friendly(uid), socket.FD, socket.underlying))
socket.send(msg, flags=zmq.NOBLOCK)
if remote.isConnected or msg in self.healthMessages:
self.metrics.add_event(self.mt_outgoing_size, len(msg))
else:
logger.debug('Remote {} is not connected - message will not be sent immediately.'
'If this problem does not resolve itself - check your firewall settings'
.format(z85_to_friendly(uid)))
return True, err_str
except zmq.Again:
logger.warning('{} could not transmit message to {}'.format(self, z85_to_friendly(uid)))
except InvalidMessageExceedingSizeException as ex:
err_str = '{}Cannot transmit message. Error {}'.format(CONNECTION_PREFIX, ex)
logger.warning(err_str)
return False, err_str
def processReceived(self, limit):
if limit <= 0:
return 0
num_processed = 0
for num_processed in range(limit):
if len(self.rxMsgs) == 0:
return num_processed
msg, ident = self.rxMsgs.popleft()
frm = self.remotesByKeys[ident].name \
if ident in self.remotesByKeys else ident
if self.handlePingPong(msg, frm, ident):
continue
if not self.onlyListener and ident not in self.remotesByKeys:
logger.warning('{} received message from unknown remote {}'
.format(self, z85_to_friendly(ident)))
continue
try:
msg = self.deserializeMsg(msg)
except Exception as e:
logger.error('Error {} while converting message {} '
'to JSON from {}'.format(e, msg, z85_to_friendly(ident)))
continue
msg = self.doProcessReceived(msg, frm, ident)
if msg:
self.msgHandler((msg, frm))
return num_processed + 1
def handlePingPong(self, msg, frm, ident):
if msg in (self.pingMessage, self.pongMessage):
if msg == self.pingMessage:
nodeName = z85_to_friendly(frm)
logger.trace('{} got ping from {}'.format(self, nodeName))
self.sendPingPong(frm, is_ping=False)
if not self.config.ENABLE_HEARTBEATS and self.config.PING_RECONNECT_ENABLED and nodeName in self.connecteds:
if self.remote_ping_stats.get(nodeName):
self.remote_ping_stats[nodeName] += 1
else:
self.remote_ping_stats[nodeName] = 1
if self.remote_ping_stats[
nodeName] > self.config.PINGS_BEFORE_SOCKET_RECONNECTION:
logger.info(
"Reconnecting {} due to numerous consecutive pings"
.format(nodeName))
self.remote_ping_stats[nodeName] = 0
self.reconnectRemoteWithName(nodeName)
if msg == self.pongMessage:
if ident in self.remotesByKeys:
self.remotesByKeys[ident].setConnected()
self._resend_to_disconnected(frm, ident)
logger.trace('{} got pong from {}'.format(
self, z85_to_friendly(frm)))
return True
return False
def flushOutBoxes(self) -> None:
"""
Clear the outBoxes and transmit batched messages to remotes.
"""
removedRemotes = []
for rid, msgs in self.outBoxes.items():
try:
dest = self.remotes[rid].name
except KeyError:
removedRemotes.append(rid)
continue
if msgs:
if self._should_batch(msgs):
logger.trace(
"{} batching {} msgs to {} into fewer transmissions".
format(self, len(msgs), dest))
logger.trace(" messages: {}".format(msgs))
batches = split_messages_on_batches(list(msgs),
self._make_batch,
self._test_batch_len,
)
msgs.clear()
if batches:
for batch, size in batches:
logger.trace("{} sending payload to {}: {}".format(
self, dest, batch))
self.metrics.add_event(MetricsName.TRANSPORT_BATCH_SIZE, size)
# Setting timeout to never expire
self.transmit(
batch,
rid,
timeout=self.messageTimeout,
serialized=True,
is_batch=True
)
else:
logger.error("{} cannot create batch(es) for {}".format(self, dest))
else:
while msgs:
msg = msgs.popleft()
logger.trace(
"{} sending msg {} to {}".format(self, msg, dest))
self.metrics.add_event(MetricsName.TRANSPORT_BATCH_SIZE, 1)
# Setting timeout to never expire
self.transmit(msg, rid, timeout=self.messageTimeout,
serialized=True, is_batch=False)
for rid in removedRemotes:
logger.info("{}{} has removed rid {}".
format(CONNECTION_PREFIX, self, z85_to_friendly(rid)), extra={"cli": False})
msgs = self.outBoxes[rid]
if msgs:
self.discard(msgs,
"{}rid {} no longer available"
.format(CONNECTION_PREFIX,
z85_to_friendly(rid)),
logMethod=logger.debug)
del self.outBoxes[rid]
def flushOutBoxes(self) -> None:
"""
Clear the outBoxes and transmit batched messages to remotes.
"""
removedRemotes = []
for rid, msgs in self.outBoxes.items():
try:
dest = self.remotes[rid].name
except KeyError:
removedRemotes.append(rid)
continue
if msgs:
if self._should_batch(msgs):
logger.trace(
"{} batching {} msgs to {} into fewer transmissions".
format(self, len(msgs), dest))
logger.trace(" messages: {}".format(msgs))
batches = split_messages_on_batches(list(msgs),
self._make_batch,
self._test_batch_len,
)
msgs.clear()
if batches:
for batch, size in batches:
logger.trace("{} sending payload to {}: {}".format(
self, dest, batch))
self.metrics.add_event(MetricsName.TRANSPORT_BATCH_SIZE, size)
# Setting timeout to never expire
self.transmit(
batch,
rid,
timeout=self.messageTimeout,
serialized=True)
else:
logger.error("{} cannot create batch(es) for {}".format(self, dest))
else:
while msgs:
msg = msgs.popleft()
logger.trace(
"{} sending msg {} to {}".format(self, msg, dest))
self.metrics.add_event(MetricsName.TRANSPORT_BATCH_SIZE, 1)
# Setting timeout to never expire
self.transmit(msg, rid, timeout=self.messageTimeout,
serialized=True)
for rid in removedRemotes:
logger.warning("{}{} has removed rid {}"
.format(CONNECTION_PREFIX, self,
z85_to_friendly(rid)),
extra={"cli": False})
msgs = self.outBoxes[rid]
if msgs:
self.discard(msgs,
"{}rid {} no longer available"
.format(CONNECTION_PREFIX,
z85_to_friendly(rid)),
logMethod=logger.debug)
del self.outBoxes[rid]
def handlePingPong(self, msg, frm, ident):
if msg in (self.pingMessage, self.pongMessage):
if msg == self.pingMessage:
logger.trace('{} got ping from {}'.format(self, z85_to_friendly(frm)))
self.sendPingPong(frm, is_ping=False)
if msg == self.pongMessage:
if ident in self.remotesByKeys:
self.remotesByKeys[ident].setConnected()
self._resend_to_disconnected(frm, ident)
logger.trace('{} got pong from {}'.format(self, z85_to_friendly(frm)))
return True
return False
def handlePingPong(self, msg, frm, ident):
if msg in (self.pingMessage, self.pongMessage):
if msg == self.pingMessage:
logger.trace('{} got ping from {}'.format(self, z85_to_friendly(frm)))
self.sendPingPong(frm, is_ping=False)
if msg == self.pongMessage:
if ident in self.remotesByKeys:
self.remotesByKeys[ident].setConnected()
self._resend_to_disconnected(frm, ident)
logger.trace('{} got pong from {}'.format(self, z85_to_friendly(frm)))
return True
return False
def _receiveFromRemotes(self, quotaPerRemote) -> int:
"""
Receives messages from remotes
:param quotaPerRemote: number of messages to receive from one remote
:return: number of received messages
"""
assert quotaPerRemote
totalReceived = 0
for ident, remote in self.remotesByKeys.items():
if not remote.socket:
continue
i = 0
sock = remote.socket
while i < quotaPerRemote:
try:
msg, = sock.recv_multipart(flags=zmq.NOBLOCK)
if not msg:
# Router probing sends empty message on connection
continue
i += 1
logger.trace("{} received a message from remote {} by socket {} {}", self,
z85_to_friendly(ident), sock.FD, sock.underlying)
self._verifyAndAppend(msg, ident)
except zmq.Again as e:
break
except zmq.ZMQError as e:
logger.debug(
"Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e))
if i > 0:
logger.trace('{} got {} messages through remote {}'.
format(self, i, remote))
totalReceived += i
return totalReceived
def _resend_to_disconnected(self, to, ident):
if not self._can_resend_to_disconnected(to, ident):
return
logger.trace('{} resending stashed messages to {}'.format(self, z85_to_friendly(to)))
msgs = self._stashed_to_disconnected[to]
while msgs:
msg = msgs.popleft()
ZStack.send(self, msg, to)
def _resend_to_disconnected(self, to, ident):
if not self._can_resend_to_disconnected(to, ident):
return
logger.trace('{} resending stashed messages to {}'.format(self, z85_to_friendly(to)))
msgs = self._stashed_to_disconnected[to]
while msgs:
msg = msgs.popleft()
ZStack.send(self, msg, to)
def sendPingPong(self, remote: Union[str, Remote], is_ping=True):
msg = self.pingMessage if is_ping else self.pongMessage
action = 'ping' if is_ping else 'pong'
name = remote if isinstance(remote, (str, bytes)) else remote.name
r = self.send(msg, name)
if r[0] is True:
logger.debug('{} {}ed {}'.format(self.name, action, z85_to_friendly(name)))
elif r[0] is False:
logger.debug('{} failed to {} {} {}'
.format(self.name, action, z85_to_friendly(name), r[1]),
extra={"cli": False})
# try to re-send pongs later
if not is_ping:
self._stashed_pongs.add(name)
elif r[0] is None:
logger.debug('{} will be sending in batch'.format(self))
else:
logger.error('{}{} got an unexpected return value {} while sending'.
format(CONNECTION_PREFIX, self, r))
return r[0]
def sendPingPong(self, remote: Union[str, Remote], is_ping=True):
msg = self.pingMessage if is_ping else self.pongMessage
action = 'ping' if is_ping else 'pong'
name = remote if isinstance(remote, (str, bytes)) else remote.name
r = self.send(msg, name)
if r[0] is True:
logger.debug('{} {}ed {}'.format(self.name, action, z85_to_friendly(name)))
elif r[0] is False:
logger.debug('{} failed to {} {} {}'
.format(self.name, action, z85_to_friendly(name), r[1]),
extra={"cli": False})
# try to re-send pongs later
if not is_ping:
self._stashed_pongs.add(name)
elif r[0] is None:
logger.debug('{} will be sending in batch'.format(self))
else:
logger.error('{}{} got an unexpected return value {} while sending'.
format(CONNECTION_PREFIX, self, r))
return r[0]
def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False):
remote = self.remotes.get(uid)
err_str = None
if not remote:
logger.debug("Remote {} does not exist!".format(z85_to_friendly(uid)))
return False, err_str
socket = remote.socket
if not socket:
logger.debug('{} has uninitialised socket '
'for remote {}'.format(self, z85_to_friendly(uid)))
return False, err_str
try:
if not serialized:
msg = self.prepare_to_send(msg)
logger.trace('{} transmitting message {} to {} by socket {} {}'
.format(self, msg, z85_to_friendly(uid), socket.FD, socket.underlying))
socket.send(msg, flags=zmq.NOBLOCK)
if remote.isConnected or msg in self.healthMessages:
self.metrics.add_event(self.mt_outgoing_size, len(msg))
else:
logger.warning('Remote {} is not connected - message will not be sent immediately.'
'If this problem does not resolve itself - check your firewall settings'
.format(z85_to_friendly(uid)))
# We should not stash ping/pongs as this may lead to incorrect reconnection logic
# (replying by old pongs for new connection and masking connection issues)
# TODO: since we can not remove ping/pongs from Batches at this phase, just do not stash Batches at all
if not is_batch:
self._stashed_to_disconnected \
.setdefault(uid, deque(maxlen=self.config.ZMQ_STASH_TO_NOT_CONNECTED_QUEUE_SIZE)) \
.append(msg)
return True, err_str
except zmq.Again:
logger.warning('{} could not transmit message to {}'.format(self, z85_to_friendly(uid)))
except InvalidMessageExceedingSizeException as ex:
err_str = '{}Cannot transmit message. Error {}'.format(CONNECTION_PREFIX, ex)
logger.warning(err_str)
return False, err_str
def _verifyAndAppend(self, msg, ident):
try:
self.metrics.add_event(self.mt_incoming_size, len(msg))
self.msgLenVal.validate(msg)
decoded = msg.decode()
except (UnicodeDecodeError, InvalidMessageExceedingSizeException) as ex:
errstr = 'Message will be discarded due to {}'.format(ex)
frm = self.remotesByKeys[ident].name if ident in self.remotesByKeys else ident
logger.error("Got from {} {}".format(z85_to_friendly(frm), errstr))
self.msgRejectHandler(errstr, frm)
return False
self.rxMsgs.append((decoded, ident))
return True
def _verifyAndAppend(self, msg, ident):
try:
self.metrics.add_event(self.mt_incoming_size, len(msg))
self.msgLenVal.validate(msg)
decoded = msg.decode()
except (UnicodeDecodeError, InvalidMessageExceedingSizeException) as ex:
errstr = 'Message will be discarded due to {}'.format(ex)
frm = self.remotesByKeys[ident].name if ident in self.remotesByKeys else ident
logger.error("Got from {} {}".format(z85_to_friendly(frm), errstr))
self.msgRejectHandler(errstr, frm)
return False
self.rxMsgs.append((decoded, ident))
return True
def connect(self,
name=None,
remoteId=None,
ha=None,
verKeyRaw=None,
publicKeyRaw=None):
"""
Connect to the node specified by name.
"""
if not name:
raise ValueError('Remote name should be specified')
publicKey = None
if name in self.remotes:
remote = self.remotes[name]
else:
publicKey = z85.encode(
publicKeyRaw) if publicKeyRaw else self.getPublicKey(name)
verKey = z85.encode(verKeyRaw) if verKeyRaw else self.getVerKey(
name)
if not ha or not publicKey or (self.isRestricted and not verKey):
raise ValueError(
'{} doesnt have enough info to connect. '
'Need ha, public key and verkey. {} {} {}'.format(
name, ha, verKey, publicKey))
remote = self.addRemote(name, ha, verKey, publicKey)
public, secret = self.selfEncKeys
remote.connect(self.ctx, public, secret)
logger.info("{}{} looking for {} at {}:{}".format(
CONNECTION_PREFIX, self, name or remote.name, *remote.ha),
extra={
"cli": "PLAIN",
"tags": ["node-looking"]
})
# This should be scheduled as an async task
self.sendPingPong(remote, is_ping=True)
# re-send previously stashed pings/pongs from unknown remotes
logger.trace("{} stashed pongs: {}".format(self.name,
str(self._stashed_pongs)))
if publicKey in self._stashed_pongs:
logger.trace("{} sending stashed pongs to {}".format(
self.name, str(z85_to_friendly(publicKey))))
self._stashed_pongs.discard(publicKey)
self.sendPingPong(name, is_ping=False)
return remote.uid
def transmit(self, msg, uid, timeout=None, serialized=False):
remote = self.remotes.get(uid)
err_str = None
if not remote:
logger.debug("Remote {} does not exist!".format(z85_to_friendly(uid)))
return False, err_str
socket = remote.socket
if not socket:
logger.debug('{} has uninitialised socket '
'for remote {}'.format(self, z85_to_friendly(uid)))
return False, err_str
try:
if not serialized:
msg = self.prepare_to_send(msg)
logger.trace('{} transmitting message {} to {} by socket {} {}'
.format(self, msg, z85_to_friendly(uid), socket.FD, socket.underlying))
socket.send(msg, flags=zmq.NOBLOCK)
if remote.isConnected or msg in self.healthMessages:
self.metrics.add_event(self.mt_outgoing_size, len(msg))
else:
logger.warning('Remote {} is not connected - message will not be sent immediately.'
'If this problem does not resolve itself - check your firewall settings'
.format(z85_to_friendly(uid)))
self._stashed_to_disconnected \
.setdefault(uid, deque(maxlen=self.config.ZMQ_STASH_TO_NOT_CONNECTED_QUEUE_SIZE)) \
.append(msg)
return True, err_str
except zmq.Again:
logger.warning('{} could not transmit message to {}'.format(self, z85_to_friendly(uid)))
except InvalidMessageExceedingSizeException as ex:
err_str = '{}Cannot transmit message. Error {}'.format(CONNECTION_PREFIX, ex)
logger.warning(err_str)
return False, err_str
def connect(self,
name=None,
remoteId=None,
ha=None,
verKeyRaw=None,
publicKeyRaw=None):
"""
Connect to the node specified by name.
"""
if not name:
raise ValueError('Remote name should be specified')
publicKey = None
if name in self.remotes:
remote = self.remotes[name]
else:
publicKey = z85.encode(
publicKeyRaw) if publicKeyRaw else self.getPublicKey(name)
verKey = z85.encode(
verKeyRaw) if verKeyRaw else self.getVerKey(name)
if not ha or not publicKey or (self.isRestricted and not verKey):
raise ValueError('{} doesnt have enough info to connect. '
'Need ha, public key and verkey. {} {} {}'.
format(name, ha, verKey, publicKey))
remote = self.addRemote(name, ha, verKey, publicKey)
public, secret = self.selfEncKeys
remote.connect(self.ctx, public, secret)
logger.info("{}{} looking for {} at {}:{}"
.format(CONNECTION_PREFIX, self,
name or remote.name, *remote.ha),
extra={"cli": "PLAIN", "tags": ["node-looking"]})
# This should be scheduled as an async task
self.sendPingPong(remote, is_ping=True)
# re-send previously stashed pings/pongs from unknown remotes
logger.trace("{} stashed pongs: {}".format(self.name, str(self._stashed_pongs)))
if publicKey in self._stashed_pongs:
logger.trace("{} sending stashed pongs to {}".format(self.name, str(z85_to_friendly(publicKey))))
self._stashed_pongs.discard(publicKey)
self.sendPingPong(name, is_ping=False)
return remote.uid