def processReceived(self, limit): if limit <= 0: return 0 num_processed = 0 for num_processed in range(limit): if len(self.rxMsgs) == 0: return num_processed msg, ident = self.rxMsgs.popleft() frm = self.remotesByKeys[ident].name \ if ident in self.remotesByKeys else ident if self.handlePingPong(msg, frm, ident): continue if not self.onlyListener and ident not in self.remotesByKeys: logger.warning('{} received message from unknown remote {}' .format(self, z85_to_friendly(ident))) continue try: msg = self.deserializeMsg(msg) except Exception as e: logger.error('Error {} while converting message {} ' 'to JSON from {}'.format(e, msg, z85_to_friendly(ident))) continue msg = self.doProcessReceived(msg, frm, ident) if msg: self.msgHandler((msg, frm)) return num_processed + 1
def processReceived(self, limit): if limit <= 0: return 0 num_processed = 0 for num_processed in range(limit): if len(self.rxMsgs) == 0: return num_processed msg, ident = self.rxMsgs.popleft() frm = self.remotesByKeys[ident].name \ if ident in self.remotesByKeys else ident if self.handlePingPong(msg, frm, ident): continue if not self.onlyListener and ident not in self.remotesByKeys: logger.warning( '{} received message from unknown remote {}'.format( self, z85_to_friendly(ident))) continue try: msg = self.deserializeMsg(msg) except Exception as e: logger.error('Error {} while converting message {} ' 'to JSON from {}'.format(e, msg, z85_to_friendly(ident))) continue # We have received non-ping-pong message from some remote, we can clean this counter if OP_FIELD_NAME not in msg or msg[OP_FIELD_NAME] != BATCH: self.remote_ping_stats[z85_to_friendly(frm)] = 0 msg = self.doProcessReceived(msg, frm, ident) if msg: self.msgHandler((msg, frm)) return num_processed + 1
def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False): remote = self.remotes.get(uid) err_str = None if not remote: logger.debug("Remote {} does not exist!".format(z85_to_friendly(uid))) return False, err_str socket = remote.socket if not socket: logger.debug('{} has uninitialised socket ' 'for remote {}'.format(self, z85_to_friendly(uid))) return False, err_str try: if not serialized: msg = self.prepare_to_send(msg) logger.trace('{} transmitting message {} to {} by socket {} {}' .format(self, msg, z85_to_friendly(uid), socket.FD, socket.underlying)) socket.send(msg, flags=zmq.NOBLOCK) if remote.isConnected or msg in self.healthMessages: self.metrics.add_event(self.mt_outgoing_size, len(msg)) else: logger.debug('Remote {} is not connected - message will not be sent immediately.' 'If this problem does not resolve itself - check your firewall settings' .format(z85_to_friendly(uid))) return True, err_str except zmq.Again: logger.warning('{} could not transmit message to {}'.format(self, z85_to_friendly(uid))) except InvalidMessageExceedingSizeException as ex: err_str = '{}Cannot transmit message. Error {}'.format(CONNECTION_PREFIX, ex) logger.warning(err_str) return False, err_str
def processReceived(self, limit): if limit <= 0: return 0 num_processed = 0 for num_processed in range(limit): if len(self.rxMsgs) == 0: return num_processed msg, ident = self.rxMsgs.popleft() frm = self.remotesByKeys[ident].name \ if ident in self.remotesByKeys else ident if self.handlePingPong(msg, frm, ident): continue if not self.onlyListener and ident not in self.remotesByKeys: logger.warning('{} received message from unknown remote {}' .format(self, z85_to_friendly(ident))) continue try: msg = self.deserializeMsg(msg) except Exception as e: logger.error('Error {} while converting message {} ' 'to JSON from {}'.format(e, msg, z85_to_friendly(ident))) continue msg = self.doProcessReceived(msg, frm, ident) if msg: self.msgHandler((msg, frm)) return num_processed + 1
def handlePingPong(self, msg, frm, ident): if msg in (self.pingMessage, self.pongMessage): if msg == self.pingMessage: nodeName = z85_to_friendly(frm) logger.trace('{} got ping from {}'.format(self, nodeName)) self.sendPingPong(frm, is_ping=False) if not self.config.ENABLE_HEARTBEATS and self.config.PING_RECONNECT_ENABLED and nodeName in self.connecteds: if self.remote_ping_stats.get(nodeName): self.remote_ping_stats[nodeName] += 1 else: self.remote_ping_stats[nodeName] = 1 if self.remote_ping_stats[ nodeName] > self.config.PINGS_BEFORE_SOCKET_RECONNECTION: logger.info( "Reconnecting {} due to numerous consecutive pings" .format(nodeName)) self.remote_ping_stats[nodeName] = 0 self.reconnectRemoteWithName(nodeName) if msg == self.pongMessage: if ident in self.remotesByKeys: self.remotesByKeys[ident].setConnected() self._resend_to_disconnected(frm, ident) logger.trace('{} got pong from {}'.format( self, z85_to_friendly(frm))) return True return False
def flushOutBoxes(self) -> None: """ Clear the outBoxes and transmit batched messages to remotes. """ removedRemotes = [] for rid, msgs in self.outBoxes.items(): try: dest = self.remotes[rid].name except KeyError: removedRemotes.append(rid) continue if msgs: if self._should_batch(msgs): logger.trace( "{} batching {} msgs to {} into fewer transmissions". format(self, len(msgs), dest)) logger.trace(" messages: {}".format(msgs)) batches = split_messages_on_batches(list(msgs), self._make_batch, self._test_batch_len, ) msgs.clear() if batches: for batch, size in batches: logger.trace("{} sending payload to {}: {}".format( self, dest, batch)) self.metrics.add_event(MetricsName.TRANSPORT_BATCH_SIZE, size) # Setting timeout to never expire self.transmit( batch, rid, timeout=self.messageTimeout, serialized=True, is_batch=True ) else: logger.error("{} cannot create batch(es) for {}".format(self, dest)) else: while msgs: msg = msgs.popleft() logger.trace( "{} sending msg {} to {}".format(self, msg, dest)) self.metrics.add_event(MetricsName.TRANSPORT_BATCH_SIZE, 1) # Setting timeout to never expire self.transmit(msg, rid, timeout=self.messageTimeout, serialized=True, is_batch=False) for rid in removedRemotes: logger.info("{}{} has removed rid {}". format(CONNECTION_PREFIX, self, z85_to_friendly(rid)), extra={"cli": False}) msgs = self.outBoxes[rid] if msgs: self.discard(msgs, "{}rid {} no longer available" .format(CONNECTION_PREFIX, z85_to_friendly(rid)), logMethod=logger.debug) del self.outBoxes[rid]
def flushOutBoxes(self) -> None: """ Clear the outBoxes and transmit batched messages to remotes. """ removedRemotes = [] for rid, msgs in self.outBoxes.items(): try: dest = self.remotes[rid].name except KeyError: removedRemotes.append(rid) continue if msgs: if self._should_batch(msgs): logger.trace( "{} batching {} msgs to {} into fewer transmissions". format(self, len(msgs), dest)) logger.trace(" messages: {}".format(msgs)) batches = split_messages_on_batches(list(msgs), self._make_batch, self._test_batch_len, ) msgs.clear() if batches: for batch, size in batches: logger.trace("{} sending payload to {}: {}".format( self, dest, batch)) self.metrics.add_event(MetricsName.TRANSPORT_BATCH_SIZE, size) # Setting timeout to never expire self.transmit( batch, rid, timeout=self.messageTimeout, serialized=True) else: logger.error("{} cannot create batch(es) for {}".format(self, dest)) else: while msgs: msg = msgs.popleft() logger.trace( "{} sending msg {} to {}".format(self, msg, dest)) self.metrics.add_event(MetricsName.TRANSPORT_BATCH_SIZE, 1) # Setting timeout to never expire self.transmit(msg, rid, timeout=self.messageTimeout, serialized=True) for rid in removedRemotes: logger.warning("{}{} has removed rid {}" .format(CONNECTION_PREFIX, self, z85_to_friendly(rid)), extra={"cli": False}) msgs = self.outBoxes[rid] if msgs: self.discard(msgs, "{}rid {} no longer available" .format(CONNECTION_PREFIX, z85_to_friendly(rid)), logMethod=logger.debug) del self.outBoxes[rid]
def handlePingPong(self, msg, frm, ident): if msg in (self.pingMessage, self.pongMessage): if msg == self.pingMessage: logger.trace('{} got ping from {}'.format(self, z85_to_friendly(frm))) self.sendPingPong(frm, is_ping=False) if msg == self.pongMessage: if ident in self.remotesByKeys: self.remotesByKeys[ident].setConnected() self._resend_to_disconnected(frm, ident) logger.trace('{} got pong from {}'.format(self, z85_to_friendly(frm))) return True return False
def handlePingPong(self, msg, frm, ident): if msg in (self.pingMessage, self.pongMessage): if msg == self.pingMessage: logger.trace('{} got ping from {}'.format(self, z85_to_friendly(frm))) self.sendPingPong(frm, is_ping=False) if msg == self.pongMessage: if ident in self.remotesByKeys: self.remotesByKeys[ident].setConnected() self._resend_to_disconnected(frm, ident) logger.trace('{} got pong from {}'.format(self, z85_to_friendly(frm))) return True return False
def _receiveFromRemotes(self, quotaPerRemote) -> int: """ Receives messages from remotes :param quotaPerRemote: number of messages to receive from one remote :return: number of received messages """ assert quotaPerRemote totalReceived = 0 for ident, remote in self.remotesByKeys.items(): if not remote.socket: continue i = 0 sock = remote.socket while i < quotaPerRemote: try: msg, = sock.recv_multipart(flags=zmq.NOBLOCK) if not msg: # Router probing sends empty message on connection continue i += 1 logger.trace("{} received a message from remote {} by socket {} {}", self, z85_to_friendly(ident), sock.FD, sock.underlying) self._verifyAndAppend(msg, ident) except zmq.Again as e: break except zmq.ZMQError as e: logger.debug( "Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e)) if i > 0: logger.trace('{} got {} messages through remote {}'. format(self, i, remote)) totalReceived += i return totalReceived
def _resend_to_disconnected(self, to, ident): if not self._can_resend_to_disconnected(to, ident): return logger.trace('{} resending stashed messages to {}'.format(self, z85_to_friendly(to))) msgs = self._stashed_to_disconnected[to] while msgs: msg = msgs.popleft() ZStack.send(self, msg, to)
def _resend_to_disconnected(self, to, ident): if not self._can_resend_to_disconnected(to, ident): return logger.trace('{} resending stashed messages to {}'.format(self, z85_to_friendly(to))) msgs = self._stashed_to_disconnected[to] while msgs: msg = msgs.popleft() ZStack.send(self, msg, to)
def sendPingPong(self, remote: Union[str, Remote], is_ping=True): msg = self.pingMessage if is_ping else self.pongMessage action = 'ping' if is_ping else 'pong' name = remote if isinstance(remote, (str, bytes)) else remote.name r = self.send(msg, name) if r[0] is True: logger.debug('{} {}ed {}'.format(self.name, action, z85_to_friendly(name))) elif r[0] is False: logger.debug('{} failed to {} {} {}' .format(self.name, action, z85_to_friendly(name), r[1]), extra={"cli": False}) # try to re-send pongs later if not is_ping: self._stashed_pongs.add(name) elif r[0] is None: logger.debug('{} will be sending in batch'.format(self)) else: logger.error('{}{} got an unexpected return value {} while sending'. format(CONNECTION_PREFIX, self, r)) return r[0]
def sendPingPong(self, remote: Union[str, Remote], is_ping=True): msg = self.pingMessage if is_ping else self.pongMessage action = 'ping' if is_ping else 'pong' name = remote if isinstance(remote, (str, bytes)) else remote.name r = self.send(msg, name) if r[0] is True: logger.debug('{} {}ed {}'.format(self.name, action, z85_to_friendly(name))) elif r[0] is False: logger.debug('{} failed to {} {} {}' .format(self.name, action, z85_to_friendly(name), r[1]), extra={"cli": False}) # try to re-send pongs later if not is_ping: self._stashed_pongs.add(name) elif r[0] is None: logger.debug('{} will be sending in batch'.format(self)) else: logger.error('{}{} got an unexpected return value {} while sending'. format(CONNECTION_PREFIX, self, r)) return r[0]
def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False): remote = self.remotes.get(uid) err_str = None if not remote: logger.debug("Remote {} does not exist!".format(z85_to_friendly(uid))) return False, err_str socket = remote.socket if not socket: logger.debug('{} has uninitialised socket ' 'for remote {}'.format(self, z85_to_friendly(uid))) return False, err_str try: if not serialized: msg = self.prepare_to_send(msg) logger.trace('{} transmitting message {} to {} by socket {} {}' .format(self, msg, z85_to_friendly(uid), socket.FD, socket.underlying)) socket.send(msg, flags=zmq.NOBLOCK) if remote.isConnected or msg in self.healthMessages: self.metrics.add_event(self.mt_outgoing_size, len(msg)) else: logger.warning('Remote {} is not connected - message will not be sent immediately.' 'If this problem does not resolve itself - check your firewall settings' .format(z85_to_friendly(uid))) # We should not stash ping/pongs as this may lead to incorrect reconnection logic # (replying by old pongs for new connection and masking connection issues) # TODO: since we can not remove ping/pongs from Batches at this phase, just do not stash Batches at all if not is_batch: self._stashed_to_disconnected \ .setdefault(uid, deque(maxlen=self.config.ZMQ_STASH_TO_NOT_CONNECTED_QUEUE_SIZE)) \ .append(msg) return True, err_str except zmq.Again: logger.warning('{} could not transmit message to {}'.format(self, z85_to_friendly(uid))) except InvalidMessageExceedingSizeException as ex: err_str = '{}Cannot transmit message. Error {}'.format(CONNECTION_PREFIX, ex) logger.warning(err_str) return False, err_str
def _verifyAndAppend(self, msg, ident): try: self.metrics.add_event(self.mt_incoming_size, len(msg)) self.msgLenVal.validate(msg) decoded = msg.decode() except (UnicodeDecodeError, InvalidMessageExceedingSizeException) as ex: errstr = 'Message will be discarded due to {}'.format(ex) frm = self.remotesByKeys[ident].name if ident in self.remotesByKeys else ident logger.error("Got from {} {}".format(z85_to_friendly(frm), errstr)) self.msgRejectHandler(errstr, frm) return False self.rxMsgs.append((decoded, ident)) return True
def _verifyAndAppend(self, msg, ident): try: self.metrics.add_event(self.mt_incoming_size, len(msg)) self.msgLenVal.validate(msg) decoded = msg.decode() except (UnicodeDecodeError, InvalidMessageExceedingSizeException) as ex: errstr = 'Message will be discarded due to {}'.format(ex) frm = self.remotesByKeys[ident].name if ident in self.remotesByKeys else ident logger.error("Got from {} {}".format(z85_to_friendly(frm), errstr)) self.msgRejectHandler(errstr, frm) return False self.rxMsgs.append((decoded, ident)) return True
def connect(self, name=None, remoteId=None, ha=None, verKeyRaw=None, publicKeyRaw=None): """ Connect to the node specified by name. """ if not name: raise ValueError('Remote name should be specified') publicKey = None if name in self.remotes: remote = self.remotes[name] else: publicKey = z85.encode( publicKeyRaw) if publicKeyRaw else self.getPublicKey(name) verKey = z85.encode(verKeyRaw) if verKeyRaw else self.getVerKey( name) if not ha or not publicKey or (self.isRestricted and not verKey): raise ValueError( '{} doesnt have enough info to connect. ' 'Need ha, public key and verkey. {} {} {}'.format( name, ha, verKey, publicKey)) remote = self.addRemote(name, ha, verKey, publicKey) public, secret = self.selfEncKeys remote.connect(self.ctx, public, secret) logger.info("{}{} looking for {} at {}:{}".format( CONNECTION_PREFIX, self, name or remote.name, *remote.ha), extra={ "cli": "PLAIN", "tags": ["node-looking"] }) # This should be scheduled as an async task self.sendPingPong(remote, is_ping=True) # re-send previously stashed pings/pongs from unknown remotes logger.trace("{} stashed pongs: {}".format(self.name, str(self._stashed_pongs))) if publicKey in self._stashed_pongs: logger.trace("{} sending stashed pongs to {}".format( self.name, str(z85_to_friendly(publicKey)))) self._stashed_pongs.discard(publicKey) self.sendPingPong(name, is_ping=False) return remote.uid
def transmit(self, msg, uid, timeout=None, serialized=False): remote = self.remotes.get(uid) err_str = None if not remote: logger.debug("Remote {} does not exist!".format(z85_to_friendly(uid))) return False, err_str socket = remote.socket if not socket: logger.debug('{} has uninitialised socket ' 'for remote {}'.format(self, z85_to_friendly(uid))) return False, err_str try: if not serialized: msg = self.prepare_to_send(msg) logger.trace('{} transmitting message {} to {} by socket {} {}' .format(self, msg, z85_to_friendly(uid), socket.FD, socket.underlying)) socket.send(msg, flags=zmq.NOBLOCK) if remote.isConnected or msg in self.healthMessages: self.metrics.add_event(self.mt_outgoing_size, len(msg)) else: logger.warning('Remote {} is not connected - message will not be sent immediately.' 'If this problem does not resolve itself - check your firewall settings' .format(z85_to_friendly(uid))) self._stashed_to_disconnected \ .setdefault(uid, deque(maxlen=self.config.ZMQ_STASH_TO_NOT_CONNECTED_QUEUE_SIZE)) \ .append(msg) return True, err_str except zmq.Again: logger.warning('{} could not transmit message to {}'.format(self, z85_to_friendly(uid))) except InvalidMessageExceedingSizeException as ex: err_str = '{}Cannot transmit message. Error {}'.format(CONNECTION_PREFIX, ex) logger.warning(err_str) return False, err_str
def connect(self, name=None, remoteId=None, ha=None, verKeyRaw=None, publicKeyRaw=None): """ Connect to the node specified by name. """ if not name: raise ValueError('Remote name should be specified') publicKey = None if name in self.remotes: remote = self.remotes[name] else: publicKey = z85.encode( publicKeyRaw) if publicKeyRaw else self.getPublicKey(name) verKey = z85.encode( verKeyRaw) if verKeyRaw else self.getVerKey(name) if not ha or not publicKey or (self.isRestricted and not verKey): raise ValueError('{} doesnt have enough info to connect. ' 'Need ha, public key and verkey. {} {} {}'. format(name, ha, verKey, publicKey)) remote = self.addRemote(name, ha, verKey, publicKey) public, secret = self.selfEncKeys remote.connect(self.ctx, public, secret) logger.info("{}{} looking for {} at {}:{}" .format(CONNECTION_PREFIX, self, name or remote.name, *remote.ha), extra={"cli": "PLAIN", "tags": ["node-looking"]}) # This should be scheduled as an async task self.sendPingPong(remote, is_ping=True) # re-send previously stashed pings/pongs from unknown remotes logger.trace("{} stashed pongs: {}".format(self.name, str(self._stashed_pongs))) if publicKey in self._stashed_pongs: logger.trace("{} sending stashed pongs to {}".format(self.name, str(z85_to_friendly(publicKey)))) self._stashed_pongs.discard(publicKey) self.sendPingPong(name, is_ping=False) return remote.uid