def testFullSigning(): # stored securely/privately seed = randombytes(32) # generates key pair based on seed sk = SigningKey(seed=seed) # helper for signing signer = Signer(sk) # this is the public key used to verify signatures (securely shared # before-hand with recipient) verkey = signer.verhex # the message to be signed msg = b'1234' # the signature sig = signer.signature(msg) # helper for verification vr = Verifier(verkey) # verification isVerified = vr.verify(sig, msg) assert isVerified
def __init__(self, identifier=None, seed=None, alias=None): """ Initialize the signer with an identifier and a seed. :param identifier: some identifier that directly or indirectly references this client :param seed: the seed used to generate a signing key. """ # should be stored securely/privately self.seed = seed if seed else randombytes(32) # generates key pair based on seed self.sk = SigningKey(seed=self.seed) # helper for signing self.naclSigner = NaclSigner(self.sk) # this is the public key used to verify signatures (securely shared # before-hand with recipient) hex_verkey = hexlify(self.naclSigner.verraw) self.verkey = hexToFriendly(hex_verkey) self._identifier = identifier or self.verkey self._alias = alias
def setupSigning(self): # Setup its signer from the signing key stored at disk and for all # verification keys stored at disk, add Verifier _, sk = self.selfSigKeys self.signer = Signer(z85.decode(sk)) for vk in self.getAllVerKeys(): self.addVerifier(vk)
class SimpleSigner(Signer): """ A simple implementation of Signer. This signer creates a public key and a private key using the seed value provided in the constructor. It internally uses the NaclSigner to generate the signature and keys. """ # TODO: Do we need both alias and identifier? def __init__(self, identifier=None, seed=None, alias=None): """ Initialize the signer with an identifier and a seed. :param identifier: some identifier that directly or indirectly references this client :param seed: the seed used to generate a signing key. """ # should be stored securely/privately self.seed = seed if seed else randombytes(32) # generates key pair based on seed self.sk = SigningKey(seed=self.seed) # helper for signing self.naclSigner = NaclSigner(self.sk) # this is the public key used to verify signatures (securely shared # before-hand with recipient) hex_verkey = hexlify(self.naclSigner.verraw) self.verkey = hexToFriendly(hex_verkey) self._identifier = identifier or self.verkey self._alias = alias @property def alias(self) -> str: return self._alias @property def identifier(self) -> str: return self._identifier @property def seedHex(self) -> bytes: return hexlify(self.seed) def sign(self, msg: Dict) -> Dict: """ Return a signature for the given message. """ ser = serialize_msg_for_signing(msg, topLevelKeysToIgnore=[f.SIG.nm, f.SIGS.nm]) bsig = self.naclSigner.signature(ser) sig = base58.b58encode(bsig).decode("utf-8") return sig
def testKeyConversionFromEd25519ToCurve25519(): signer = Signer() sk = signer.keyraw vk = signer.verraw # Check when keys are passed as raw bytes secretKey = ed25519SkToCurve25519(sk) publicKey = ed25519PkToCurve25519(vk) assert PrivateKey(secretKey).public_key.__bytes__() == publicKey assert ed25519PkToCurve25519(vk, toHex=True) == \ hexlify(PrivateKey(secretKey).public_key.__bytes__()) # Check when keys are passed as hex secretKey = ed25519SkToCurve25519(hexlify(sk)) publicKey = ed25519PkToCurve25519(hexlify(vk)) assert PrivateKey(secretKey).public_key.__bytes__() == publicKey
def gen_defs(cls, ips, nodeCount, starting_port): """ Generates some default steward and node definitions for tests :param ips: array of ip addresses :param nodeCount: number of stewards/nodes :param starting_port: ports are assigned incremental starting with this :return: duple of steward and node definitions """ if not ips: ips = ['127.0.0.1'] * nodeCount else: if len(ips) != nodeCount: if len(ips) > nodeCount: ips = ips[:nodeCount] else: ips += ['127.0.0.1'] * (nodeCount - len(ips)) steward_defs = [] node_defs = [] for i in range(1, nodeCount + 1): d = adict() d.name = "Steward" + str(i) d.sigseed = cls.getSigningSeed(d.name) s_signer = DidSigner(seed=d.sigseed) d.nym = s_signer.identifier d.verkey = s_signer.verkey steward_defs.append(d) name = "Node" + str(i) sigseed = cls.getSigningSeed(name) node_defs.append(NodeDef( name=name, ip=ips[i - 1], port=starting_port + (i * 2) - 1, client_port=starting_port + (i * 2), idx=i, sigseed=sigseed, verkey=Signer(sigseed).verhex, steward_nym=d.nym)) return steward_defs, node_defs
def __init__(self, identifier=None, seed=None, alias=None): """ Initialize the signer with an identifier and a seed. :param identifier: some identifier that directly or indirectly references this client :param seed: the seed used to generate a signing key. """ # should be stored securely/privately self.seed = seed if seed else randombytes(32) # generates key pair based on seed self.sk = SigningKey(seed=self.seed) # helper for signing self.naclSigner = NaclSigner(self.sk) Signer.__init__(self) DidIdentity.__init__( self, identifier, rawVerkey=self.naclSigner.verraw) self._alias = alias
def __init__(self, identifier=None, seed=None, alias=None): """ Initialize the signer with an identifier and a seed. :param identifier: some identifier that directly or indirectly references this client :param seed: the seed used to generate a signing key. """ # should be stored securely/privately self.seed = seed if seed else randombytes(32) # generates key pair based on seed self.sk = SigningKey(seed=self.seed) # helper for signing self.naclSigner = NaclSigner(self.sk) Signer.__init__(self) DidIdentity.__init__(self, identifier, rawVerkey=self.naclSigner.verraw) self._alias = alias
class ZStack(NetworkInterface): # Assuming only one listener per stack for now. PublicKeyDirName = 'public_keys' PrivateKeyDirName = 'private_keys' VerifKeyDirName = 'verif_keys' SigKeyDirName = 'sig_keys' sigLen = 64 pingMessage = 'pi' pongMessage = 'po' healthMessages = {pingMessage.encode(), pongMessage.encode()} # TODO: This is not implemented, implement this messageTimeout = 3 def __init__(self, name, ha, basedirpath, msgHandler, restricted=True, seed=None, onlyListener=False, config=None, msgRejectHandler=None): self._name = name self.ha = ha self.basedirpath = basedirpath self.msgHandler = msgHandler self.seed = seed self.config = config or getConfig() self.msgRejectHandler = msgRejectHandler or self.__defaultMsgRejectHandler self.listenerQuota = self.config.DEFAULT_LISTENER_QUOTA self.senderQuota = self.config.DEFAULT_SENDER_QUOTA self.msgLenVal = MessageLenValidator(self.config.MSG_LEN_LIMIT) self.homeDir = None # As of now there would be only one file in secretKeysDir and sigKeyDir self.publicKeysDir = None self.secretKeysDir = None self.verifKeyDir = None self.sigKeyDir = None self.signer = None self.verifiers = {} self.setupDirs() self.setupOwnKeysIfNeeded() self.setupSigning() # self.poller = test.asyncio.Poller() self.restricted = restricted self.ctx = None # type: Context self.listener = None self.auth = None # Each remote is identified uniquely by the name self._remotes = {} # type: Dict[str, Remote] self.remotesByKeys = {} # Indicates if this stack will maintain any remotes or will # communicate simply to listeners. Used in ClientZStack self.onlyListener = onlyListener self.peersWithoutRemotes = set() self._conns = set() # type: Set[str] self.rxMsgs = deque() self._created = time.perf_counter() self.last_heartbeat_at = None def __defaultMsgRejectHandler(self, reason: str, frm): pass @property def remotes(self): return self._remotes @property def created(self): return self._created @property def name(self): return self._name @staticmethod def isRemoteConnected(r) -> bool: return r.isConnected def removeRemote(self, remote: Remote, clear=True): """ Currently not using clear """ name = remote.name pkey = remote.publicKey vkey = remote.verKey if name in self.remotes: self.remotes.pop(name) self.remotesByKeys.pop(pkey, None) self.verifiers.pop(vkey, None) else: logger.debug('No remote named {} present') @staticmethod def initLocalKeys(name, baseDir, sigseed, override=False): sDir = os.path.join(baseDir, '__sDir') eDir = os.path.join(baseDir, '__eDir') os.makedirs(sDir, exist_ok=True) os.makedirs(eDir, exist_ok=True) (public_key, secret_key), (verif_key, sig_key) = createEncAndSigKeys(eDir, sDir, name, seed=sigseed) homeDir = ZStack.homeDirPath(baseDir, name) verifDirPath = ZStack.verifDirPath(homeDir) sigDirPath = ZStack.sigDirPath(homeDir) secretDirPath = ZStack.secretDirPath(homeDir) pubDirPath = ZStack.publicDirPath(homeDir) for d in (homeDir, verifDirPath, sigDirPath, secretDirPath, pubDirPath): os.makedirs(d, exist_ok=True) moveKeyFilesToCorrectLocations(sDir, verifDirPath, sigDirPath) moveKeyFilesToCorrectLocations(eDir, pubDirPath, secretDirPath) shutil.rmtree(sDir) shutil.rmtree(eDir) return hexlify(public_key).decode(), hexlify(verif_key).decode() @staticmethod def initRemoteKeys(name, remoteName, baseDir, verkey, override=False): homeDir = ZStack.homeDirPath(baseDir, name) verifDirPath = ZStack.verifDirPath(homeDir) pubDirPath = ZStack.publicDirPath(homeDir) for d in (homeDir, verifDirPath, pubDirPath): os.makedirs(d, exist_ok=True) if isHex(verkey): verkey = unhexlify(verkey) createCertsFromKeys(verifDirPath, remoteName, z85.encode(verkey)) public_key = ed25519PkToCurve25519(verkey) createCertsFromKeys(pubDirPath, remoteName, z85.encode(public_key)) def onHostAddressChanged(self): # we don't store remote data like ip, port, domain name, etc, so # nothing to do here pass @staticmethod def areKeysSetup(name, baseDir): homeDir = ZStack.homeDirPath(baseDir, name) verifDirPath = ZStack.verifDirPath(homeDir) pubDirPath = ZStack.publicDirPath(homeDir) sigDirPath = ZStack.sigDirPath(homeDir) secretDirPath = ZStack.secretDirPath(homeDir) for d in (verifDirPath, pubDirPath): if not os.path.isfile(os.path.join(d, '{}.key'.format(name))): return False for d in (sigDirPath, secretDirPath): if not os.path.isfile(os.path.join(d, '{}.key_secret'.format(name))): return False return True @staticmethod def keyDirNames(): return ZStack.PublicKeyDirName, ZStack.PrivateKeyDirName, \ ZStack.VerifKeyDirName, ZStack.SigKeyDirName @staticmethod def getHaFromLocal(name, basedirpath): return None def __repr__(self): return self.name @staticmethod def homeDirPath(baseDirPath, name): return os.path.join(baseDirPath, name) @staticmethod def publicDirPath(homeDirPath): return os.path.join(homeDirPath, ZStack.PublicKeyDirName) @staticmethod def secretDirPath(homeDirPath): return os.path.join(homeDirPath, ZStack.PrivateKeyDirName) @staticmethod def verifDirPath(homeDirPath): return os.path.join(homeDirPath, ZStack.VerifKeyDirName) @staticmethod def sigDirPath(homeDirPath): return os.path.join(homeDirPath, ZStack.SigKeyDirName) @staticmethod def learnKeysFromOthers(baseDir, name, others): homeDir = ZStack.homeDirPath(baseDir, name) verifDirPath = ZStack.verifDirPath(homeDir) pubDirPath = ZStack.publicDirPath(homeDir) for d in (homeDir, verifDirPath, pubDirPath): os.makedirs(d, exist_ok=True) for other in others: createCertsFromKeys(verifDirPath, other.name, other.verKey) createCertsFromKeys(pubDirPath, other.name, other.publicKey) def tellKeysToOthers(self, others): for other in others: createCertsFromKeys(other.verifKeyDir, self.name, self.verKey) createCertsFromKeys(other.publicKeysDir, self.name, self.publicKey) def setupDirs(self): self.homeDir = self.homeDirPath(self.basedirpath, self.name) self.publicKeysDir = self.publicDirPath(self.homeDir) self.secretKeysDir = self.secretDirPath(self.homeDir) self.verifKeyDir = self.verifDirPath(self.homeDir) self.sigKeyDir = self.sigDirPath(self.homeDir) for d in (self.homeDir, self.publicKeysDir, self.secretKeysDir, self.verifKeyDir, self.sigKeyDir): os.makedirs(d, exist_ok=True) def setupOwnKeysIfNeeded(self): if not os.listdir(self.sigKeyDir): # If signing keys are not present, secret (private keys) should # not be present since they should be converted keys. assert not os.listdir(self.secretKeysDir) # Seed should be present assert self.seed, 'Keys are not setup for {}'.format(self) logger.info("Signing and Encryption keys were not found for {}. " "Creating them now".format(self), extra={"cli": False}) tdirS = os.path.join(self.homeDir, '__skeys__') tdirE = os.path.join(self.homeDir, '__ekeys__') os.makedirs(tdirS, exist_ok=True) os.makedirs(tdirE, exist_ok=True) createEncAndSigKeys(tdirE, tdirS, self.name, self.seed) moveKeyFilesToCorrectLocations(tdirE, self.publicKeysDir, self.secretKeysDir) moveKeyFilesToCorrectLocations(tdirS, self.verifKeyDir, self.sigKeyDir) shutil.rmtree(tdirE) shutil.rmtree(tdirS) def setupAuth(self, restricted=True, force=False): if self.auth and not force: raise RuntimeError('Listener already setup') location = self.publicKeysDir if restricted else zmq.auth.CURVE_ALLOW_ANY # self.auth = AsyncioAuthenticator(self.ctx) self.auth = MultiZapAuthenticator(self.ctx) self.auth.start() self.auth.allow('0.0.0.0') self.auth.configure_curve(domain='*', location=location) def teardownAuth(self): if self.auth: self.auth.stop() def setupSigning(self): # Setup its signer from the signing key stored at disk and for all # verification keys stored at disk, add Verifier _, sk = self.selfSigKeys self.signer = Signer(z85.decode(sk)) for vk in self.getAllVerKeys(): self.addVerifier(vk) def addVerifier(self, verkey): self.verifiers[verkey] = Verifier(z85.decode(verkey)) def start(self, restricted=None, reSetupAuth=True): # self.ctx = test.asyncio.Context.instance() self.ctx = zmq.Context.instance() if self.config.MAX_SOCKETS: self.ctx.MAX_SOCKETS = self.config.MAX_SOCKETS restricted = self.restricted if restricted is None else restricted logger.debug('{} starting with restricted as {} and reSetupAuth ' 'as {}'.format(self, restricted, reSetupAuth), extra={ "cli": False, "demo": False }) self.setupAuth(restricted, force=reSetupAuth) self.open() def stop(self): if self.opened: logger.info('stack {} closing its listener'.format(self), extra={ "cli": False, "demo": False }) self.close() self.teardownAuth() logger.info("stack {} stopped".format(self), extra={ "cli": False, "demo": False }) @property def opened(self): return self.listener is not None def open(self): # noinspection PyUnresolvedReferences self.listener = self.ctx.socket(zmq.ROUTER) # noinspection PyUnresolvedReferences # self.poller.register(self.listener, test.POLLIN) public, secret = self.selfEncKeys self.listener.curve_secretkey = secret self.listener.curve_publickey = public self.listener.curve_server = True self.listener.identity = self.publicKey logger.debug('{} will bind its listener at {}'.format( self, self.ha[1])) set_keepalive(self.listener, self.config) set_zmq_internal_queue_length(self.listener, self.config) self.listener.bind('tcp://*:{}'.format(self.ha[1])) def close(self): self.listener.unbind(self.listener.LAST_ENDPOINT) self.listener.close(linger=0) self.listener = None logger.debug('{} starting to disconnect remotes'.format(self)) for r in self.remotes.values(): r.disconnect() self.remotesByKeys.pop(r.publicKey, None) self._remotes = {} if self.remotesByKeys: logger.debug( '{} found remotes that were only in remotesByKeys and ' 'not in remotes. This is suspicious') for r in self.remotesByKeys.values(): r.disconnect() self.remotesByKeys = {} self._conns = set() @property def selfEncKeys(self): serverSecretFile = os.path.join(self.secretKeysDir, "{}.key_secret".format(self.name)) return zmq.auth.load_certificate(serverSecretFile) @property def selfSigKeys(self): serverSecretFile = os.path.join(self.sigKeyDir, "{}.key_secret".format(self.name)) return zmq.auth.load_certificate(serverSecretFile) @property def isRestricted(self): return not self.auth.allow_any if self.auth is not None \ else self.restricted @property def isKeySharing(self): # TODO: Change name after removing test return not self.isRestricted def isConnectedTo(self, name: str = None, ha: Tuple = None): if self.onlyListener: return self.hasRemote(name) return super().isConnectedTo(name, ha) def hasRemote(self, name): if self.onlyListener: if isinstance(name, str): name = name.encode() if name in self.peersWithoutRemotes: return True return super().hasRemote(name) def removeRemoteByName(self, name: str): if self.onlyListener: if name in self.peersWithoutRemotes: self.peersWithoutRemotes.remove(name) return True else: return super().removeRemoteByName(name) def getHa(self, name): # Return HA as None when its a `peersWithoutRemote` if self.onlyListener: if isinstance(name, str): name = name.encode() if name in self.peersWithoutRemotes: return None return super().getHa(name) async def service(self, limit=None) -> int: """ Service `limit` number of received messages in this stack. :param limit: the maximum number of messages to be processed. If None, processes all of the messages in rxMsgs. :return: the number of messages processed. """ if self.listener: await self._serviceStack(self.age) else: logger.debug("{} is stopped".format(self)) r = len(self.rxMsgs) if r > 0: pracLimit = limit if limit else sys.maxsize return self.processReceived(pracLimit) return 0 def _verifyAndAppend(self, msg, ident): try: self.msgLenVal.validate(msg) decoded = msg.decode() except (UnicodeDecodeError, InvalidMessageExceedingSizeException) as ex: errstr = 'Message will be discarded due to {}'.format(ex) frm = self.remotesByKeys[ ident].name if ident in self.remotesByKeys else ident logger.error("Got from {} {}".format(frm, errstr)) self.msgRejectHandler(errstr, frm) return False self.rxMsgs.append((decoded, ident)) return True def _receiveFromListener(self, quota) -> int: """ Receives messages from listener :param quota: number of messages to receive :return: number of received messages """ assert quota i = 0 while i < quota: try: ident, msg = self.listener.recv_multipart(flags=zmq.NOBLOCK) if not msg: # Router probing sends empty message on connection continue i += 1 if self.onlyListener and ident not in self.remotesByKeys: self.peersWithoutRemotes.add(ident) self._verifyAndAppend(msg, ident) except zmq.Again: break if i > 0: logger.trace('{} got {} messages through listener'.format(self, i)) return i def _receiveFromRemotes(self, quotaPerRemote) -> int: """ Receives messages from remotes :param quotaPerRemote: number of messages to receive from one remote :return: number of received messages """ assert quotaPerRemote totalReceived = 0 for ident, remote in self.remotesByKeys.items(): if not remote.socket: continue i = 0 sock = remote.socket while i < quotaPerRemote: try: msg, = sock.recv_multipart(flags=zmq.NOBLOCK) if not msg: # Router probing sends empty message on connection continue i += 1 self._verifyAndAppend(msg, ident) except zmq.Again: break if i > 0: logger.trace('{} got {} messages through remote {}'.format( self, i, remote)) totalReceived += i return totalReceived async def _serviceStack(self, age): # TODO: age is unused # These checks are kept here and not moved to a function since # `_serviceStack` is called very often and function call is an overhead if self.config.ENABLE_HEARTBEATS and ( self.last_heartbeat_at is None or (time.perf_counter() - self.last_heartbeat_at) >= self.config.HEARTBEAT_FREQ): self.send_heartbeats() self._receiveFromListener(quota=self.listenerQuota) self._receiveFromRemotes(quotaPerRemote=self.senderQuota) return len(self.rxMsgs) def processReceived(self, limit): if limit <= 0: return 0 for x in range(limit): try: msg, ident = self.rxMsgs.popleft() frm = self.remotesByKeys[ident].name \ if ident in self.remotesByKeys else ident r = self.handlePingPong(msg, frm, ident) if r: continue try: msg = self.deserializeMsg(msg) except Exception as e: logger.error('Error {} while converting message {} ' 'to JSON from {}'.format(e, msg, ident)) continue msg = self.doProcessReceived(msg, frm, ident) if msg: self.msgHandler((msg, frm)) except IndexError: break return x + 1 def doProcessReceived(self, msg, frm, ident): return msg def connect(self, name=None, remoteId=None, ha=None, verKeyRaw=None, publicKeyRaw=None): """ Connect to the node specified by name. """ if not name: raise ValueError('Remote name should be specified') if name in self.remotes: remote = self.remotes[name] else: publicKey = z85.encode( publicKeyRaw) if publicKeyRaw else self.getPublicKey(name) verKey = z85.encode(verKeyRaw) if verKeyRaw else self.getVerKey( name) if not ha or not publicKey or (self.isRestricted and not verKey): raise ValueError( '{} doesnt have enough info to connect. ' 'Need ha, public key and verkey. {} {} {}'.format( name, ha, verKey, publicKey)) remote = self.addRemote(name, ha, verKey, publicKey) public, secret = self.selfEncKeys remote.connect(self.ctx, public, secret) logger.info("{}{} looking for {} at {}:{}".format( CONNECTION_PREFIX, self, name or remote.name, *remote.ha), extra={ "cli": "PLAIN", "tags": ["node-looking"] }) # This should be scheduled as an async task self.sendPingPong(remote, is_ping=True) return remote.uid def reconnectRemote(self, remote): """ Disconnect remote and connect to it again :param remote: instance of Remote from self.remotes :param remoteName: name of remote :return: """ assert remote logger.debug('{} reconnecting to {}'.format(self, remote)) public, secret = self.selfEncKeys remote.disconnect() remote.connect(self.ctx, public, secret) self.sendPingPong(remote, is_ping=True) def reconnectRemoteWithName(self, remoteName): assert remoteName assert remoteName in self.remotes self.reconnectRemote(self.remotes[remoteName]) def disconnectByName(self, name: str): assert name remote = self.remotes.get(name) if not remote: logger.debug('{} did not find any remote ' 'by name {} to disconnect'.format(self, name)) return None remote.disconnect() return remote def addRemote(self, name, ha, remoteVerkey, remotePublicKey): remote = Remote(name, ha, remoteVerkey, remotePublicKey) self.remotes[name] = remote # TODO: Use weakref to remote below instead self.remotesByKeys[remotePublicKey] = remote if remoteVerkey: self.addVerifier(remoteVerkey) else: logger.debug('{} adding a remote {}({}) without a verkey'.format( self, name, ha)) return remote def sendPingPong(self, remote: Union[str, Remote], is_ping=True): msg = self.pingMessage if is_ping else self.pongMessage action = 'ping' if is_ping else 'pong' name = remote if isinstance(remote, (str, bytes)) else remote.name r = self.send(msg, name) if r[0] is True: logger.debug('{} {}ed {}'.format(self.name, action, name)) elif r[0] is False: # TODO: This fails the first time as socket is not established, # need to make it retriable logger.debug('{} failed to {} {} {}'.format( self.name, action, name, r[1]), extra={"cli": False}) elif r[0] is None: logger.debug('{} will be sending in batch'.format(self)) else: logger.warning('{}{} got an unexpected return value {}' ' while sending'.format(CONNECTION_PREFIX, self, r)) return r[0] def handlePingPong(self, msg, frm, ident): if msg in (self.pingMessage, self.pongMessage): if msg == self.pingMessage: logger.debug('{} got ping from {}'.format(self, frm)) self.sendPingPong(frm, is_ping=False) if msg == self.pongMessage: if ident in self.remotesByKeys: self.remotesByKeys[ident].setConnected() logger.debug('{} got pong from {}'.format(self, frm)) return True return False def send_heartbeats(self): # Sends heartbeat (ping) to all logger.debug('{} sending heartbeat to all remotes'.format(self)) for remote in self.remotes: self.sendPingPong(remote) self.last_heartbeat_at = time.perf_counter() def send(self, msg: Any, remoteName: str = None, ha=None): if self.onlyListener: return self.transmitThroughListener(msg, remoteName) else: if remoteName is None: r = [] e = [] # Serializing beforehand since to avoid serializing for each # remote try: msg = self.prepare_to_send(msg) except InvalidMessageExceedingSizeException as ex: err_str = '{}Cannot send message. Error {}'.format( CONNECTION_PREFIX, ex) logger.error(err_str) return False, err_str for uid in self.remotes: res, err = self.transmit(msg, uid, serialized=True) r.append(res) e.append(err) e = list(filter(lambda x: x is not None, e)) ret_err = None if len(e) == 0 else "\n".join(e) return all(r), ret_err else: return self.transmit(msg, remoteName) def transmit(self, msg, uid, timeout=None, serialized=False): remote = self.remotes.get(uid) err_str = None if not remote: logger.debug("Remote {} does not exist!".format(uid)) return False, err_str socket = remote.socket if not socket: logger.debug('{} has uninitialised socket ' 'for remote {}'.format(self, uid)) return False, err_str try: if not serialized: msg = self.prepare_to_send(msg) # socket.send(self.signedMsg(msg), flags=zmq.NOBLOCK) socket.send(msg, flags=zmq.NOBLOCK) logger.debug('{} transmitting message {} to {}'.format( self, msg, uid)) if not remote.isConnected and msg not in self.healthMessages: logger.debug('Remote {} is not connected - ' 'message will not be sent immediately.' 'If this problem does not resolve itself - ' 'check your firewall settings'.format(uid)) return True, err_str except zmq.Again: logger.debug('{} could not transmit message to {}'.format( self, uid)) except InvalidMessageExceedingSizeException as ex: err_str = '{}Cannot transmit message. Error {}'.format( CONNECTION_PREFIX, ex) logger.error(err_str) return False, err_str def transmitThroughListener(self, msg, ident): if isinstance(ident, str): ident = ident.encode() if ident not in self.peersWithoutRemotes: logger.debug('{} not sending message {} to {}'.format( self, msg, ident)) logger.debug( "This is a temporary workaround for not being able to " "disconnect a ROUTER's remote") return False, None try: msg = self.prepare_to_send(msg) # noinspection PyUnresolvedReferences # self.listener.send_multipart([ident, self.signedMsg(msg)], # flags=zmq.NOBLOCK) logger.trace( '{} transmitting {} to {} through listener socket'.format( self, msg, ident)) self.listener.send_multipart([ident, msg], flags=zmq.NOBLOCK) return True, None except zmq.Again: return False, None except InvalidMessageExceedingSizeException as ex: err_str = '{}Cannot transmit message. Error {}'.format( CONNECTION_PREFIX, ex) logger.error(err_str) return False, err_str except Exception as e: err_str = '{}{} got error {} while sending through listener to {}'\ .format(CONNECTION_PREFIX, self, e, ident) logger.error(err_str) return False, err_str return True, None @staticmethod def serializeMsg(msg): if isinstance(msg, Mapping): msg = json.dumps(msg) if isinstance(msg, str): msg = msg.encode() assert isinstance(msg, bytes) return msg @staticmethod def deserializeMsg(msg): if isinstance(msg, bytes): msg = msg.decode() msg = json.loads(msg) return msg def signedMsg(self, msg: bytes, signer: Signer = None): sig = self.signer.signature(msg) return msg + sig def verify(self, msg, by): if self.isKeySharing: return True if by not in self.remotesByKeys: return False verKey = self.remotesByKeys[by].verKey r = self.verifiers[verKey].verify(msg[-self.sigLen:], msg[:-self.sigLen]) return r @staticmethod def loadPubKeyFromDisk(directory, name): filePath = os.path.join(directory, "{}.key".format(name)) try: public, _ = zmq.auth.load_certificate(filePath) return public except (ValueError, IOError) as ex: raise KeyError from ex @staticmethod def loadSecKeyFromDisk(directory, name): filePath = os.path.join(directory, "{}.key_secret".format(name)) try: _, secret = zmq.auth.load_certificate(filePath) return secret except (ValueError, IOError) as ex: raise KeyError from ex @property def publicKey(self): return self.getPublicKey(self.name) @property def publicKeyRaw(self): return z85.decode(self.publicKey) @property def pubhex(self): return hexlify(z85.decode(self.publicKey)) def getPublicKey(self, name): try: return self.loadPubKeyFromDisk(self.publicKeysDir, name) except KeyError: raise PublicKeyNotFoundOnDisk(self.name, name) @property def verKey(self): return self.getVerKey(self.name) @property def verKeyRaw(self): if self.verKey: return z85.decode(self.verKey) return None @property def verhex(self): if self.verKey: return hexlify(z85.decode(self.verKey)) return None def getVerKey(self, name): try: return self.loadPubKeyFromDisk(self.verifKeyDir, name) except KeyError: if self.isRestricted: raise VerKeyNotFoundOnDisk(self.name, name) return None @property def sigKey(self): return self.loadSecKeyFromDisk(self.sigKeyDir, self.name) # TODO: Change name to sighex after removing test @property def keyhex(self): return hexlify(z85.decode(self.sigKey)) @property def priKey(self): return self.loadSecKeyFromDisk(self.secretKeysDir, self.name) @property def prihex(self): return hexlify(z85.decode(self.priKey)) def getAllVerKeys(self): keys = [] for key_file in os.listdir(self.verifKeyDir): if key_file.endswith(".key"): serverVerifFile = os.path.join(self.verifKeyDir, key_file) serverPublic, _ = zmq.auth.load_certificate(serverVerifFile) keys.append(serverPublic) return keys def setRestricted(self, restricted: bool): if self.isRestricted != restricted: logger.debug('{} setting restricted to {}'.format( self, restricted)) self.stop() # TODO: REMOVE, it will make code slow, only doing to allow the # socket to become available again time.sleep(1) self.start(restricted, reSetupAuth=True) def _safeRemove(self, filePath): try: os.remove(filePath) except Exception as ex: logger.debug('{} could delete file {} due to {}'.format( self, filePath, ex)) def clearLocalRoleKeep(self): for d in (self.secretKeysDir, self.sigKeyDir): filePath = os.path.join(d, "{}.key_secret".format(self.name)) self._safeRemove(filePath) for d in (self.publicKeysDir, self.verifKeyDir): filePath = os.path.join(d, "{}.key".format(self.name)) self._safeRemove(filePath) def clearRemoteRoleKeeps(self): for d in (self.secretKeysDir, self.sigKeyDir): for key_file in os.listdir(d): if key_file != '{}.key_secret'.format(self.name): self._safeRemove(os.path.join(d, key_file)) for d in (self.publicKeysDir, self.verifKeyDir): for key_file in os.listdir(d): if key_file != '{}.key'.format(self.name): self._safeRemove(os.path.join(d, key_file)) def clearAllDir(self): shutil.rmtree(self.homeDir) # TODO: Members below are just for the time till RAET replacement is # complete, they need to be removed then. @property def nameRemotes(self): logger.debug('{} proxy method used on {}'.format( inspect.stack()[0][3], self)) return self.remotes @property def keep(self): logger.debug('{} proxy method used on {}'.format( inspect.stack()[0][3], self)) if not hasattr(self, '_keep'): self._keep = DummyKeep(self) return self._keep def clearLocalKeep(self): pass def clearRemoteKeeps(self): pass def prepare_to_send(self, msg: Any): msg_bytes = self.serializeMsg(msg) self.msgLenVal.validate(msg_bytes) return msg_bytes
def getLocalVerKey(roleName, baseDir=None): sighex = getLocalRoleKeyByName(roleName, baseDir, 'sighex') signer = Signer(sighex) return signer.verhex.decode()
steward_def.verkey = steward_signer.verkey client_seed = generateSeed() logger.info('Client seed will be %s', client_seed) client_seed = client_seed.encode() clent_signer = DidSigner(seed=client_seed) client_def = adict() client_def.name = node_name + "-client" client_def.sigseed = client_seed client_def.nym = clent_signer.identifier client_def.verkey = clent_signer.verkey node_seed = nodeSeeds[node_num] logger.info('Client seed will be %s', node_seed) node_seed = node_seed.encode() node_signer = Signer(node_seed) node_def = adict() node_def.name = node_name node_def.ip = nodeIps[node_num] node_def.port = args.node_port node_def.client_port = args.client_port node_def.sigseed = node_seed node_def.verkey = node_signer.verhex node_def.steward_nym = steward_signer.identifier steward_defs.append(steward_def) node_defs.append(node_def) client_defs.append(client_def) # 1. INIT DOMAIN LEDGER GENESIS FILE seq_no = 1
class ZStack(NetworkInterface): # Assuming only one listener per stack for now. PublicKeyDirName = 'public_keys' PrivateKeyDirName = 'private_keys' VerifKeyDirName = 'verif_keys' SigKeyDirName = 'sig_keys' sigLen = 64 pingMessage = 'pi' pongMessage = 'po' healthMessages = {pingMessage.encode(), pongMessage.encode()} # TODO: This is not implemented, implement this messageTimeout = 3 _RemoteClass = Remote def __init__(self, name, ha, basedirpath, msgHandler, restricted=True, seed=None, onlyListener=False, config=None, msgRejectHandler=None, queue_size=0, create_listener_monitor=False, metrics=NullMetricsCollector(), mt_incoming_size=None, mt_outgoing_size=None): self._name = name self.ha = ha self.basedirpath = basedirpath self.msgHandler = msgHandler self.seed = seed self.queue_size = queue_size self.config = config or getConfig() self.msgRejectHandler = msgRejectHandler or self.__defaultMsgRejectHandler self.metrics = metrics self.mt_incoming_size = mt_incoming_size self.mt_outgoing_size = mt_outgoing_size self.listenerQuota = self.config.DEFAULT_LISTENER_QUOTA self.listenerSize = self.config.DEFAULT_LISTENER_SIZE self.senderQuota = self.config.DEFAULT_SENDER_QUOTA self.msgLenVal = MessageLenValidator(self.config.MSG_LEN_LIMIT) self.homeDir = None # As of now there would be only one file in secretKeysDir and sigKeyDir self.publicKeysDir = None self.secretKeysDir = None self.verifKeyDir = None self.sigKeyDir = None self.signer = None self.verifiers = {} self.setupDirs() self.setupOwnKeysIfNeeded() self.setupSigning() # self.poller = test.asyncio.Poller() self.restricted = restricted self.ctx = None # type: Context self.listener = None self.create_listener_monitor = create_listener_monitor self.listener_monitor = None self.auth = None # Each remote is identified uniquely by the name self._remotes = {} # type: Dict[str, Remote] self.remotesByKeys = {} # Indicates if this stack will maintain any remotes or will # communicate simply to listeners. Used in ClientZStack self.onlyListener = onlyListener self._conns = set() # type: Set[str] self.rxMsgs = deque() self._created = time.perf_counter() self.last_heartbeat_at = None self._stashed_to_disconnected = {} self._stashed_pongs = set() self._received_pings = set() def __defaultMsgRejectHandler(self, reason: str, frm): pass @property def remotes(self): return self._remotes @property def created(self): return self._created @property def name(self): return self._name @staticmethod def isRemoteConnected(r) -> bool: return r.isConnected def removeRemote(self, remote: Remote, clear=True): """ Currently not using clear """ name = remote.name pkey = remote.publicKey vkey = remote.verKey if name in self.remotes: self.remotes.pop(name) self.remotesByKeys.pop(pkey, None) self.verifiers.pop(vkey, None) else: logger.info('No remote named {} present') @staticmethod def initLocalKeys(name, baseDir, sigseed, override=False): sDir = os.path.join(baseDir, '__sDir') eDir = os.path.join(baseDir, '__eDir') os.makedirs(sDir, exist_ok=True) os.makedirs(eDir, exist_ok=True) (public_key, secret_key), (verif_key, sig_key) = \ createEncAndSigKeys(eDir, sDir, name, seed=sigseed) homeDir = ZStack.homeDirPath(baseDir, name) verifDirPath = ZStack.verifDirPath(homeDir) sigDirPath = ZStack.sigDirPath(homeDir) secretDirPath = ZStack.secretDirPath(homeDir) pubDirPath = ZStack.publicDirPath(homeDir) for d in (homeDir, verifDirPath, sigDirPath, secretDirPath, pubDirPath): os.makedirs(d, exist_ok=True) moveKeyFilesToCorrectLocations(sDir, verifDirPath, sigDirPath) moveKeyFilesToCorrectLocations(eDir, pubDirPath, secretDirPath) shutil.rmtree(sDir) shutil.rmtree(eDir) return hexlify(public_key).decode(), hexlify(verif_key).decode() @staticmethod def initRemoteKeys(name, remoteName, baseDir, verkey, override=False): homeDir = ZStack.homeDirPath(baseDir, name) verifDirPath = ZStack.verifDirPath(homeDir) pubDirPath = ZStack.publicDirPath(homeDir) for d in (homeDir, verifDirPath, pubDirPath): os.makedirs(d, exist_ok=True) if isHex(verkey): verkey = unhexlify(verkey) createCertsFromKeys(verifDirPath, remoteName, z85.encode(verkey)) public_key = ed25519PkToCurve25519(verkey) createCertsFromKeys(pubDirPath, remoteName, z85.encode(public_key)) def onHostAddressChanged(self): # we don't store remote data like ip, port, domain name, etc, so # nothing to do here pass @staticmethod def areKeysSetup(name, baseDir): homeDir = ZStack.homeDirPath(baseDir, name) verifDirPath = ZStack.verifDirPath(homeDir) pubDirPath = ZStack.publicDirPath(homeDir) sigDirPath = ZStack.sigDirPath(homeDir) secretDirPath = ZStack.secretDirPath(homeDir) for d in (verifDirPath, pubDirPath): if not os.path.isfile(os.path.join(d, '{}.key'.format(name))): return False for d in (sigDirPath, secretDirPath): if not os.path.isfile(os.path.join(d, '{}.key_secret'.format(name))): return False return True @staticmethod def keyDirNames(): return ZStack.PublicKeyDirName, ZStack.PrivateKeyDirName, \ ZStack.VerifKeyDirName, ZStack.SigKeyDirName @staticmethod def getHaFromLocal(name, basedirpath): return None def __repr__(self): return self.name @staticmethod def homeDirPath(baseDirPath, name): return os.path.join(os.path.expanduser(baseDirPath), name) @staticmethod def publicDirPath(homeDirPath): return os.path.join(homeDirPath, ZStack.PublicKeyDirName) @staticmethod def secretDirPath(homeDirPath): return os.path.join(homeDirPath, ZStack.PrivateKeyDirName) @staticmethod def verifDirPath(homeDirPath): return os.path.join(homeDirPath, ZStack.VerifKeyDirName) @staticmethod def sigDirPath(homeDirPath): return os.path.join(homeDirPath, ZStack.SigKeyDirName) @staticmethod def learnKeysFromOthers(baseDir, name, others): homeDir = ZStack.homeDirPath(baseDir, name) verifDirPath = ZStack.verifDirPath(homeDir) pubDirPath = ZStack.publicDirPath(homeDir) for d in (homeDir, verifDirPath, pubDirPath): os.makedirs(d, exist_ok=True) for other in others: createCertsFromKeys(verifDirPath, other.name, other.verKey) createCertsFromKeys(pubDirPath, other.name, other.publicKey) def tellKeysToOthers(self, others): for other in others: createCertsFromKeys(other.verifKeyDir, self.name, self.verKey) createCertsFromKeys(other.publicKeysDir, self.name, self.publicKey) def setupDirs(self): self.homeDir = self.homeDirPath(self.basedirpath, self.name) self.publicKeysDir = self.publicDirPath(self.homeDir) self.secretKeysDir = self.secretDirPath(self.homeDir) self.verifKeyDir = self.verifDirPath(self.homeDir) self.sigKeyDir = self.sigDirPath(self.homeDir) for d in (self.homeDir, self.publicKeysDir, self.secretKeysDir, self.verifKeyDir, self.sigKeyDir): os.makedirs(d, exist_ok=True) def setupOwnKeysIfNeeded(self): if not os.listdir(self.sigKeyDir): # If signing keys are not present, secret (private keys) should # not be present since they should be converted keys. assert not os.listdir(self.secretKeysDir) # Seed should be present assert self.seed, 'Keys are not setup for {}'.format(self) logger.display("Signing and Encryption keys were not found for {}. Creating them now". format(self), extra={"cli": False}) tdirS = os.path.join(self.homeDir, '__skeys__') tdirE = os.path.join(self.homeDir, '__ekeys__') os.makedirs(tdirS, exist_ok=True) os.makedirs(tdirE, exist_ok=True) createEncAndSigKeys(tdirE, tdirS, self.name, self.seed) moveKeyFilesToCorrectLocations(tdirE, self.publicKeysDir, self.secretKeysDir) moveKeyFilesToCorrectLocations(tdirS, self.verifKeyDir, self.sigKeyDir) shutil.rmtree(tdirE) shutil.rmtree(tdirS) def setupAuth(self, restricted=True, force=False): if self.auth and not force: raise RuntimeError('Listener already setup') location = self.publicKeysDir if restricted else zmq.auth.CURVE_ALLOW_ANY # self.auth = AsyncioAuthenticator(self.ctx) self.auth = MultiZapAuthenticator(self.ctx) self.auth.start() self.auth.allow('0.0.0.0') self.auth.configure_curve(domain='*', location=location) def teardownAuth(self): if self.auth: self.auth.stop() def setupSigning(self): # Setup its signer from the signing key stored at disk and for all # verification keys stored at disk, add Verifier _, sk = self.selfSigKeys self.signer = Signer(z85.decode(sk)) for vk in self.getAllVerKeys(): self.addVerifier(vk) def addVerifier(self, verkey): self.verifiers[verkey] = Verifier(z85.decode(verkey)) def start(self, restricted=None, reSetupAuth=True): # self.ctx = test.asyncio.Context.instance() self.ctx = zmq.Context.instance() if self.config.MAX_SOCKETS: self.ctx.MAX_SOCKETS = self.config.MAX_SOCKETS restricted = self.restricted if restricted is None else restricted logger.debug('{} starting with restricted as {} and reSetupAuth ' 'as {}'.format(self, restricted, reSetupAuth), extra={"cli": False, "demo": False}) self.setupAuth(restricted, force=reSetupAuth) self.open() def stop(self): if self.opened: logger.display('stack {} closing its listener'.format(self), extra={"cli": False, "demo": False}) self.close() self.teardownAuth() logger.display("stack {} stopped".format(self), extra={"cli": False, "demo": False}) @property def opened(self): return self.listener is not None def open(self): # noinspection PyUnresolvedReferences self.listener = self.ctx.socket(zmq.ROUTER) if self.create_listener_monitor: self.listener_monitor = self.listener.get_monitor_socket() # noinspection PyUnresolvedReferences # self.poller.register(self.listener, test.POLLIN) public, secret = self.selfEncKeys self.listener.curve_secretkey = secret self.listener.curve_publickey = public self.listener.curve_server = True self.listener.identity = self.publicKey logger.debug( '{} will bind its listener at {}:{}'.format(self, self.ha[0], self.ha[1])) set_keepalive(self.listener, self.config) set_zmq_internal_queue_size(self.listener, self.queue_size) # Cycle to deal with "Address already in use" in case of immediate stack restart. bound = False bind_retries = 0 while not bound: try: self.listener.bind( '{protocol}://{ip}:{port}'.format(ip=self.ha[0], port=self.ha[1], protocol=ZMQ_NETWORK_PROTOCOL) ) bound = True except zmq.error.ZMQError as zmq_err: bind_retries += 1 if bind_retries == 50: raise zmq_err time.sleep(0.2) def close(self): if self.listener_monitor is not None: self.listener.disable_monitor() self.listener_monitor = None self.listener.unbind(self.listener.LAST_ENDPOINT) self.listener.close(linger=0) self.listener = None logger.debug('{} starting to disconnect remotes'.format(self)) for r in self.remotes.values(): r.disconnect() self.remotesByKeys.pop(r.publicKey, None) self._remotes = {} if self.remotesByKeys: logger.debug('{} found remotes that were only in remotesByKeys and ' 'not in remotes. This is suspicious') for r in self.remotesByKeys.values(): r.disconnect() self.remotesByKeys = {} self._conns = set() @property def selfEncKeys(self): serverSecretFile = os.path.join(self.secretKeysDir, "{}.key_secret".format(self.name)) return zmq.auth.load_certificate(serverSecretFile) @property def selfSigKeys(self): serverSecretFile = os.path.join(self.sigKeyDir, "{}.key_secret".format(self.name)) return zmq.auth.load_certificate(serverSecretFile) @property def isRestricted(self): return not self.auth.allow_any if self.auth is not None \ else self.restricted @property def isKeySharing(self): # TODO: Change name after removing test return not self.isRestricted def isConnectedTo(self, name: str = None, ha: Tuple = None): return not self.onlyListener and super().isConnectedTo(name, ha) def hasRemote(self, name): return not self.onlyListener and super().hasRemote(name) def removeRemoteByName(self, name: str): return not self.onlyListener and super().removeRemoteByName(name) def getHa(self, name): # Return HA as None when its a `peersWithoutRemote` if self.onlyListener: return None return super().getHa(name) async def service(self, limit=None, quota: Optional[Quota] = None) -> int: """ Service `limit` number of received messages in this stack. :param limit: the maximum number of messages to be processed. If None, processes all of the messages in rxMsgs. :return: the number of messages processed. """ if self.listener: await self._serviceStack(self.age, quota) else: logger.info("{} is stopped".format(self)) r = len(self.rxMsgs) if r > 0: pracLimit = limit if limit else sys.maxsize return self.processReceived(pracLimit) return 0 def _verifyAndAppend(self, msg, ident): try: self.metrics.add_event(self.mt_incoming_size, len(msg)) self.msgLenVal.validate(msg) decoded = msg.decode() except (UnicodeDecodeError, InvalidMessageExceedingSizeException) as ex: errstr = 'Message will be discarded due to {}'.format(ex) frm = self.remotesByKeys[ident].name if ident in self.remotesByKeys else ident logger.error("Got from {} {}".format(z85_to_friendly(frm), errstr)) self.msgRejectHandler(errstr, frm) return False self.rxMsgs.append((decoded, ident)) return True def _receiveFromListener(self, quota: Quota) -> int: """ Receives messages from listener :param quota: number of messages to receive :return: number of received messages """ i = 0 incoming_size = 0 while i < quota.count and incoming_size < quota.size: try: ident, msg = self.listener.recv_multipart(flags=zmq.NOBLOCK) if not msg: # Router probing sends empty message on connection continue incoming_size += len(msg) i += 1 self._verifyAndAppend(msg, ident) except zmq.Again: break if i > 0: logger.trace('{} got {} messages through listener'. format(self, i)) return i def _receiveFromRemotes(self, quotaPerRemote) -> int: """ Receives messages from remotes :param quotaPerRemote: number of messages to receive from one remote :return: number of received messages """ assert quotaPerRemote totalReceived = 0 for ident, remote in self.remotesByKeys.items(): if not remote.socket: continue i = 0 sock = remote.socket while i < quotaPerRemote: try: msg, = sock.recv_multipart(flags=zmq.NOBLOCK) if not msg: # Router probing sends empty message on connection continue i += 1 self._verifyAndAppend(msg, ident) except zmq.Again: break if i > 0: logger.trace('{} got {} messages through remote {}'. format(self, i, remote)) totalReceived += i return totalReceived async def _serviceStack(self, age, quota: Optional[Quota] = None): # TODO: age is unused # These checks are kept here and not moved to a function since # `_serviceStack` is called very often and function call is an overhead if self.config.ENABLE_HEARTBEATS and ( self.last_heartbeat_at is None or (time.perf_counter() - self.last_heartbeat_at) >= self.config.HEARTBEAT_FREQ): self.send_heartbeats() if quota is None: quota = Quota(count=self.listenerQuota, size=self.listenerSize) self._receiveFromListener(quota) self._receiveFromRemotes(quotaPerRemote=self.senderQuota) return len(self.rxMsgs) def processReceived(self, limit): if limit <= 0: return 0 num_processed = 0 for num_processed in range(limit): if len(self.rxMsgs) == 0: return num_processed msg, ident = self.rxMsgs.popleft() frm = self.remotesByKeys[ident].name \ if ident in self.remotesByKeys else ident if self.handlePingPong(msg, frm, ident): continue if not self.onlyListener and ident not in self.remotesByKeys: logger.warning('{} received message from unknown remote {}' .format(self, z85_to_friendly(ident))) continue try: msg = self.deserializeMsg(msg) except Exception as e: logger.error('Error {} while converting message {} ' 'to JSON from {}'.format(e, msg, z85_to_friendly(ident))) continue msg = self.doProcessReceived(msg, frm, ident) if msg: self.msgHandler((msg, frm)) return num_processed + 1 def doProcessReceived(self, msg, frm, ident): return msg def connect(self, name=None, remoteId=None, ha=None, verKeyRaw=None, publicKeyRaw=None): """ Connect to the node specified by name. """ if not name: raise ValueError('Remote name should be specified') publicKey = None if name in self.remotes: remote = self.remotes[name] else: publicKey = z85.encode( publicKeyRaw) if publicKeyRaw else self.getPublicKey(name) verKey = z85.encode( verKeyRaw) if verKeyRaw else self.getVerKey(name) if not ha or not publicKey or (self.isRestricted and not verKey): raise ValueError('{} doesnt have enough info to connect. ' 'Need ha, public key and verkey. {} {} {}'. format(name, ha, verKey, publicKey)) remote = self.addRemote(name, ha, verKey, publicKey) public, secret = self.selfEncKeys remote.connect(self.ctx, public, secret) logger.info("{}{} looking for {} at {}:{}" .format(CONNECTION_PREFIX, self, name or remote.name, *remote.ha), extra={"cli": "PLAIN", "tags": ["node-looking"]}) # This should be scheduled as an async task self.sendPingPong(remote, is_ping=True) # re-send previously stashed pings/pongs from unknown remotes logger.trace("{} stashed pongs: {}".format(self.name, str(self._stashed_pongs))) if publicKey in self._stashed_pongs: logger.trace("{} sending stashed pongs to {}".format(self.name, str(z85_to_friendly(publicKey)))) self._stashed_pongs.discard(publicKey) self.sendPingPong(name, is_ping=False) return remote.uid def reconnectRemote(self, remote): """ Disconnect remote and connect to it again :param remote: instance of Remote from self.remotes :param remoteName: name of remote :return: """ if not isinstance(remote, Remote): raise PlenumTypeError('remote', remote, Remote) logger.info('{} reconnecting to {}'.format(self, remote)) public, secret = self.selfEncKeys remote.disconnect() remote.connect(self.ctx, public, secret) self.sendPingPong(remote, is_ping=True) def reconnectRemoteWithName(self, remoteName): if remoteName not in self.remotes: raise PlenumValueError( 'remoteName', remoteName, "one of {}".format(self.remotes) ) self.reconnectRemote(self.remotes[remoteName]) def disconnectByName(self, remoteName: str): if not remoteName: raise PlenumValueError( 'remoteName', remoteName, "non-empty string" ) remote = self.remotes.get(remoteName) if not remote: logger.debug('{} did not find any remote ' 'by name {} to disconnect' .format(self, remoteName)) return None remote.disconnect() return remote def addRemote(self, name, ha, remoteVerkey, remotePublicKey): if not name: raise PlenumValueError('name', name, 'non-empty') remote = self._RemoteClass(name, ha, remoteVerkey, remotePublicKey, self.queue_size, self.ha[0]) self.remotes[name] = remote # TODO: Use weakref to remote below instead self.remotesByKeys[remotePublicKey] = remote if remoteVerkey: self.addVerifier(remoteVerkey) else: logger.display('{} adding a remote {}({}) without a verkey'.format(self, name, ha)) return remote def sendPingPong(self, remote: Union[str, Remote], is_ping=True): msg = self.pingMessage if is_ping else self.pongMessage action = 'ping' if is_ping else 'pong' name = remote if isinstance(remote, (str, bytes)) else remote.name r = self.send(msg, name) if r[0] is True: logger.debug('{} {}ed {}'.format(self.name, action, z85_to_friendly(name))) elif r[0] is False: logger.debug('{} failed to {} {} {}' .format(self.name, action, z85_to_friendly(name), r[1]), extra={"cli": False}) # try to re-send pongs later if not is_ping: self._stashed_pongs.add(name) elif r[0] is None: logger.debug('{} will be sending in batch'.format(self)) else: logger.error('{}{} got an unexpected return value {} while sending'. format(CONNECTION_PREFIX, self, r)) return r[0] def handlePingPong(self, msg, frm, ident): if msg in (self.pingMessage, self.pongMessage): if msg == self.pingMessage: logger.trace('{} got ping from {}'.format(self, z85_to_friendly(frm))) self.sendPingPong(frm, is_ping=False) if msg == self.pongMessage: if ident in self.remotesByKeys: self.remotesByKeys[ident].setConnected() self._resend_to_disconnected(frm, ident) logger.trace('{} got pong from {}'.format(self, z85_to_friendly(frm))) return True return False def _can_resend_to_disconnected(self, to, ident): if to not in self._stashed_to_disconnected: return False if ident not in self.remotesByKeys: return False if not self.remotesByKeys[ident].isConnected: return False return True def _resend_to_disconnected(self, to, ident): if not self._can_resend_to_disconnected(to, ident): return logger.trace('{} resending stashed messages to {}'.format(self, z85_to_friendly(to))) msgs = self._stashed_to_disconnected[to] while msgs: msg = msgs.popleft() ZStack.send(self, msg, to) def send_heartbeats(self): # Sends heartbeat (ping) to all logger.debug('{} sending heartbeat to all remotes'.format(self)) for remote in self.remotes: self.sendPingPong(remote) self.last_heartbeat_at = time.perf_counter() def send(self, msg: Any, remoteName: str = None, ha=None): if self.onlyListener: return self.transmitThroughListener(msg, remoteName) else: if remoteName is None: r = [] e = [] # Serializing beforehand since to avoid serializing for each # remote try: msg = self.prepare_to_send(msg) except InvalidMessageExceedingSizeException as ex: err_str = '{}Cannot send message. Error {}'.format(CONNECTION_PREFIX, ex) logger.warning(err_str) return False, err_str for uid in self.remotes: res, err = self.transmit(msg, uid, serialized=True) r.append(res) e.append(err) e = list(filter(lambda x: x is not None, e)) ret_err = None if len(e) == 0 else "\n".join(e) return all(r), ret_err else: return self.transmit(msg, remoteName) def transmit(self, msg, uid, timeout=None, serialized=False): remote = self.remotes.get(uid) err_str = None if not remote: logger.debug("Remote {} does not exist!".format(z85_to_friendly(uid))) return False, err_str socket = remote.socket if not socket: logger.debug('{} has uninitialised socket ' 'for remote {}'.format(self, z85_to_friendly(uid))) return False, err_str try: if not serialized: msg = self.prepare_to_send(msg) logger.trace('{} transmitting message {} to {} by socket {} {}' .format(self, msg, z85_to_friendly(uid), socket.FD, socket.underlying)) socket.send(msg, flags=zmq.NOBLOCK) if remote.isConnected or msg in self.healthMessages: self.metrics.add_event(self.mt_outgoing_size, len(msg)) else: logger.warning('Remote {} is not connected - message will not be sent immediately.' 'If this problem does not resolve itself - check your firewall settings' .format(z85_to_friendly(uid))) self._stashed_to_disconnected \ .setdefault(uid, deque(maxlen=self.config.ZMQ_STASH_TO_NOT_CONNECTED_QUEUE_SIZE)) \ .append(msg) return True, err_str except zmq.Again: logger.warning('{} could not transmit message to {}'.format(self, z85_to_friendly(uid))) except InvalidMessageExceedingSizeException as ex: err_str = '{}Cannot transmit message. Error {}'.format(CONNECTION_PREFIX, ex) logger.warning(err_str) return False, err_str def transmitThroughListener(self, msg, ident) -> Tuple[bool, Optional[str]]: if isinstance(ident, str): ident = ident.encode() try: msg = self.prepare_to_send(msg) # noinspection PyUnresolvedReferences # self.listener.send_multipart([ident, self.signedMsg(msg)], # flags=zmq.NOBLOCK) logger.trace('{} transmitting {} to {} through listener socket'. format(self, msg, ident)) self.metrics.add_event(self.mt_outgoing_size, len(msg)) self.listener.send_multipart([ident, msg], flags=zmq.NOBLOCK) except zmq.Again: return False, None except InvalidMessageExceedingSizeException as ex: err_str = '{}Cannot transmit message. Error {}'.format(CONNECTION_PREFIX, ex) logger.warning(err_str) return False, err_str except Exception as e: err_str = '{}{} got error {} while sending through listener to {}' \ .format(CONNECTION_PREFIX, self, e, ident) logger.warning(err_str) return False, err_str return True, None @staticmethod def serializeMsg(msg): if isinstance(msg, Mapping): msg = json.dumps(msg) if isinstance(msg, str): msg = msg.encode() assert isinstance(msg, bytes) return msg @staticmethod def deserializeMsg(msg): if isinstance(msg, bytes): msg = msg.decode() msg = json.loads(msg) return msg def signedMsg(self, msg: bytes, signer: Signer = None): sig = self.signer.signature(msg) return msg + sig def verify(self, msg, by): if self.isKeySharing: return True if by not in self.remotesByKeys: return False verKey = self.remotesByKeys[by].verKey r = self.verifiers[verKey].verify( msg[-self.sigLen:], msg[:-self.sigLen]) return r @staticmethod def loadPubKeyFromDisk(directory, name): filePath = os.path.join(directory, "{}.key".format(name)) try: public, _ = zmq.auth.load_certificate(filePath) return public except (ValueError, IOError) as ex: raise KeyError from ex @staticmethod def loadSecKeyFromDisk(directory, name): filePath = os.path.join(directory, "{}.key_secret".format(name)) try: _, secret = zmq.auth.load_certificate(filePath) return secret except (ValueError, IOError) as ex: raise KeyError from ex @property def publicKey(self): return self.getPublicKey(self.name) @property def publicKeyRaw(self): return z85.decode(self.publicKey) @property def pubhex(self): return hexlify(z85.decode(self.publicKey)) def getPublicKey(self, name): try: return self.loadPubKeyFromDisk(self.publicKeysDir, name) except KeyError: raise PublicKeyNotFoundOnDisk(self.name, name) @property def verKey(self): return self.getVerKey(self.name) @property def verKeyRaw(self): if self.verKey: return z85.decode(self.verKey) return None @property def verhex(self): if self.verKey: return hexlify(z85.decode(self.verKey)) return None def getVerKey(self, name): try: return self.loadPubKeyFromDisk(self.verifKeyDir, name) except KeyError: if self.isRestricted: raise VerKeyNotFoundOnDisk(self.name, name) return None @property def sigKey(self): return self.loadSecKeyFromDisk(self.sigKeyDir, self.name) # TODO: Change name to sighex after removing test @property def keyhex(self): return hexlify(z85.decode(self.sigKey)) @property def priKey(self): return self.loadSecKeyFromDisk(self.secretKeysDir, self.name) @property def prihex(self): return hexlify(z85.decode(self.priKey)) def getAllVerKeys(self): keys = [] for key_file in os.listdir(self.verifKeyDir): if key_file.endswith(".key"): serverVerifFile = os.path.join(self.verifKeyDir, key_file) serverPublic, _ = zmq.auth.load_certificate(serverVerifFile) keys.append(serverPublic) return keys def setRestricted(self, restricted: bool): if self.isRestricted != restricted: logger.debug('{} setting restricted to {}'. format(self, restricted)) self.stop() # TODO: REMOVE, it will make code slow, only doing to allow the # socket to become available again time.sleep(1) self.start(restricted, reSetupAuth=True) def _safeRemove(self, filePath): try: os.remove(filePath) except Exception as ex: logger.info('{} could delete file {} due to {}'.format(self, filePath, ex)) def clearLocalRoleKeep(self): for d in (self.secretKeysDir, self.sigKeyDir): filePath = os.path.join(d, "{}.key_secret".format(self.name)) self._safeRemove(filePath) for d in (self.publicKeysDir, self.verifKeyDir): filePath = os.path.join(d, "{}.key".format(self.name)) self._safeRemove(filePath) def clearRemoteRoleKeeps(self): for d in (self.secretKeysDir, self.sigKeyDir): for key_file in os.listdir(d): if key_file != '{}.key_secret'.format(self.name): self._safeRemove(os.path.join(d, key_file)) for d in (self.publicKeysDir, self.verifKeyDir): for key_file in os.listdir(d): if key_file != '{}.key'.format(self.name): self._safeRemove(os.path.join(d, key_file)) def clearAllDir(self): shutil.rmtree(self.homeDir) def prepare_to_send(self, msg: Any): msg_bytes = self.serializeMsg(msg) self.msgLenVal.validate(msg_bytes) return msg_bytes @staticmethod def get_monitor_events(monitor_socket, non_block=True): events = [] # noinspection PyUnresolvedReferences flags = zmq.NOBLOCK if non_block else 0 while True: try: # noinspection PyUnresolvedReferences message = recv_monitor_message(monitor_socket, flags) events.append(message) except zmq.Again: break return events
def gen_defs(cls, ips, steward_seeds, node_seeds, node_count, starting_port): if not ips: ips = ['127.0.0.1'] * node_count else: if len(ips) != node_count: if len(ips) > node_count: ips = ips[:node_count] else: ips += ['127.0.0.1'] * (node_count - len(ips)) if (steward_seeds == None): steward_seeds = [] for i in range(1, node_count + 1): seed = "Steward" + str(i) seed = ('0' * (32 - len(seed)) + seed) steward_seeds.append(seed) elif len(steward_seeds) != node_count: if len(steward_seeds) > node_count: steward_seeds = steward_seeds[:node_count] else: current_steward_seeds_list_length = len(steward_seeds) for i in range(current_steward_seeds_list_length + 1, node_count + 1): seed = "Steward" + str(i) seed = ('0' * (32 - len(seed)) + seed) steward_seeds.append(seed) if (node_seeds == None): node_seeds = [] for i in range(1, node_count + 1): seed = "Node" + str(i) seed = ('0' * (32 - len(seed)) + seed) node_seeds.append(seed) elif len(node_seeds) != node_count: if len(node_seeds) > node_count: node_seeds = node_seeds[:node_count] else: current_node_seeds_list_length = len(node_seeds) for i in range(current_node_seeds_list_length + 1, node_count + 1): seed = "Node" + str(i) seed = ('0' * (32 - len(seed)) + seed) node_seeds.append(seed) steward_defs = [] node_defs = [] for i in range(1, node_count + 1): d = adict() d.name = "Steward" + str(i) d.sigseed = cls.get_signing_seed(steward_seeds[i - 1]) s_signer = DidSigner(seed=d.sigseed) d.nym = s_signer.identifier d.verkey = s_signer.verkey steward_defs.append(d) name = "Node" + str(i) sigseed = cls.get_signing_seed(node_seeds[i - 1]) node_defs.append( NodeDef(name=name, ip=ips[i - 1], port=starting_port + (i * 2) - 1, client_port=starting_port + (i * 2), idx=i, sigseed=sigseed, verkey=Signer(sigseed).verhex, steward_nym=d.nym)) return steward_defs, node_defs
class ZStack(NetworkInterface): # Assuming only one listener per stack for now. PublicKeyDirName = 'public_keys' PrivateKeyDirName = 'private_keys' VerifKeyDirName = 'verif_keys' SigKeyDirName = 'sig_keys' sigLen = 64 pingMessage = 'pi' pongMessage = 'po' healthMessages = {pingMessage.encode(), pongMessage.encode()} # TODO: This is not implemented, implement this messageTimeout = 3 _RemoteClass = Remote def __init__(self, name, ha, basedirpath, msgHandler, restricted=True, seed=None, onlyListener=False, config=None, msgRejectHandler=None, queue_size=0, create_listener_monitor=False, metrics=NullMetricsCollector(), mt_incoming_size=None, mt_outgoing_size=None, timer=None): self._name = name self.ha = ha self.basedirpath = basedirpath self.msgHandler = msgHandler self.seed = seed self.queue_size = queue_size self.config = config or getConfig() self.msgRejectHandler = msgRejectHandler or self.__defaultMsgRejectHandler self._node_mode = None self._stashed_unknown_remote_msgs = deque(maxlen=self.config.ZMQ_STASH_UNKNOWN_REMOTE_MSGS_QUEUE_SIZE) self.metrics = metrics self.mt_incoming_size = mt_incoming_size self.mt_outgoing_size = mt_outgoing_size self.listenerQuota = self.config.DEFAULT_LISTENER_QUOTA self.listenerSize = self.config.DEFAULT_LISTENER_SIZE self.senderQuota = self.config.DEFAULT_SENDER_QUOTA self.msgLenVal = MessageLenValidator(self.config.MSG_LEN_LIMIT) self.homeDir = None # As of now there would be only one file in secretKeysDir and sigKeyDir self.publicKeysDir = None self.secretKeysDir = None self.verifKeyDir = None self.sigKeyDir = None self.signer = None self.verifiers = {} self.setupDirs() self.setupOwnKeysIfNeeded() self.setupSigning() # self.poller = test.asyncio.Poller() self.restricted = restricted self.ctx = None # type: Context self.listener = None self.create_listener_monitor = create_listener_monitor self.listener_monitor = None self.auth = None # Each remote is identified uniquely by the name self._remotes = {} # type: Dict[str, Remote] self.remotesByKeys = {} self.remote_ping_stats = {} # Indicates if this stack will maintain any remotes or will # communicate simply to listeners. Used in ClientZStack self.onlyListener = onlyListener self._conns = set() # type: Set[str] self.rxMsgs = deque() self._created = time.perf_counter() self.last_heartbeat_at = None self._stashed_pongs = set() self._received_pings = set() self._client_message_provider = ClientMessageProvider(self.name, self.config, self.prepare_to_send, self.metrics, self.mt_outgoing_size, timer) def __defaultMsgRejectHandler(self, reason: str, frm): pass @property def remotes(self): return self._remotes @property def created(self): return self._created @property def name(self): return self._name def set_mode(self, value): self._node_mode = value @staticmethod def isRemoteConnected(r) -> bool: return r.isConnected def removeRemote(self, remote: Remote, clear=True): """ Currently not using clear """ name = remote.name pkey = remote.publicKey vkey = remote.verKey if name in self.remotes: self.remotes.pop(name) self.remotesByKeys.pop(pkey, None) self.verifiers.pop(vkey, None) else: logger.info('No remote named {} present') @staticmethod def initLocalKeys(name, baseDir, sigseed, override=False): sDir = os.path.join(baseDir, '__sDir') eDir = os.path.join(baseDir, '__eDir') os.makedirs(sDir, exist_ok=True) os.makedirs(eDir, exist_ok=True) (public_key, secret_key), (verif_key, sig_key) = \ createEncAndSigKeys(eDir, sDir, name, seed=sigseed) homeDir = ZStack.homeDirPath(baseDir, name) verifDirPath = ZStack.verifDirPath(homeDir) sigDirPath = ZStack.sigDirPath(homeDir) secretDirPath = ZStack.secretDirPath(homeDir) pubDirPath = ZStack.publicDirPath(homeDir) for d in (homeDir, verifDirPath, sigDirPath, secretDirPath, pubDirPath): os.makedirs(d, exist_ok=True) moveKeyFilesToCorrectLocations(sDir, verifDirPath, sigDirPath) moveKeyFilesToCorrectLocations(eDir, pubDirPath, secretDirPath) shutil.rmtree(sDir) shutil.rmtree(eDir) return hexlify(public_key).decode(), hexlify(verif_key).decode() @staticmethod def initRemoteKeys(name, remoteName, baseDir, verkey, override=False): homeDir = ZStack.homeDirPath(baseDir, name) verifDirPath = ZStack.verifDirPath(homeDir) pubDirPath = ZStack.publicDirPath(homeDir) for d in (homeDir, verifDirPath, pubDirPath): os.makedirs(d, exist_ok=True) if isHex(verkey): verkey = unhexlify(verkey) createCertsFromKeys(verifDirPath, remoteName, z85.encode(verkey)) public_key = ed25519PkToCurve25519(verkey) createCertsFromKeys(pubDirPath, remoteName, z85.encode(public_key)) def onHostAddressChanged(self): # we don't store remote data like ip, port, domain name, etc, so # nothing to do here pass @staticmethod def areKeysSetup(name, baseDir): homeDir = ZStack.homeDirPath(baseDir, name) verifDirPath = ZStack.verifDirPath(homeDir) pubDirPath = ZStack.publicDirPath(homeDir) sigDirPath = ZStack.sigDirPath(homeDir) secretDirPath = ZStack.secretDirPath(homeDir) for d in (verifDirPath, pubDirPath): if not os.path.isfile(os.path.join(d, '{}.key'.format(name))): return False for d in (sigDirPath, secretDirPath): if not os.path.isfile(os.path.join(d, '{}.key_secret'.format(name))): return False return True @staticmethod def keyDirNames(): return ZStack.PublicKeyDirName, ZStack.PrivateKeyDirName, \ ZStack.VerifKeyDirName, ZStack.SigKeyDirName @staticmethod def getHaFromLocal(name, basedirpath): return None def __repr__(self): return self.name @staticmethod def homeDirPath(baseDirPath, name): return os.path.join(os.path.expanduser(baseDirPath), name) @staticmethod def publicDirPath(homeDirPath): return os.path.join(homeDirPath, ZStack.PublicKeyDirName) @staticmethod def secretDirPath(homeDirPath): return os.path.join(homeDirPath, ZStack.PrivateKeyDirName) @staticmethod def verifDirPath(homeDirPath): return os.path.join(homeDirPath, ZStack.VerifKeyDirName) @staticmethod def sigDirPath(homeDirPath): return os.path.join(homeDirPath, ZStack.SigKeyDirName) @staticmethod def learnKeysFromOthers(baseDir, name, others): homeDir = ZStack.homeDirPath(baseDir, name) verifDirPath = ZStack.verifDirPath(homeDir) pubDirPath = ZStack.publicDirPath(homeDir) for d in (homeDir, verifDirPath, pubDirPath): os.makedirs(d, exist_ok=True) for other in others: createCertsFromKeys(verifDirPath, other.name, other.verKey) createCertsFromKeys(pubDirPath, other.name, other.publicKey) def tellKeysToOthers(self, others): for other in others: createCertsFromKeys(other.verifKeyDir, self.name, self.verKey) createCertsFromKeys(other.publicKeysDir, self.name, self.publicKey) def setupDirs(self): self.homeDir = self.homeDirPath(self.basedirpath, self.name) self.publicKeysDir = self.publicDirPath(self.homeDir) self.secretKeysDir = self.secretDirPath(self.homeDir) self.verifKeyDir = self.verifDirPath(self.homeDir) self.sigKeyDir = self.sigDirPath(self.homeDir) for d in (self.homeDir, self.publicKeysDir, self.secretKeysDir, self.verifKeyDir, self.sigKeyDir): os.makedirs(d, exist_ok=True) def setupOwnKeysIfNeeded(self): if not os.listdir(self.sigKeyDir): # If signing keys are not present, secret (private keys) should # not be present since they should be converted keys. assert not os.listdir(self.secretKeysDir) # Seed should be present assert self.seed, 'Keys are not setup for {}'.format(self) logger.display("Signing and Encryption keys were not found for {}. Creating them now". format(self), extra={"cli": False}) tdirS = os.path.join(self.homeDir, '__skeys__') tdirE = os.path.join(self.homeDir, '__ekeys__') os.makedirs(tdirS, exist_ok=True) os.makedirs(tdirE, exist_ok=True) createEncAndSigKeys(tdirE, tdirS, self.name, self.seed) moveKeyFilesToCorrectLocations(tdirE, self.publicKeysDir, self.secretKeysDir) moveKeyFilesToCorrectLocations(tdirS, self.verifKeyDir, self.sigKeyDir) shutil.rmtree(tdirE) shutil.rmtree(tdirS) def setupAuth(self, restricted=True, force=False): if self.auth and not force: raise RuntimeError('Listener already setup') location = self.publicKeysDir if restricted else zmq.auth.CURVE_ALLOW_ANY # self.auth = AsyncioAuthenticator(self.ctx) self.auth = MultiZapAuthenticator(self.ctx) self.auth.start() self.auth.allow('0.0.0.0') self.auth.configure_curve(domain='*', location=location) def teardownAuth(self): if self.auth: self.auth.stop() def setupSigning(self): # Setup its signer from the signing key stored at disk and for all # verification keys stored at disk, add Verifier _, sk = self.selfSigKeys self.signer = Signer(z85.decode(sk)) for vk in self.getAllVerKeys(): self.addVerifier(vk) def addVerifier(self, verkey): self.verifiers[verkey] = Verifier(z85.decode(verkey)) def start(self, restricted=None, reSetupAuth=True): # self.ctx = test.asyncio.Context.instance() if self.config.NEW_CTXT_INSTANCE: self.ctx = zmq.Context() else: self.ctx = zmq.Context.instance() if self.config.MAX_SOCKETS: self.ctx.MAX_SOCKETS = self.config.MAX_SOCKETS restricted = self.restricted if restricted is None else restricted logger.debug('{} starting with restricted as {} and reSetupAuth ' 'as {}'.format(self, restricted, reSetupAuth), extra={"cli": False, "demo": False}) self.setupAuth(restricted, force=reSetupAuth) self.open() def stop(self): if self.opened: logger.display('stack {} closing its listener'.format(self), extra={"cli": False, "demo": False}) self.close() logger.display("stack {} stopped".format(self), extra={"cli": False, "demo": False}) @property def opened(self): return self.listener is not None def open(self): # noinspection PyUnresolvedReferences self.listener = self.ctx.socket(zmq.ROUTER) self._client_message_provider.listener = self.listener self.listener.setsockopt(zmq.ROUTER_MANDATORY, 1) self.listener.setsockopt(zmq.ROUTER_HANDOVER, self.config.ROUTER_HANDOVER) if self.create_listener_monitor: self.listener_monitor = self.listener.get_monitor_socket() # noinspection PyUnresolvedReferences # self.poller.register(self.listener, test.POLLIN) public, secret = self.selfEncKeys self.listener.curve_secretkey = secret self.listener.curve_publickey = public self.listener.curve_server = True self.listener.identity = self.publicKey logger.info( '{} will bind its listener at {}:{}'.format(self, self.ha[0], self.ha[1])) set_keepalive(self.listener, self.config) set_zmq_internal_queue_size(self.listener, self.queue_size) # Cycle to deal with "Address already in use" in case of immediate stack restart. bound = False sleep_between_bind_retries = 0.2 bind_retry_time = 0 while not bound: try: self.listener.bind( '{protocol}://{ip}:{port}'.format(ip=self.ha[0], port=self.ha[1], protocol=ZMQ_NETWORK_PROTOCOL) ) bound = True except zmq.error.ZMQError as zmq_err: logger.warning("{} can not bind to {}:{}. Will try in {} secs.". format(self, self.ha[0], self.ha[1], sleep_between_bind_retries)) bind_retry_time += sleep_between_bind_retries if bind_retry_time > self.config.MAX_WAIT_FOR_BIND_SUCCESS: logger.warning("{} can not bind to {}:{} for {} secs. Going to restart the service.". format(self, self.ha[0], self.ha[1], self.config.MAX_WAIT_FOR_BIND_SUCCESS)) raise zmq_err time.sleep(sleep_between_bind_retries) def close(self): if self.config.NEW_CTXT_INSTANCE: self.ctx.destroy(linger=0) self.listener = None self._remotes = {} self.remotesByKeys = {} self._conns = set() else: if self.listener_monitor is not None: self.listener.disable_monitor() self.listener_monitor = None self.listener.unbind(self.listener.LAST_ENDPOINT) self.listener.close(linger=0) self.listener = None logger.debug('{} starting to disconnect remotes'.format(self)) for r in self.remotes.values(): r.disconnect() self.remotesByKeys.pop(r.publicKey, None) self._remotes = {} if self.remotesByKeys: logger.debug('{} found remotes that were only in remotesByKeys and ' 'not in remotes. This is suspicious') for r in self.remotesByKeys.values(): r.disconnect() self.remotesByKeys = {} self._conns = set() self.teardownAuth() @property def selfEncKeys(self): serverSecretFile = os.path.join(self.secretKeysDir, "{}.key_secret".format(self.name)) return zmq.auth.load_certificate(serverSecretFile) @property def selfSigKeys(self): serverSecretFile = os.path.join(self.sigKeyDir, "{}.key_secret".format(self.name)) return zmq.auth.load_certificate(serverSecretFile) @property def isRestricted(self): return not self.auth.allow_any if self.auth is not None \ else self.restricted @property def isKeySharing(self): # TODO: Change name after removing test return not self.isRestricted def isConnectedTo(self, name: str = None, ha: Tuple = None): return not self.onlyListener and super().isConnectedTo(name, ha) def hasRemote(self, name): return not self.onlyListener and super().hasRemote(name) def removeRemoteByName(self, name: str): return not self.onlyListener and super().removeRemoteByName(name) def getHa(self, name): # Return HA as None when its a `peersWithoutRemote` if self.onlyListener: return None return super().getHa(name) async def service(self, limit=None, quota: Optional[Quota] = None) -> int: """ Service `limit` number of received messages in this stack. :param limit: the maximum number of messages to be processed. If None, processes all of the messages in rxMsgs. :return: the number of messages processed. """ if self.listener: await self._serviceStack(self.age, quota) else: logger.info("{} is stopped".format(self)) r = len(self.rxMsgs) if r > 0: pracLimit = limit if limit else sys.maxsize return self.processReceived(pracLimit) return 0 def _verifyAndAppend(self, msg, ident): try: ident.decode() except ValueError: logger.error("Identifier {} is not decoded into UTF-8 string. " "Request will not be processed".format(ident)) return False try: self.metrics.add_event(self.mt_incoming_size, len(msg)) self.msgLenVal.validate(msg) decoded = msg.decode() except (UnicodeDecodeError, InvalidMessageExceedingSizeException) as ex: errstr = 'Message will be discarded due to {}'.format(ex) frm = self.remotesByKeys[ident].name if ident in self.remotesByKeys else ident logger.error("Got from {} {}".format(z85_to_friendly(frm), errstr)) self.msgRejectHandler(errstr, frm) return False self.rxMsgs.append((decoded, ident)) return True def _receiveFromListener(self, quota: Quota) -> int: """ Receives messages from listener :param quota: number of messages to receive :return: number of received messages """ i = 0 incoming_size = 0 while i < quota.count and incoming_size < quota.size: try: ident, msg = self.listener.recv_multipart(flags=zmq.NOBLOCK) if not msg: # Router probing sends empty message on connection continue incoming_size += len(msg) i += 1 self._verifyAndAppend(msg, ident) except zmq.Again as e: break except zmq.ZMQError as e: logger.debug("Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e)) if i > 0: logger.trace('{} got {} messages through listener'. format(self, i)) return i def _receiveFromRemotes(self, quotaPerRemote) -> int: """ Receives messages from remotes :param quotaPerRemote: number of messages to receive from one remote :return: number of received messages """ assert quotaPerRemote totalReceived = 0 for ident, remote in self.remotesByKeys.items(): if not remote.socket: continue i = 0 sock = remote.socket while i < quotaPerRemote: try: msg, = sock.recv_multipart(flags=zmq.NOBLOCK) if not msg: # Router probing sends empty message on connection continue i += 1 logger.trace("{} received a message from remote {} by socket {} {}", self, z85_to_friendly(ident), sock.FD, sock.underlying) self._verifyAndAppend(msg, ident) except zmq.Again as e: break except zmq.ZMQError as e: logger.debug( "Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e)) if i > 0: logger.trace('{} got {} messages through remote {}'. format(self, i, remote)) totalReceived += i return totalReceived async def _serviceStack(self, age, quota: Optional[Quota] = None): # TODO: age is unused # These checks are kept here and not moved to a function since # `_serviceStack` is called very often and function call is an overhead if self.config.ENABLE_HEARTBEATS and ( self.last_heartbeat_at is None or (time.perf_counter() - self.last_heartbeat_at) >= self.config.HEARTBEAT_FREQ): self.send_heartbeats() if quota is None: quota = Quota(count=self.listenerQuota, size=self.listenerSize) self._receiveFromListener(quota) self._receiveFromRemotes(quotaPerRemote=self.senderQuota) return len(self.rxMsgs) def processReceived(self, limit): if limit <= 0: return 0 num_processed = 0 for num_processed in range(limit): if len(self.rxMsgs) == 0: return num_processed msg, ident = self.rxMsgs.popleft() frm = self.remotesByKeys[ident].name \ if ident in self.remotesByKeys else ident if not self.config.RETRY_CONNECT and ident in self.remotesByKeys: self.remotesByKeys[ident].setConnected() if self.handlePingPong(msg, frm, ident): continue if not self.onlyListener and ident not in self.remotesByKeys: if self._node_mode != Mode.participating: logger.info('{} not yet caught-up, stashing message from unknown remote' .format(self, z85_to_friendly(ident))) self._stashed_unknown_remote_msgs.append((msg, ident)) else: logger.warning('{} received message from unknown remote {}' .format(self, z85_to_friendly(ident))) continue try: msg = self.deserializeMsg(msg) except Exception as e: logger.error('Error {} while converting message {} ' 'to JSON from {}'.format(e, msg, z85_to_friendly(ident))) continue # We have received non-ping-pong message from some remote, we can clean this counter if OP_FIELD_NAME not in msg or msg[OP_FIELD_NAME] != BATCH: self.remote_ping_stats[z85_to_friendly(frm)] = 0 msg = self.doProcessReceived(msg, frm, ident) if msg: self.msgHandler((msg, frm)) return num_processed + 1 def doProcessReceived(self, msg, frm, ident): return msg def connect(self, name=None, remoteId=None, ha=None, verKeyRaw=None, publicKeyRaw=None): """ Connect to the node specified by name. """ if not name: raise ValueError('Remote name should be specified') publicKey = None if name in self.remotes: remote = self.remotes[name] else: publicKey = z85.encode( publicKeyRaw) if publicKeyRaw else self.getPublicKey(name) verKey = z85.encode( verKeyRaw) if verKeyRaw else self.getVerKey(name) if not ha or not publicKey or (self.isRestricted and not verKey): raise ValueError('{} doesnt have enough info to connect. ' 'Need ha, public key and verkey. {} {} {}'. format(name, ha, verKey, publicKey)) remote = self.addRemote(name, ha, verKey, publicKey) public, secret = self.selfEncKeys remote.connect(self.ctx, public, secret) logger.info("{}{} looking for {} at {}:{}" .format(CONNECTION_PREFIX, self, name or remote.name, *remote.ha), extra={"cli": "PLAIN", "tags": ["node-looking"]}) # This should be scheduled as an async task self.sendPingPong(remote, is_ping=True) # re-send previously stashed pings/pongs from unknown remotes logger.trace("{} stashed pongs: {}".format(self.name, str(self._stashed_pongs))) if publicKey in self._stashed_pongs: logger.trace("{} sending stashed pongs to {}".format(self.name, str(z85_to_friendly(publicKey)))) self._stashed_pongs.discard(publicKey) self.sendPingPong(name, is_ping=False) return remote.uid def reconnectRemote(self, remote): """ Disconnect remote and connect to it again :param remote: instance of Remote from self.remotes :param remoteName: name of remote :return: """ if not isinstance(remote, Remote): raise PlenumTypeError('remote', remote, Remote) logger.info('{} reconnecting to {}'.format(self, remote)) public, secret = self.selfEncKeys remote.disconnect() remote.connect(self.ctx, public, secret) self.sendPingPong(remote, is_ping=True) def reconnectRemoteWithName(self, remoteName): if remoteName not in self.remotes: raise PlenumValueError( 'remoteName', remoteName, "one of {}".format(self.remotes) ) self.reconnectRemote(self.remotes[remoteName]) def disconnectByName(self, remoteName: str): if not remoteName: raise PlenumValueError( 'remoteName', remoteName, "non-empty string" ) remote = self.remotes.get(remoteName) if not remote: logger.debug('{} did not find any remote ' 'by name {} to disconnect' .format(self, remoteName)) return None remote.disconnect() return remote def addRemote(self, name, ha, remoteVerkey, remotePublicKey): if not name: raise PlenumValueError('name', name, 'non-empty') remote = self._RemoteClass(name, ha, remoteVerkey, remotePublicKey, self.queue_size, self.ha[0]) self.remotes[name] = remote # TODO: Use weakref to remote below instead self.remotesByKeys[remotePublicKey] = remote if remoteVerkey: self.addVerifier(remoteVerkey) else: logger.display('{} adding a remote {}({}) without a verkey'.format(self, name, ha)) return remote def sendPingPong(self, remote: Union[str, Remote], is_ping=True): msg = self.pingMessage if is_ping else self.pongMessage action = 'ping' if is_ping else 'pong' name = remote if isinstance(remote, (str, bytes)) else remote.name # Do not use Batches for sending health messages r = self.transmit(msg, name, is_batch=False) # r = self.send(msg, name) if r[0] is True: logger.debug('{} {}ed {}'.format(self.name, action, z85_to_friendly(name))) elif r[0] is False: logger.debug('{} failed to {} {} {}' .format(self.name, action, z85_to_friendly(name), r[1]), extra={"cli": False}) # try to re-send pongs later if not is_ping: self._stashed_pongs.add(name) elif r[0] is None: logger.debug('{} will be sending in batch'.format(self)) else: logger.error('{}{} got an unexpected return value {} while sending'. format(CONNECTION_PREFIX, self, r)) return r[0] def handlePingPong(self, msg, frm, ident): if msg in (self.pingMessage, self.pongMessage): if msg == self.pingMessage: nodeName = z85_to_friendly(frm) logger.trace('{} got ping from {}'.format(self, nodeName)) self.sendPingPong(frm, is_ping=False) if not self.config.ENABLE_HEARTBEATS and self.config.PING_RECONNECT_ENABLED and nodeName in self.connecteds: if self.remote_ping_stats.get(nodeName): self.remote_ping_stats[nodeName] += 1 else: self.remote_ping_stats[nodeName] = 1 if self.remote_ping_stats[nodeName] > self.config.PINGS_BEFORE_SOCKET_RECONNECTION: logger.info("Reconnecting {} due to numerous consecutive pings".format(nodeName)) self.remote_ping_stats[nodeName] = 0 self.reconnectRemoteWithName(nodeName) if msg == self.pongMessage: if ident in self.remotesByKeys: self.remotesByKeys[ident].setConnected() logger.trace('{} got pong from {}'.format(self, z85_to_friendly(frm))) return True return False def process_unknown_remote_msgs(self): logger.info('Processing messages from previously unknown remotes.') for msg in self._stashed_unknown_remote_msgs: self.rxMsgs.append(msg) self._stashed_unknown_remote_msgs.clear() self.processReceived(limit=sys.maxsize) def send_heartbeats(self): # Sends heartbeat (ping) to all logger.debug('{} sending heartbeat to all remotes'.format(self)) for remote in self.remotes: self.sendPingPong(remote) self.last_heartbeat_at = time.perf_counter() def send(self, msg: Any, remoteName: str = None, ha=None): if self.onlyListener: return self._client_message_provider.transmit_through_listener(msg, remoteName) else: is_batch = isinstance(msg, Mapping) and msg.get(OP_FIELD_NAME) == BATCH if remoteName is None: r = [] e = [] # Serializing beforehand since to avoid serializing for each # remote try: msg = self.prepare_to_send(msg) except InvalidMessageExceedingSizeException as ex: err_str = '{}Cannot send message. Error {}'.format(CONNECTION_PREFIX, ex) logger.warning(err_str) return False, err_str for uid in self.remotes: res, err = self.transmit(msg, uid, serialized=True, is_batch=is_batch) r.append(res) e.append(err) e = list(filter(lambda x: x is not None, e)) ret_err = None if len(e) == 0 else "\n".join(e) return all(r), ret_err else: return self.transmit(msg, remoteName, is_batch=is_batch) def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False): remote = self.remotes.get(uid) err_str = None if not remote: logger.debug("Remote {} does not exist!".format(z85_to_friendly(uid))) return False, err_str socket = remote.socket if not socket: logger.debug('{} has uninitialised socket ' 'for remote {}'.format(self, z85_to_friendly(uid))) return False, err_str try: if not serialized: msg = self.prepare_to_send(msg) logger.trace('{} transmitting message {} to {} by socket {} {}' .format(self, msg, z85_to_friendly(uid), socket.FD, socket.underlying)) socket.send(msg, flags=zmq.NOBLOCK) if remote.isConnected or msg in self.healthMessages: self.metrics.add_event(self.mt_outgoing_size, len(msg)) else: logger.debug('Remote {} is not connected - message will not be sent immediately.' 'If this problem does not resolve itself - check your firewall settings' .format(z85_to_friendly(uid))) return True, err_str except zmq.Again: logger.warning('{} could not transmit message to {}'.format(self, z85_to_friendly(uid))) except InvalidMessageExceedingSizeException as ex: err_str = '{}Cannot transmit message. Error {}'.format(CONNECTION_PREFIX, ex) logger.warning(err_str) return False, err_str @staticmethod def serializeMsg(msg): if isinstance(msg, Mapping): msg = json.dumps(msg) if isinstance(msg, str): msg = msg.encode() assert isinstance(msg, bytes) return msg @staticmethod def deserializeMsg(msg): if isinstance(msg, bytes): msg = msg.decode() msg = json.loads(msg) return msg def signedMsg(self, msg: bytes, signer: Signer = None): sig = self.signer.signature(msg) return msg + sig def verify(self, msg, by): if self.isKeySharing: return True if by not in self.remotesByKeys: return False verKey = self.remotesByKeys[by].verKey r = self.verifiers[verKey].verify( msg[-self.sigLen:], msg[:-self.sigLen]) return r @staticmethod def loadPubKeyFromDisk(directory, name): filePath = os.path.join(directory, "{}.key".format(name)) try: public, _ = zmq.auth.load_certificate(filePath) return public except (ValueError, IOError) as ex: raise KeyError from ex @staticmethod def loadSecKeyFromDisk(directory, name): filePath = os.path.join(directory, "{}.key_secret".format(name)) try: _, secret = zmq.auth.load_certificate(filePath) return secret except (ValueError, IOError) as ex: raise KeyError from ex @property def publicKey(self): return self.getPublicKey(self.name) @property def publicKeyRaw(self): return z85.decode(self.publicKey) @property def pubhex(self): return hexlify(z85.decode(self.publicKey)) def getPublicKey(self, name): try: return self.loadPubKeyFromDisk(self.publicKeysDir, name) except KeyError: raise PublicKeyNotFoundOnDisk(self.name, name) @property def verKey(self): return self.getVerKey(self.name) @property def verKeyRaw(self): if self.verKey: return z85.decode(self.verKey) return None @property def verhex(self): if self.verKey: return hexlify(z85.decode(self.verKey)) return None def getVerKey(self, name): try: return self.loadPubKeyFromDisk(self.verifKeyDir, name) except KeyError: if self.isRestricted: raise VerKeyNotFoundOnDisk(self.name, name) return None @property def sigKey(self): return self.loadSecKeyFromDisk(self.sigKeyDir, self.name) # TODO: Change name to sighex after removing test @property def keyhex(self): return hexlify(z85.decode(self.sigKey)) @property def priKey(self): return self.loadSecKeyFromDisk(self.secretKeysDir, self.name) @property def prihex(self): return hexlify(z85.decode(self.priKey)) def getAllVerKeys(self): keys = [] for key_file in os.listdir(self.verifKeyDir): if key_file.endswith(".key"): serverVerifFile = os.path.join(self.verifKeyDir, key_file) serverPublic, _ = zmq.auth.load_certificate(serverVerifFile) keys.append(serverPublic) return keys def setRestricted(self, restricted: bool): if self.isRestricted != restricted: logger.debug('{} setting restricted to {}'. format(self, restricted)) self.stop() # TODO: REMOVE, it will make code slow, only doing to allow the # socket to become available again time.sleep(1) self.start(restricted, reSetupAuth=True) def _safeRemove(self, filePath): try: os.remove(filePath) except Exception as ex: logger.info('{} could delete file {} due to {}'.format(self, filePath, ex)) def clearLocalRoleKeep(self): for d in (self.secretKeysDir, self.sigKeyDir): filePath = os.path.join(d, "{}.key_secret".format(self.name)) self._safeRemove(filePath) for d in (self.publicKeysDir, self.verifKeyDir): filePath = os.path.join(d, "{}.key".format(self.name)) self._safeRemove(filePath) def clearRemoteRoleKeeps(self): for d in (self.secretKeysDir, self.sigKeyDir): for key_file in os.listdir(d): if key_file != '{}.key_secret'.format(self.name): self._safeRemove(os.path.join(d, key_file)) for d in (self.publicKeysDir, self.verifKeyDir): for key_file in os.listdir(d): if key_file != '{}.key'.format(self.name): self._safeRemove(os.path.join(d, key_file)) def clearAllDir(self): shutil.rmtree(self.homeDir) def prepare_to_send(self, msg: Any): msg_bytes = self.serializeMsg(msg) return msg_bytes @staticmethod def get_monitor_events(monitor_socket, non_block=True): events = [] # noinspection PyUnresolvedReferences flags = zmq.NOBLOCK if non_block else 0 while True: try: # noinspection PyUnresolvedReferences message = recv_monitor_message(monitor_socket, flags) events.append(message) except zmq.Again: break return events
def bootstrapTestNodesCore(config, envName, appendToLedgers, domainTxnFieldOrder, ips, nodeCount, clientCount, nodeNum, startingPort, nodeParamsFileName): baseDir = config.baseDir if not os.path.exists(baseDir): os.makedirs(baseDir, exist_ok=True) localNodes = not ips if localNodes: ips = ['127.0.0.1'] * nodeCount else: ips = ips.split(",") if len(ips) != nodeCount: if len(ips) > nodeCount: ips = ips[:nodeCount] else: ips += ['127.0.0.1'] * (nodeCount - len(ips)) if hasattr(config, "ENVS") and envName: poolTxnFile = config.ENVS[envName].poolLedger domainTxnFile = config.ENVS[envName].domainLedger else: poolTxnFile = config.poolTransactionsFile domainTxnFile = config.domainTransactionsFile poolLedger = Ledger(CompactMerkleTree(), dataDir=baseDir, fileName=poolTxnFile) domainLedger = Ledger( CompactMerkleTree(), serializer=CompactSerializer(fields=domainTxnFieldOrder), dataDir=baseDir, fileName=domainTxnFile) if not appendToLedgers: poolLedger.reset() domainLedger.reset() trusteeName = "Trustee1" sigseed = TestNetworkSetup.getSigningSeed(trusteeName) verkey = Signer(sigseed).verhex trusteeNym = TestNetworkSetup.getNymFromVerkey(verkey) txn = { TARGET_NYM: trusteeNym, TXN_TYPE: NYM, # TODO: Trustees dont exist in Plenum, but only in Sovrin. # This should be moved to Sovrin ROLE: TRUSTEE, ALIAS: trusteeName, TXN_ID: sha256(trusteeName.encode()).hexdigest() } domainLedger.add(txn) steward1Nym = None for num in range(1, nodeCount + 1): stewardName = "Steward" + str(num) sigseed = TestNetworkSetup.getSigningSeed(stewardName) verkey = Signer(sigseed).verhex stewardNym = TestNetworkSetup.getNymFromVerkey(verkey) txn = { TARGET_NYM: stewardNym, TXN_TYPE: NYM, ROLE: STEWARD, ALIAS: stewardName, TXN_ID: sha256(stewardName.encode()).hexdigest() } if num == 1: steward1Nym = stewardNym else: # The first steward adds every steward txn[f.IDENTIFIER.nm] = steward1Nym domainLedger.add(txn) nodeName = "Node" + str(num) nodePort, clientPort = startingPort + (num * 2 - 1), startingPort \ + (num * 2) ip = ips[num - 1] sigseed = TestNetworkSetup.getSigningSeed(nodeName) if nodeNum == num: _, verkey = initLocalKeys(nodeName, baseDir, sigseed, True, config=config) _, verkey = initLocalKeys(nodeName + CLIENT_STACK_SUFFIX, baseDir, sigseed, True, config=config) verkey = verkey.encode() print("This node with name {} will use ports {} and {} for " "nodestack and clientstack respectively".format( nodeName, nodePort, clientPort)) if not localNodes: paramsFilePath = os.path.join(baseDir, nodeParamsFileName) print('Nodes will not run locally, so writing ' '{}'.format(paramsFilePath)) TestNetworkSetup.writeNodeParamsFile( paramsFilePath, nodeName, nodePort, clientPort) else: verkey = Signer(sigseed).verhex txn = { TARGET_NYM: TestNetworkSetup.getNymFromVerkey(verkey), TXN_TYPE: NODE, f.IDENTIFIER.nm: stewardNym, DATA: { CLIENT_IP: ip, ALIAS: nodeName, CLIENT_PORT: clientPort, NODE_IP: ip, NODE_PORT: nodePort, SERVICES: [VALIDATOR] }, TXN_ID: sha256(nodeName.encode()).hexdigest() } poolLedger.add(txn) for num in range(1, clientCount + 1): clientName = "Client" + str(num) sigseed = TestNetworkSetup.getSigningSeed(clientName) verkey = Signer(sigseed).verhex txn = { f.IDENTIFIER.nm: steward1Nym, TARGET_NYM: TestNetworkSetup.getNymFromVerkey(verkey), TXN_TYPE: NYM, ALIAS: clientName, TXN_ID: sha256(clientName.encode()).hexdigest() } domainLedger.add(txn) poolLedger.stop() domainLedger.stop()
def __init__(self, name: str, nodeReg: Dict[str, HA] = None, ha: Union[HA, Tuple[str, int]] = None, basedirpath: str = None, genesis_dir: str = None, ledger_dir: str = None, keys_dir: str = None, plugins_dir: str = None, config=None, sighex: str = None): """ Creates a new client. :param name: unique identifier for the client :param nodeReg: names and host addresses of all nodes in the pool :param ha: tuple of host and port """ self.config = config or getConfig() dataDir = self.config.clientDataDir or "data/clients" self.basedirpath = basedirpath or self.config.CLI_BASE_DIR self.basedirpath = os.path.expanduser(self.basedirpath) signer = Signer(sighex) sighex = signer.keyraw verkey = rawToFriendly(signer.verraw) self.stackName = verkey # TODO: Have a way for a client to have a user friendly name. Does it # matter now, it used to matter in some CLI exampples in the past. # self.name = name self.name = self.stackName or 'Client~' + str(id(self)) self.genesis_dir = genesis_dir or self.basedirpath self.ledger_dir = ledger_dir or os.path.join(self.basedirpath, dataDir, self.name) self.plugins_dir = plugins_dir or self.basedirpath _keys_dir = keys_dir or self.basedirpath self.keys_dir = os.path.join(_keys_dir, "keys") cha = None if self.exists(self.stackName, self.keys_dir): cha = self.nodeStackClass.getHaFromLocal( self.stackName, self.keys_dir) if cha: cha = HA(*cha) logger.debug("Client {} ignoring given ha {} and using {}". format(self.name, ha, cha)) if not cha: cha = ha if isinstance(ha, HA) else HA(*ha) self.reqRepStore = self.getReqRepStore() self.txnLog = self.getTxnLogStore() HasFileStorage.__init__(self, self.ledger_dir) # TODO: Find a proper name self.alias = name if not nodeReg: self.mode = None HasPoolManager.__init__(self) self.ledgerManager = LedgerManager(self, ownedByNode=False) self.ledgerManager.addLedger( POOL_LEDGER_ID, self.ledger, preCatchupStartClbk=self.prePoolLedgerCatchup, postCatchupCompleteClbk=self.postPoolLedgerCaughtUp, postTxnAddedToLedgerClbk=self.postTxnFromCatchupAddedToLedger) else: cliNodeReg = OrderedDict() for nm, (ip, port) in nodeReg.items(): cliNodeReg[nm] = HA(ip, port) self.nodeReg = cliNodeReg self.mode = Mode.discovered HasActionQueue.__init__(self) self.setPoolParams() stackargs = dict(name=self.stackName, ha=cha, main=False, # stops incoming vacuous joins auth_mode=AuthMode.ALLOW_ANY.value) stackargs['basedirpath'] = self.keys_dir self.created = time.perf_counter() # noinspection PyCallingNonCallable # TODO I think this is a bug here, sighex is getting passed in the seed # parameter self.nodestack = self.nodeStackClass(stackargs, self.handleOneNodeMsg, self.nodeReg, sighex) self.nodestack.onConnsChanged = self.onConnsChanged if self.nodeReg: logger.info( "Client {} initialized with the following node registry:".format( self.alias)) lengths = [max(x) for x in zip(*[ (len(name), len(host), len(str(port))) for name, (host, port) in self.nodeReg.items()])] fmt = " {{:<{}}} listens at {{:<{}}} on port {{:>{}}}".format( *lengths) for name, (host, port) in self.nodeReg.items(): logger.info(fmt.format(name, host, port)) else: logger.info( "Client {} found an empty node registry:".format(self.alias)) Motor.__init__(self) self.inBox = deque() self.nodestack.connectNicelyUntil = 0 # don't need to connect # nicely as a client # TODO: Need to have couple of tests around `reqsPendingConnection` # where we check with and without pool ledger # Stores the requests that need to be sent to the nodes when the client # has made sufficient connections to the nodes. self.reqsPendingConnection = deque() # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REQACK self.expectingAcksFor = {} # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REPLY self.expectingRepliesFor = {} self._observers = {} # type Dict[str, Callable] self._observerSet = set() # makes it easier to guard against duplicates plugins_to_load = self.config.PluginsToLoad if hasattr(self.config, "PluginsToLoad") else None tp = loadPlugins(self.plugins_dir, plugins_to_load) logger.debug("total plugins loaded in client: {}".format(tp)) self._multi_sig_verifier = self._create_multi_sig_verifier() self._read_only_requests = set()
def __init__(self, name: str, nodeReg: Dict[str, HA] = None, ha: Union[HA, Tuple[str, int]] = None, basedirpath: str = None, config=None, sighex: str = None): """ Creates a new client. :param name: unique identifier for the client :param nodeReg: names and host addresses of all nodes in the pool :param ha: tuple of host and port """ self.config = config or getConfig() basedirpath = self.config.baseDir if not basedirpath else basedirpath self.basedirpath = basedirpath signer = Signer(sighex) sighex = signer.keyraw verkey = rawToFriendly(signer.verraw) self.stackName = verkey # TODO: Have a way for a client to have a user friendly name. Does it # matter now, it used to matter in some CLI exampples in the past. # self.name = name self.name = self.stackName cha = None # If client information already exists is RAET then use that if self.exists(self.stackName, basedirpath): cha = getHaFromLocalEstate(self.stackName, basedirpath) if cha: cha = HA(*cha) logger.debug( "Client {} ignoring given ha {} and using {}".format( self.name, ha, cha)) if not cha: cha = ha if isinstance(ha, HA) else HA(*ha) self.reqRepStore = self.getReqRepStore() self.txnLog = self.getTxnLogStore() self.dataDir = self.config.clientDataDir or "data/clients" HasFileStorage.__init__(self, self.name, baseDir=self.basedirpath, dataDir=self.dataDir) # TODO: Find a proper name self.alias = name self._ledger = None if not nodeReg: self.mode = None HasPoolManager.__init__(self) self.ledgerManager = LedgerManager(self, ownedByNode=False) self.ledgerManager.addLedger( 0, self.ledger, postCatchupCompleteClbk=self.postPoolLedgerCaughtUp, postTxnAddedToLedgerClbk=self.postTxnFromCatchupAddedToLedger) else: cliNodeReg = OrderedDict() for nm, (ip, port) in nodeReg.items(): cliNodeReg[nm] = HA(ip, port) self.nodeReg = cliNodeReg self.mode = Mode.discovered HasActionQueue.__init__(self) self.setF() stackargs = dict( name=self.stackName, ha=cha, main=False, # stops incoming vacuous joins auto=2) stackargs['basedirpath'] = basedirpath self.created = time.perf_counter() # noinspection PyCallingNonCallable self.nodestack = self.nodeStackClass(stackargs, self.handleOneNodeMsg, self.nodeReg, sighex) self.nodestack.onConnsChanged = self.onConnsChanged if self.nodeReg: logger.info( "Client {} initialized with the following node registry:". format(self.alias)) lengths = [ max(x) for x in zip(*[(len(name), len(host), len(str(port))) for name, (host, port) in self.nodeReg.items()]) ] fmt = " {{:<{}}} listens at {{:<{}}} on port {{:>{}}}".format( *lengths) for name, (host, port) in self.nodeReg.items(): logger.info(fmt.format(name, host, port)) else: logger.info("Client {} found an empty node registry:".format( self.alias)) Motor.__init__(self) self.inBox = deque() self.nodestack.connectNicelyUntil = 0 # don't need to connect # nicely as a client # TODO: Need to have couple of tests around `reqsPendingConnection` # where we check with and without pool ledger # Stores the requests that need to be sent to the nodes when the client # has made sufficient connections to the nodes. self.reqsPendingConnection = deque() # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REQACK self.expectingAcksFor = {} # Tuple of identifier and reqId as key and value as tuple of set of # nodes which are expected to send REPLY self.expectingRepliesFor = {} tp = loadPlugins(self.basedirpath) logger.debug("total plugins loaded in client: {}".format(tp))