def _check_consistency_after_commit(self, txn_root, state_root=None): if self.ledger.root_hash != txn_root: raise PlenumValueError( 'txnRoot', txn_root, ("equal to current ledger root hash {}".format( self.ledger.root_hash))) if state_root is not None and self.state is not None: if self.state.committedHeadHash != state_root: raise PlenumValueError( 'stateRoot', state_root, ("equal to current state root hash {}".format( self.state.committedHeadHash)))
def apply_data(self, msg, sender): if msg.msg_type != BATCH: raise PlenumValueError('msg.msg_type', msg.msg_type, BATCH) logger.debug("{} got BATCH {} from Observable Node {}".format( OBSERVER_PREFIX, msg, sender)) # 1. make sure that the message is dict batch = msg.msg if not isinstance(batch, dict): batch = self._node.toDict(batch) # 2. check whether we can process the message (that it's not already processed and is the next expected seq_no) if not self._can_process(batch): return # 3. stash the batch self._add_batch(batch, sender) # 4. check that the batch can be applied # - this is the next expected batch (according to seq_nos) # - either state proof is valid, or a consensus of f+1 equal batches from different nodes if not self._can_apply(batch): return # 5. apply the batch (write to ledger and state) self._do_apply_data(batch) # 6. check whether we can apply stashed batches with next seq_no self._process_stashed_messages()
def commit(self, txnCount, stateRoot, txnRoot, ppTime) -> List: """ :param txnCount: The number of requests to commit (The actual requests are picked up from the uncommitted list from the ledger) :param stateRoot: The state trie root after the txns are committed :param txnRoot: The txn merkle root after the txns are committed :return: list of committed transactions """ (seqNoStart, seqNoEnd), committedTxns = \ self.ledger.commitTxns(txnCount) stateRoot = base58.b58decode(stateRoot.encode()) # TODO test for that if self.ledger.root_hash != txnRoot: # Probably the following fail should trigger catchup # TODO add repr / str for Ledger class and dump it here as well raise PlenumValueError( 'txnRoot', txnRoot, ("equal to current ledger root hash {}".format( self.ledger.root_hash))) self.state.commit(rootHash=stateRoot) if self.ts_store: self.ts_store.set(ppTime, stateRoot) return committedTxns
def reconnectRemoteWithName(self, remoteName): if remoteName not in self.remotes: raise PlenumValueError( 'remoteName', remoteName, "one of {}".format(self.remotes) ) self.reconnectRemote(self.remotes[remoteName])
def __init__(self, config, *, chroot=None): if config is None: raise ValueUndefinedError('config') if chroot is not None and not chroot.startswith("/"): raise PlenumValueError('chroot', chroot, "starts with '/'") self.config = config self.chroot = chroot
def __init__(self, max_size=None, **kwargs): if max_size is not None: if not isinstance(max_size, int): raise PlenumTypeError('max_size', max_size, int) if not max_size > 0: raise PlenumValueError('max_size', max_size, '> 0') self.max_size = max_size super().__init__(**kwargs)
def apply_batch(self, state_root, ledger_size): """ :param state_root: uncommitted state root :param ledger_size: ledger size, after committing (future ledger size) :return: """ if not state_root: raise PlenumValueError('state_root', state_root, "No state root given") if ledger_size < 0: raise PlenumValueError('ledger_size', ledger_size, "Incorrect size of ledger") self.un_committed.append((state_root, ledger_size))
def randomStr(size): if not isinstance(size, int): raise PlenumTypeError('size', size, int) if not size > 0: raise PlenumValueError('size', size, '> 0') # Approach 1 rv = randombytes(size // 2).hex() return rv if size % 2 == 0 else rv + hex(randombytes_uniform(15))[-1]
def __init__(self, max_length=GENERAL_LIMIT_SIZE, can_be_empty=False, **kwargs): if not max_length > 0: raise PlenumValueError('max_length', max_length, '> 0') super().__init__(**kwargs) self._max_length = max_length self._can_be_empty = can_be_empty
def merkleInfo(self, seqNo): seqNo = int(seqNo) if seqNo <= 0: raise PlenumValueError('seqNo', seqNo, '> 0') rootHash = self.tree.merkle_tree_hash(0, seqNo) auditPath = self.tree.inclusion_proof(seqNo - 1, seqNo) return { F.rootHash.name: self.hashToStr(rootHash), F.auditPath.name: [self.hashToStr(h) for h in auditPath] }
def disconnectByName(self, remoteName: str): if not remoteName: raise PlenumValueError('remoteName', remoteName, "non-empty string") remote = self.remotes.get(remoteName) if not remote: logger.debug('{} did not find any remote ' 'by name {} to disconnect'.format(self, remoteName)) return None remote.disconnect() return remote
def addRemote(self, name, ha, remoteVerkey, remotePublicKey): if not name: raise PlenumValueError('name', name, 'non-empty') remote = self._RemoteClass(name, ha, remoteVerkey, remotePublicKey, self.queue_size, self.ha[0]) self.remotes[name] = remote # TODO: Use weakref to remote below instead self.remotesByKeys[remotePublicKey] = remote if remoteVerkey: self.addVerifier(remoteVerkey) else: logger.display('{} adding a remote {}({}) without a verkey'.format(self, name, ha)) return remote
def test_PlenumValueError(): aaa = 1 expected = "= 0" exc = PlenumValueError('aaa', aaa, expected, prefix="PREFIX") assert isinstance(exc, PlenumError) assert isinstance(exc, ValueError) exc_str = str(exc) assert exc_str.startswith("PREFIX: ") assert "variable 'aaa'" in exc_str assert "value {}".format(aaa) in exc_str assert "expected: {}".format(expected) in exc_str
def on_future_view_vchd_msg(self, view_no, frm, from_current_state: bool = False): if not ((view_no > self.view_no) or (self.view_no == 0 and from_current_state)): raise PlenumValueError( 'view_no', view_no, ("= 0 or > {}" if from_current_state else "> {}").format(self.view_no), prefix=self ) if view_no not in self._next_view_indications: self._next_view_indications[view_no] = set() self._next_view_indications[view_no].add(frm) self._start_view_change_if_possible(view_no)
def cleanSeed(seed=None): if seed: bts = seedFromHex(seed) if not bts: if isinstance(seed, str): seed = seed.encode('utf-8') bts = bytes(seed) if len(seed) != 32: raise PlenumValueError( 'seed', seed, '64-length hexadecimal string or 32-length in bytes representation' ) return bts
def _commit(ledger, state, txn_count, state_root, txn_root): _, committedTxns = ledger.commitTxns(txn_count) state_root = state_roots_serializer.deserialize( state_root.encode()) if isinstance(state_root, str) else state_root # TODO test for that if ledger.root_hash != txn_root: # Probably the following fail should trigger catchup # TODO add repr / str for Ledger class and dump it here as well raise PlenumValueError( 'txnRoot', txn_root, ("equal to current ledger root hash {}".format( ledger.root_hash))) state.commit(rootHash=state_root) return committedTxns
def commit_batch(self): """ :param state_root: committed state root :param ledger_size: committed ledger size :return: tuple of next committed state and count of committed transactions """ if len(self.un_committed) == 0: raise PlenumValueError( "un_committed", self.un_committed, "commit_batch was called, but there is no tracked uncommitted states" ) uncommitted_hash, uncommitted_size = self.un_committed.popleft() self.last_committed = (uncommitted_hash, uncommitted_size)
def set_root_hash(self, root_hash): if not is_string(root_hash): raise PlenumTypeError('root_hash', root_hash, bytes) if not (len(root_hash) in [0, 32]): raise PlenumValueError('root_hash', root_hash, 'length in range [0, 32]') if self.transient: self.transient_root_hash = root_hash return if root_hash == BLANK_ROOT: self.root_node = BLANK_NODE return # print(repr(root_hash)) self.root_node = self._decode_to_node(root_hash)
def __init__(self, name, encoder, decoder): if not isinstance(name, str): raise PlenumTypeError('name', name, str) if not name: raise PlenumValueError('name', name, 'a non-empty string') if not callable(encoder): raise PlenumTypeError('encoder', encoder, 'callable') if not callable(decoder): raise PlenumTypeError('decoder', decoder, 'callable') self.name = name self.encoder = encoder self.decoder = decoder
def __init__(self, windowSize, delayFunction=None): """ Limits rate of actions performed in a unit of time (window) :param windowSize: size (in seconds) of the time window events counted in :param delayFunction: function from **number of actions** to **time to wait after the last one** """ if not isinstance(windowSize, int): raise PlenumTypeError('windowSize', windowSize, int) if not windowSize > 0: raise PlenumValueError('windowSize', windowSize, '> 0') self.windowSize = windowSize self.delayFunction = delayFunction if delayFunction else self._defaultDelayFunction self.actionsLog = []
def __init__(self, inner_field_type: FieldValidator, min_length=None, max_length=None, **kwargs): if not isinstance(inner_field_type, FieldValidator): raise PlenumTypeError( 'inner_field_type', inner_field_type, FieldValidator) for k in ('min_length', 'max_length'): m = locals()[k] if m is not None: if not isinstance(m, int): raise PlenumTypeError(k, m, int) if not m > 0: raise PlenumValueError(k, m, '> 0') self.inner_field_type = inner_field_type self.min_length = min_length self.max_length = max_length super().__init__(**kwargs)
def appendTxns(self, txns: List): # These transactions are not yet committed so they do not go to # the ledger _no_seq_no_txns = [txn for txn in txns if get_seq_no(txn) is None] if _no_seq_no_txns: raise PlenumValueError( 'txns', txns, ("all txns should have defined seq_no, undefined in {}".format( _no_seq_no_txns))) uncommittedSize = self.size + len(self.uncommittedTxns) self.uncommittedTree = self.treeWithAppliedTxns( txns, self.uncommittedTree) self.uncommittedRootHash = self.uncommittedTree.root_hash self.uncommittedTxns.extend(txns) if txns: return (uncommittedSize + 1, uncommittedSize + len(txns)), txns else: return (uncommittedSize, uncommittedSize), txns
def bootstrapTestNodes(cls, config, startingPort, nodeParamsFileName, domainTxnFieldOrder, config_helper_class=PConfigHelper, node_config_helper_class=PNodeConfigHelper, chroot: str = None): parser = argparse.ArgumentParser( description="Generate pool transactions for testing") parser.add_argument('--nodes', required=True, help='node count should be less than 100', type=cls._bootstrapArgsTypeNodeCount) parser.add_argument('--clients', required=True, type=int, help='client count') parser.add_argument('--nodeNum', type=int, nargs='+', help='the number of the node that will ' 'run on this machine') parser.add_argument('--ips', help='IPs/hostnames of the nodes, provide comma ' 'separated IPs, if no of IPs provided are less' ' than number of nodes then the remaining ' 'nodes are assigned the loopback IP, ' 'i.e 127.0.0.1', type=cls._bootstrap_args_type_ips_hosts) parser.add_argument('--network', help='Network name (default sandbox)', type=str, default="sandbox", required=False) parser.add_argument( '--appendToLedgers', help="Determine if ledger files needs to be erased " "before writing new information or not.", action='store_true') args = parser.parse_args() if isinstance(args.nodeNum, int): if not (1 <= args.nodeNum <= args.nodes): raise PlenumValueError( 'args.nodeNum', args.nodeNum, ">= 1 && <= args.nodes {}".format(args.nodes)) elif isinstance(args.nodeNum, list): if any([True for x in args.nodeNum if not (1 <= x <= args.nodes)]): raise PlenumValueError( 'some items in nodeNum list', args.nodeNum, ">= 1 && <= args.nodes {}".format(args.nodes)) node_num = [args.nodeNum, None] if args.nodeNum else [None] steward_defs, node_defs = cls.gen_defs(args.ips, args.nodes, startingPort) client_defs = cls.gen_client_defs(args.clients) trustee_def = cls.gen_trustee_def(1) if args.nodeNum: # update network during node generation only # edit NETWORK_NAME in config for line in fileinput.input(['/etc/indy/indy_config.py'], inplace=True): if 'NETWORK_NAME' not in line: print(line, end="") with open('/etc/indy/indy_config.py', 'a') as cfgfile: cfgfile.write("NETWORK_NAME = '{}'".format(args.network)) for n_num in node_num: cls.bootstrapTestNodesCore( config, args.network, args.appendToLedgers, domainTxnFieldOrder, trustee_def, steward_defs, node_defs, client_defs, n_num, nodeParamsFileName, config_helper_class, node_config_helper_class) # delete unnecessary key dir in client folder key_dir = cls.setup_clibase_dir(config, args.network) key_dir = os.path.join(key_dir, "keys") if os.path.isdir(key_dir): shutil.rmtree(key_dir, ignore_errors=True)
def validate_value(seq_no): if seq_no <= 0: # Also, this field will be caught on static validation phase and Reply with this message will be sent to client raise PlenumValueError("seq_no should be number and more then 0")
def __init__(self, max_length: int, **kwargs): if not max_length > 0: raise PlenumValueError('max_length', max_length, '> 0') super().__init__(**kwargs) self._max_length = max_length
async def eventually(coroFunc: FlexFunc, *args, retryWait: float = 0.1, timeout: Optional[float] = None, ratchetSteps: Optional[int] = None, acceptableExceptions=None, verbose=True, override_timeout_limit=False) -> T: if timeout is None: timeout = 5 if not timeout > 0: raise PlenumValueError('timeout', timeout, '> 0') if not override_timeout_limit: if timeout > 240: raise PlenumValueError('timeout', timeout, "< 240 or override_timeout_limit=True") else: logger.debug('Overriding timeout limit to {} for evaluating {}'.format( timeout, coroFunc)) if acceptableExceptions and not isinstance(acceptableExceptions, Iterable): acceptableExceptions = [acceptableExceptions] start = time.perf_counter() ratchet = Ratchet.fromGoalDuration(retryWait * slowFactor, ratchetSteps, timeout * slowFactor).gen() \ if ratchetSteps else None fname = get_func_name(coroFunc) while True: remain = 0 try: remain = start + timeout * slowFactor - time.perf_counter() if remain < 0: # this provides a convenient breakpoint for a debugger logger.debug("{} last try...".format(fname), extra={"cli": False}) # noinspection PyCallingNonCallable res = coroFunc(*args) if isawaitable(res): result = await res else: result = res if verbose: recordSuccess(fname, timeout, timeout * slowFactor, remain) logger.debug( "{} succeeded with {:.2f} seconds to spare".format( fname, remain)) return result except Exception as ex: if acceptableExceptions and type(ex) not in acceptableExceptions: raise if remain >= 0: sleep_dur = next(ratchet) if ratchet else retryWait if verbose: logger.trace("{} not succeeded yet, {:.2f} seconds " "remaining..., will sleep for {}".format( fname, remain, sleep_dur)) await asyncio.sleep(sleep_dur) else: recordFail(fname, timeout) logger.error("{} failed; not trying any more because {} " "seconds have passed; args were {}".format( fname, timeout, args)) raise ex
def bootstrap_pool_ledger(cls, config, nodeParamsFileName, config_helper_class=PConfigHelper, node_config_helper_class=PNodeConfigHelper, chroot: str = None): parser = argparse.ArgumentParser( description="Generate pool transactions") parser.add_argument( '--nodeVerkeys', required=True, help='Node Verkey, provide comma separated verification keys', type=cls._bootstrap_args_type_verkeys) parser.add_argument( '--nodeBlskeys', required=True, help='Node BLS keys, provide comma separated Bls keys', type=cls._bootstrap_args_type_bls) parser.add_argument( '--nodeBlsProofs', required=True, help= 'Node Proof of possession for BLS key, provide comma separated proof', type=cls._bootstrap_args_type_bls) parser.add_argument('--nodeName', required=True, help='Node name, provide comma separated name', type=cls._bootstrap_args_type_list) parser.add_argument('--nodePort', required=True, help='Node port, provide comma separated port', type=cls._bootstrap_args_type_port) parser.add_argument('--clientPort', required=True, help='Client port, provide comma separated port', type=cls._bootstrap_args_type_port) parser.add_argument('--stewardDids', required=True, help='Stewards Dids, provide comma separated dids', type=cls._bootstrap_args_type_dids) parser.add_argument('--nodeNum', type=int, nargs='+', help='the number of the node that will ' 'run on this machine') parser.add_argument( '--ips', help='IPs/hostnames of the nodes, provide comma ' 'separated IPs, if no of IPS provided are less than number of nodes then the remaining' 'nodes are assigned the loopback IP, ' 'i.e 127.0.0.1', type=cls._bootstrap_args_type_ips_hosts) parser.add_argument('--network', required=False, help='Network name (default sandbox)', type=str, default="sandbox") parser.add_argument( '--appendToLedgers', help="Determine if ledger files needs to be erased " "before writing new information or not", action='store_true') args = parser.parse_args() if ((len(args.nodeName) != len(args.nodeVerkeys)) or (len(args.nodeBlskeys) != len(args.nodeVerkeys)) or (len(args.nodeBlskeys) != len(args.nodeBlsProofs)) or (len(args.nodePort) != len(args.nodeBlsProofs)) or (len(args.nodePort) != len(args.clientPort)) or (len(args.stewardDids) != len(args.clientPort))): raise argparse.ArgumentTypeError("Some arguments are missing.") if isinstance(args.nodeNum, int): if not (1 <= args.nodeNum <= len(args.nodeName)): raise PlenumValueError( 'args.nodeNum', args.nodeNum, ">= 1 && <= len(args.nodeName) {}".format( len(args.nodeName))) elif isinstance(args.nodeNum, list): if any([ True for x in args.nodeNum if not (1 <= x <= len(args.nodeName)) ]): raise PlenumValueError( 'some items in nodeNum list', args.nodeNum, ">=1 && <= len(args.nodeName) {}".format( len(args.nodeName))) node_num = [args.nodeNum, None] if args.nodeNum else [None] node_defs = cls.gen_node_def(args.nodeName, args.ips, args.nodePort, args.clientPort, args.nodeBlskeys, args.nodeBlsProofs, args.nodeVerkeys, args.stewardDids) if args.nodeNum: for line in fileinput.input(['/etc/indy/indy_config.py'], inplace=True): if 'NETWORK_NAME' not in line: print(line, end="") with open('/etc/indy/indy_config.py', 'a') as cfgfile: cfgfile.write("NETWORK_NAME = '{}'".format(args.network)) for n_num in node_num: cls.bootstrap_pool_ledger_core(config, args.network, args.appendToLedgers, node_defs, n_num, nodeParamsFileName, config_helper_class, node_config_helper_class)