def _split_pool_and_domain(self): pool_txns = deque() domain_txns = deque() for txn in self.genesis_txns: if get_type(txn) in [NODE]: pool_txns.appendleft(txn) elif get_type(txn) in [NYM]: domain_txns.appendleft(txn) else: raise NotImplementedError("txn type '{}' not supported") return pool_txns, domain_txns
def test_proof_in_write_reply(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): resp = sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 1) req = resp[0][0] result = resp[0][1]['result'] assert result assert get_type(result) == "buy" assert get_from(result) == req[f.IDENTIFIER.nm] assert get_req_id(result) == req[f.REQ_ID.nm] assert get_seq_no(result) assert get_txn_time(result) assert STATE_PROOF in result state_proof = result[STATE_PROOF] assert ROOT_HASH in state_proof assert MULTI_SIGNATURE in state_proof assert PROOF_NODES in state_proof multi_sig = state_proof[MULTI_SIGNATURE] assert MULTI_SIGNATURE_SIGNATURE in multi_sig assert MULTI_SIGNATURE_PARTICIPANTS in multi_sig assert MULTI_SIGNATURE_VALUE in multi_sig multi_sig_value = multi_sig[MULTI_SIGNATURE_VALUE] assert MULTI_SIGNATURE_VALUE_LEDGER_ID in multi_sig_value assert MULTI_SIGNATURE_VALUE_STATE_ROOT in multi_sig_value assert MULTI_SIGNATURE_VALUE_TXN_ROOT in multi_sig_value assert MULTI_SIGNATURE_VALUE_POOL_STATE_ROOT in multi_sig_value assert MULTI_SIGNATURE_VALUE_TIMESTAMP in multi_sig_value assert validate_multi_signature(state_proof, txnPoolNodeSet) assert validate_proof_for_write(result)
def nymsAddedInQuickSuccession(looper, nodeSet, sdk_added_raw_attribute, trustAnchor, trustAnchorWallet): usigner = DidSigner() nym = usigner.verkey idy = Identity(identifier=nym) trustAnchorWallet.addTrustAnchoredIdentity(idy) # Creating a NYM request with same nym again req = idy.ledgerRequest() trustAnchorWallet._pending.appendleft((req, idy.identifier)) reqs = trustAnchorWallet.preparePending() trustAnchor.submitReqs(*reqs) def check(): assert trustAnchorWallet._trustAnchored[nym].seqNo timeout = waits.expectedTransactionExecutionTime(len(nodeSet)) looper.run(eventually(check, timeout=timeout)) timeout = waits.expectedReqNAckQuorumTime() looper.run(eventually(checkNacks, trustAnchor, req.reqId, "is already added", retryWait=1, timeout=timeout)) count = 0 for node in nodeSet: for seq, txn in node.domainLedger.getAllTxn(): if get_type(txn) == NYM and get_payload_data(txn)[TARGET_NYM] == usigner.identifier: count += 1 assert (count == len(nodeSet))
def check_ledger_after_upgrade( node_set, allowed_actions, ledger_size, expected_version, allowed_txn_types=[NODE_UPGRADE], node_ids=None): versions = set() for node in node_set: # print(len(node.configLedger)) assert len(node.configLedger) == ledger_size ids = set() for _, txn in node.configLedger.getAllTxn(): type = get_type(txn) assert type in allowed_txn_types txn_data = get_payload_data(txn) data = txn_data if type == NODE_UPGRADE: data = txn_data[DATA] assert data[ACTION] assert data[ACTION] in allowed_actions ids.add(get_from(txn)) assert data[VERSION] versions.add(data[VERSION]) ids.add(node.id) if node_ids: assert ids == set(node_ids) assert len(versions) == 1 assert list(versions)[0] == expected_version
def _updateStateWithSingleTxn(self, txn, isCommitted=False): txn_type = get_type(txn) if txn_type in self.state_update_handlers: self.state_update_handlers[txn_type](txn, isCommitted) else: logger.debug( 'Cannot apply request of type {} to state'.format(txn_type))
def nodeExistsInLedger(self, nym): # Since PoolLedger is going to be small so using # `getAllTxn` is fine for _, txn in self.ledger.getAllTxn(): if get_type(txn) == NODE and \ get_payload_data(txn)[TARGET_NYM] == nym: return True return False
def txnPoolCliNodeReg(poolTxnData): cliNodeReg = {} for txn in poolTxnData["txns"]: if get_type(txn) == NODE: data = get_payload_data(txn)[DATA] cliNodeReg[data[ALIAS] + CLIENT_STACK_SUFFIX] = HA(data[CLIENT_IP], data[CLIENT_PORT]) return cliNodeReg
def custom_tdir_with_pool_txns(pool_txn_data, tdir_for_pool_txns, pool_transactions_file_name): ledger = create_genesis_txn_init_ledger(tdir_for_pool_txns, pool_transactions_file_name) for item in pool_txn_data["txns"]: if get_type(item) == NODE: ledger.add(item) ledger.stop() return tdir_for_pool_txns
def handleConfigTxn(self, txn) -> None: """ Handles transaction of type POOL_CONFIG :param txn: """ if get_type(txn) == POOL_CONFIG: self.writes = get_payload_data(txn)[WRITES]
def getNodesServices(self): # Returns services for each node srvs = dict() for _, txn in self.ledger.getAllTxn(): txn_data = get_payload_data(txn) if get_type(txn) == NODE and \ txn_data.get(DATA, {}).get(SERVICES) is not None: srvs.update({txn_data[TARGET_NYM]: txn_data[DATA][SERVICES]}) return srvs
def tdirWithDomainTxns(config_helper_class, poolTxnData, tdir, tconf, domainTxnOrderedFields): config_helper = config_helper_class(tconf, chroot=tdir) ledger = create_genesis_txn_init_ledger(config_helper.genesis_dir, tconf.domainTransactionsFile) for item in poolTxnData["txns"]: if get_type(item) == NYM: ledger.add(item) ledger.stop() return config_helper.genesis_dir
def transform_txn_for_ledger(txn): """ Some transactions need to be transformed before they can be stored in the ledger, eg. storing certain payload in another data store and only its hash in the ledger """ if get_type(txn) == ATTRIB: txn = DomainReqHandler.transform_attrib_for_ledger(txn) return txn
def custom_tdir_with_domain_txns(pool_txn_data, tdir_for_domain_txns, domain_txn_ordered_fields, domain_transactions_file_name): ledger = create_genesis_txn_init_ledger(tdir_for_domain_txns, domain_transactions_file_name) for item in pool_txn_data["txns"]: if get_type(item) == NYM: ledger.add(item) ledger.stop() return tdir_for_domain_txns
def getTxnsByType(self, txnType: str) -> List: txns = [] for val in self.transactionLog.iterator(include_key=False, include_value=True): txn = self.serializer.deserialize( val, fields=self.txnFieldOrdering) if get_type(txn) == txnType: txns.append(txn) return txns
def _updateStateWithSingleTxn(self, txn, isCommitted=False): typ = get_type(txn) if typ == 'buy': key, value = self.prepare_buy_for_state(txn) self.state.set(key, value) logger.trace('{} after adding to state, headhash is {}'. format(self, self.state.headHash)) else: super()._updateStateWithSingleTxn(txn, isCommitted=isCommitted)
def sendRepliesToClients(self, committedTxns, ppTime): committedTxns = list(committedTxns) req_handler = self.get_req_handler(DOMAIN_LEDGER_ID) for txn in committedTxns: if get_type(txn) == "buy": key, value = req_handler.prepare_buy_for_state(txn) _, proof = req_handler.get_value_from_state(key, with_proof=True) if proof: txn[STATE_PROOF] = proof super().sendRepliesToClients(committedTxns, ppTime)
def test_genesis_nodes(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): assert len(txnPoolNodeSet) == nodeCount for node in txnPoolNodeSet: assertEquality(node.poolLedger.size, nodeCount) stw_count = sum(1 for _, txn in node.domainLedger.getAllTxn() if (get_type(txn) == NYM) and (get_payload_data(txn).get(ROLE) == STEWARD)) assertEquality(stw_count, nodeCount) sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def _load_nodes_order_from_ledger(self): self._ordered_node_ids = OrderedDict() self._ordered_node_services = {} for _, txn in self.ledger.getAllTxn(): if get_type(txn) == NODE: txn_data = get_payload_data(txn) self._set_node_ids_in_cash(txn_data[TARGET_NYM], txn_data[DATA][ALIAS]) self._set_node_services_in_cash(txn_data[TARGET_NYM], txn_data[DATA].get(SERVICES, None))
def processLedger(self) -> None: """ Checks ledger config txns and perfomes recent one :return: """ logger.debug('{} processing config ledger for any POOL_CONFIGs'.format( self), extra={"tags": ["pool-config"]}) for _, txn in self.ledger.getAllTxn(): if get_type(txn) == POOL_CONFIG: self.handleConfigTxn(txn)
def _load_keys_for_root(self): keys = {} for _, txn in self._ledger.getAllTxn(): if get_type(txn) == NODE: data = get_payload_data(txn)[DATA] if not config.VALIDATE_BLS_SIGNATURE_WITHOUT_KEY_PROOF and \ data.get(BLS_KEY_PROOF, None) is None: logger.warning("{} has no proof of possession for BLS public key.".format(data[ALIAS])) keys[data[ALIAS]] = None else: keys[data[ALIAS]] = data.get(BLS_KEY, None) return keys
def tdirWithPoolTxns(config_helper_class, poolTxnData, tdir, tconf): import getpass logging.debug("current user when creating new pool txn file: {}". format(getpass.getuser())) config_helper = config_helper_class(tconf, chroot=tdir) ledger = create_genesis_txn_init_ledger(config_helper.genesis_dir, tconf.poolTransactionsFile) for item in poolTxnData["txns"]: if get_type(item) == NODE: ledger.add(item) ledger.stop() return config_helper.genesis_dir
def _addAttr(self, txn, isCommitted=False) -> None: """ The state trie stores the hash of the whole attribute data at: the did+attribute name if the data is plaintext (RAW) the did+hash(attribute) if the data is encrypted (ENC) If the attribute is HASH, then nothing is stored in attribute store, the trie stores a blank value for the key did+hash """ assert get_type(txn) == ATTRIB attr_type, path, value, hashed_value, value_bytes = domain.prepare_attr_for_state(txn) self.state.set(path, value_bytes) if attr_type != HASH: self.attributeStore.set(hashed_value, value)
def prepare_attr_for_state(txn): """ Make key(path)-value pair for state from ATTRIB or GET_ATTR :return: state path, state value, value for attribute store """ assert get_type(txn) == ATTRIB txn_data = get_payload_data(txn) nym = txn_data[TARGET_NYM] attr_type, attr_key, value = parse_attr_txn(txn_data) hashed_value = hash_of(value) if value else '' seq_no = get_seq_no(txn) txn_time = get_txn_time(txn) value_bytes = encode_state_value(hashed_value, seq_no, txn_time) path = make_state_path_for_attr(nym, attr_key, attr_type == HASH) return attr_type, path, value, hashed_value, value_bytes
def update_txn_with_extra_data(self, txn): """ All the data of the transaction might not be stored in ledger so the extra data that is omitted from ledger needs to be fetched from the appropriate data store :param txn: :return: """ # For RAW and ENC attributes, only hash is stored in the ledger. if get_type(txn) == ATTRIB: txn_data = get_payload_data(txn) # The key needs to be present and not None key = RAW if (RAW in txn_data and txn_data[RAW] is not None) else \ ENC if (ENC in txn_data and txn_data[ENC] is not None) else \ None if key: txn_data[key] = self.attributeStore.get(txn_data[key]) return txn
def put_into_seq_no_db(txn): # If there is no reqId, then it's genesis txn if get_req_id(txn) is None: return txn_new = copy.deepcopy(txn) operation = get_payload_data(txn_new) operation[TXN_TYPE] = get_type(txn_new) dct = { f.IDENTIFIER.nm: get_from(txn_new), f.REQ_ID.nm: get_req_id(txn_new), OPERATION: operation, } if get_protocol_version(txn_new) is not None: dct[f.PROTOCOL_VERSION.nm] = get_protocol_version(txn_new) digest = sha256(serialize_msg_for_signing(dct)).hexdigest() seq_no = get_seq_no(txn_new) ledger_id = get_ledger_id_by_txn_type(operation[TXN_TYPE]) line_to_record = str(ledger_id) + ReqIdrToTxn.delimiter + str(seq_no) dest_seq_no_db_storage.put(digest, line_to_record) return digest
def _parse_pool_transaction_file( ledger, nodeReg, cliNodeReg, nodeKeys, activeValidators, ledger_size=None): """ helper function for parseLedgerForHaAndKeys """ for _, txn in ledger.getAllTxn(to=ledger_size): if get_type(txn) == NODE: txn_data = get_payload_data(txn) nodeName = txn_data[DATA][ALIAS] clientStackName = nodeName + CLIENT_STACK_SUFFIX nHa = (txn_data[DATA][NODE_IP], txn_data[DATA][NODE_PORT]) \ if (NODE_IP in txn_data[DATA] and NODE_PORT in txn_data[DATA]) \ else None cHa = (txn_data[DATA][CLIENT_IP], txn_data[DATA][CLIENT_PORT]) \ if (CLIENT_IP in txn_data[DATA] and CLIENT_PORT in txn_data[DATA]) \ else None if nHa: nodeReg[nodeName] = HA(*nHa) if cHa: cliNodeReg[clientStackName] = HA(*cHa) try: # TODO: Need to handle abbreviated verkey key_type = 'verkey' verkey = cryptonymToHex(str(txn_data[TARGET_NYM])) key_type = 'identifier' cryptonymToHex(get_from(txn)) except ValueError: logger.exception( 'Invalid {}. Rebuild pool transactions.'.format(key_type)) exit('Invalid {}. Rebuild pool transactions.'.format(key_type)) nodeKeys[nodeName] = verkey services = txn_data[DATA].get(SERVICES) if isinstance(services, list): if VALIDATOR in services: activeValidators.add(nodeName) else: activeValidators.discard(nodeName)
def _sendIncorrectTxns(self, req, frm): ledgerId = getattr(req, f.LEDGER_ID.nm) if ledgerId == DOMAIN_LEDGER_ID: logger.info("{} being malicious and sending incorrect transactions" " for catchup request {} from {}". format(self, req, frm)) start, end = getattr(req, f.SEQ_NO_START.nm), \ getattr(req, f.SEQ_NO_END.nm) ledger = self.getLedgerForMsg(req) txns = {} for seqNo, txn in ledger.getAllTxn(start, end): # Since the type of random request is `buy` if get_type(txn) == "buy": set_type(txn, "randombuy") txns[seqNo] = txn consProof = [Ledger.hashToStr(p) for p in ledger.tree.consistency_proof(end, ledger.size)] self.sendTo(msg=CatchupRep(getattr(req, f.LEDGER_ID.nm), txns, consProof), to=frm) else: self.processCatchupReq(req, frm)
def executeAndCheckGenTxn(cli, cmd, typ, nym, role=None, data=None): checkCmdValid(cli, cmd) nymCorrect = False roleCorrect = False if role else True dataCorrect = False if data else True typeCorrect = False if typ else True role = Roles[role].value if role else role for txn in cli.genesisTransactions: txn_data = get_payload_data(txn) if txn_data.get(TARGET_NYM) == nym: nymCorrect = True if get_type(txn) == typ: typeCorrect = True if txn_data.get(ROLE) == role: roleCorrect = True if data and txn_data.get(DATA) == json.loads(data): dataCorrect = True assert typeCorrect and nymCorrect and roleCorrect and dataCorrect assert "Genesis transaction added" in cli.lastCmdOutput
def test_pool_genesis_txns(bootstrap, pool_genesis_file): serializer = JsonSerializer() with open(pool_genesis_file) as f: for line in store_utils.cleanLines(f.readlines()): txn = serializer.deserialize(line) assert get_seq_no(txn) assert get_txn_id(txn) assert get_payload_data(txn) assert get_type(txn) == NODE assert get_version(txn) == "1" assert get_protocol_version(txn) is None assert get_payload_data(txn)[TARGET_NYM] data = get_payload_data(txn).get(DATA) assert data assert data[ALIAS] assert data[CLIENT_IP] assert data[CLIENT_PORT] assert data[NODE_IP] assert data[NODE_PORT] assert data[SERVICES] == [VALIDATOR] assert data[BLS_KEY] assert data[BLS_KEY_PROOF]
def onPoolMembershipChange(self, txn): # `onPoolMembershipChange` method can be called only after txn added to ledger if get_type(txn) != NODE: return txn_data = get_payload_data(txn) if DATA not in txn_data: return nodeName = txn_data[DATA][ALIAS] nodeNym = txn_data[TARGET_NYM] self._set_node_ids_in_cash(nodeNym, nodeName) def _updateNode(txn_data): if SERVICES in txn_data[DATA]: self.nodeServicesChanged(txn_data) if txn_data[DATA][ALIAS] in self.node.nodeReg: if {NODE_IP, NODE_PORT, CLIENT_IP, CLIENT_PORT}. \ intersection(set(txn_data[DATA].keys())): self.nodeHaChanged(txn_data) if VERKEY in txn_data: self.nodeKeysChanged(txn_data) if BLS_KEY in txn_data[DATA]: self.node_blskey_changed(txn_data) # If nodeNym is never added in self._ordered_node_services, # nodeNym is never added in ledger if nodeNym not in self._ordered_node_services: txn_data[DATA].setdefault(SERVICES, []) if VALIDATOR in txn_data[DATA].get(SERVICES, []): self.addNewNodeAndConnect(txn_data) else: _updateNode(txn_data) self._set_node_services_in_cash(nodeNym, txn_data[DATA].get(SERVICES, None))
def _addClaimDef(self, txn, isCommitted=False) -> None: assert get_type(txn) == CLAIM_DEF path, value_bytes = domain.prepare_claim_def_for_state(txn) self.state.set(path, value_bytes)
def prepare_for_state(result): if get_type(result) == "buy": key, value = GetBuyHandler.prepare_buy_for_state(result) return key, value
def get_last_node_upgrade_txn(self, start_no: int = None): return self.get_upgrade_txn(lambda txn: get_type(txn) == NODE_UPGRADE and get_from(txn) == self.nodeId, start_no=start_no, reverse=True)
def filter_fees(ledger_id: int, txn: Any): origin_token_post_added_clb(ledger_id, txn, get_type(txn) != FeesTransactions.FEES.value)
def handleUpgradeTxn(self, txn) -> None: """ Handles transaction of type POOL_UPGRADE Can schedule or cancel upgrade to a newer version at specified time :param txn: """ FINALIZING_EVENT_TYPES = [ UpgradeLog.Events.succeeded, UpgradeLog.Events.failed ] if get_type(txn) != POOL_UPGRADE: return logger.info("Node '{}' handles upgrade txn {}".format( self.nodeName, txn)) txn_data = get_payload_data(txn) action = txn_data[ACTION] version = txn_data[VERSION] justification = txn_data.get(JUSTIFICATION) pkg_name = txn_data.get(PACKAGE, self.config.UPGRADE_ENTRY) upgrade_id = self.get_action_id(txn) # TODO test try: version = src_version_cls(pkg_name)(version) except InvalidVersionError as exc: logger.warning( "{} can't handle upgrade txn with version {} for package {}: {}" .format(self, version, pkg_name, exc)) return if action == START: # forced txn could have partial schedule list if self.nodeId not in txn_data[SCHEDULE]: logger.info("Node '{}' disregards upgrade txn {}".format( self.nodeName, txn)) return last_event = self.lastActionEventInfo if last_event: if (last_event.data.upgrade_id == upgrade_id and last_event.ev_type in FINALIZING_EVENT_TYPES): logger.info( "Node '{}' has already performed an upgrade with upgrade_id {}. " "Last recorded event is {}".format( self.nodeName, upgrade_id, last_event.data)) return when = txn_data[SCHEDULE][self.nodeId] failTimeout = txn_data.get(TIMEOUT, self.defaultActionTimeout) if isinstance(when, str): when = dateutil.parser.parse(when) new_ev_data = UpgradeLogData(when, version, upgrade_id, pkg_name) if self.scheduledAction: if self.scheduledAction == new_ev_data: logger.debug( "Node {} already scheduled upgrade to version '{}' ". format(self.nodeName, version)) return else: logger.info( "Node '{}' cancels previous upgrade and schedules a new one to {}" .format(self.nodeName, version)) self._cancelScheduledUpgrade(justification) logger.info("Node '{}' schedules upgrade to {}".format( self.nodeName, version)) self._scheduleUpgrade(new_ev_data, failTimeout) return if action == CANCEL: if (self.scheduledAction and self.scheduledAction.version == version): self._cancelScheduledUpgrade(justification) logger.info("Node '{}' cancels upgrade to {}".format( self.nodeName, version)) return logger.error("Got {} transaction with unsupported action {}".format( POOL_UPGRADE, action))
def pool_node_txns(poolTxnData): node_txns = [] for txn in poolTxnData["txns"]: if get_type(txn) == NODE: node_txns.append(txn) return node_txns
def _updateStateWithSingleTxn(self, txn, isCommitted=False): typ = get_type(txn) if typ == WRITE_CONF: conf = json.loads(get_payload_data(txn)[DATA]) key, val = conf.popitem() self.state.set(key.encode(), val.encode())
def test_get_type(txn): assert get_type(txn) == NYM
def _addSchema(self, txn, isCommitted=False) -> None: assert get_type(txn) == SCHEMA path, value_bytes = domain.prepare_schema_for_state(txn) self.state.set(path, value_bytes)
def test_set_type(txn): txn = set_type(txn, NODE) assert get_type(txn) == NODE
def _addRevocDef(self, txn, isCommitted=False) -> None: assert get_type(txn) == REVOC_REG_DEF path, value_bytes = domain.prepare_revoc_def_for_state(txn) self.state.set(path, value_bytes)
def _load_nodes_order_from_ledger(self): self._ordered_node_ids = OrderedDict() for _, txn in self.ledger.getAllTxn(): if get_type(txn) == NODE: txn_data = get_payload_data(txn) self._set_node_order(txn_data[TARGET_NYM], txn_data[DATA][ALIAS])
def _validate_txn_type(self, txn: Dict): if get_type(txn) != self.txn_type: raise LogicError
def hasNym(self, nym): for txn in self.txnLog.getTxnsByType(NYM): if get_type(txn) == NYM: return True return False
def prepare_for_state(self, result): if get_type(result) == "buy": from plenum.test.test_node import TestDomainRequestHandler key, value = TestDomainRequestHandler.prepare_buy_for_state(result) return key, value
def transform_txn_for_ledger(self, txn): handlers = self.request_handlers.get(get_type(txn), None) if handlers is None: raise LogicError return handlers[0].transform_txn_for_ledger(txn)