def _split_pool_and_domain(self):
pool_txns = deque()
domain_txns = deque()
for txn in self.genesis_txns:
if get_type(txn) in [NODE]:
pool_txns.appendleft(txn)
elif get_type(txn) in [NYM]:
domain_txns.appendleft(txn)
else:
raise NotImplementedError("txn type '{}' not supported")
return pool_txns, domain_txns
def test_proof_in_write_reply(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client):
resp = sdk_send_random_and_check(looper, txnPoolNodeSet,
sdk_pool_handle, sdk_wallet_client, 1)
req = resp[0][0]
result = resp[0][1]['result']
assert result
assert get_type(result) == "buy"
assert get_from(result) == req[f.IDENTIFIER.nm]
assert get_req_id(result) == req[f.REQ_ID.nm]
assert get_seq_no(result)
assert get_txn_time(result)
assert STATE_PROOF in result
state_proof = result[STATE_PROOF]
assert ROOT_HASH in state_proof
assert MULTI_SIGNATURE in state_proof
assert PROOF_NODES in state_proof
multi_sig = state_proof[MULTI_SIGNATURE]
assert MULTI_SIGNATURE_SIGNATURE in multi_sig
assert MULTI_SIGNATURE_PARTICIPANTS in multi_sig
assert MULTI_SIGNATURE_VALUE in multi_sig
multi_sig_value = multi_sig[MULTI_SIGNATURE_VALUE]
assert MULTI_SIGNATURE_VALUE_LEDGER_ID in multi_sig_value
assert MULTI_SIGNATURE_VALUE_STATE_ROOT in multi_sig_value
assert MULTI_SIGNATURE_VALUE_TXN_ROOT in multi_sig_value
assert MULTI_SIGNATURE_VALUE_POOL_STATE_ROOT in multi_sig_value
assert MULTI_SIGNATURE_VALUE_TIMESTAMP in multi_sig_value
assert validate_multi_signature(state_proof, txnPoolNodeSet)
assert validate_proof_for_write(result)
def nymsAddedInQuickSuccession(looper, nodeSet, sdk_added_raw_attribute,
trustAnchor, trustAnchorWallet):
usigner = DidSigner()
nym = usigner.verkey
idy = Identity(identifier=nym)
trustAnchorWallet.addTrustAnchoredIdentity(idy)
# Creating a NYM request with same nym again
req = idy.ledgerRequest()
trustAnchorWallet._pending.appendleft((req, idy.identifier))
reqs = trustAnchorWallet.preparePending()
trustAnchor.submitReqs(*reqs)
def check():
assert trustAnchorWallet._trustAnchored[nym].seqNo
timeout = waits.expectedTransactionExecutionTime(len(nodeSet))
looper.run(eventually(check, timeout=timeout))
timeout = waits.expectedReqNAckQuorumTime()
looper.run(eventually(checkNacks,
trustAnchor,
req.reqId,
"is already added",
retryWait=1, timeout=timeout))
count = 0
for node in nodeSet:
for seq, txn in node.domainLedger.getAllTxn():
if get_type(txn) == NYM and get_payload_data(txn)[TARGET_NYM] == usigner.identifier:
count += 1
assert (count == len(nodeSet))
def check_ledger_after_upgrade(
node_set,
allowed_actions,
ledger_size,
expected_version,
allowed_txn_types=[NODE_UPGRADE],
node_ids=None):
versions = set()
for node in node_set:
# print(len(node.configLedger))
assert len(node.configLedger) == ledger_size
ids = set()
for _, txn in node.configLedger.getAllTxn():
type = get_type(txn)
assert type in allowed_txn_types
txn_data = get_payload_data(txn)
data = txn_data
if type == NODE_UPGRADE:
data = txn_data[DATA]
assert data[ACTION]
assert data[ACTION] in allowed_actions
ids.add(get_from(txn))
assert data[VERSION]
versions.add(data[VERSION])
ids.add(node.id)
if node_ids:
assert ids == set(node_ids)
assert len(versions) == 1
assert list(versions)[0] == expected_version
def _updateStateWithSingleTxn(self, txn, isCommitted=False):
txn_type = get_type(txn)
if txn_type in self.state_update_handlers:
self.state_update_handlers[txn_type](txn, isCommitted)
else:
logger.debug(
'Cannot apply request of type {} to state'.format(txn_type))
def nodeExistsInLedger(self, nym):
# Since PoolLedger is going to be small so using
# `getAllTxn` is fine
for _, txn in self.ledger.getAllTxn():
if get_type(txn) == NODE and \
get_payload_data(txn)[TARGET_NYM] == nym:
return True
return False
def txnPoolCliNodeReg(poolTxnData):
cliNodeReg = {}
for txn in poolTxnData["txns"]:
if get_type(txn) == NODE:
data = get_payload_data(txn)[DATA]
cliNodeReg[data[ALIAS] +
CLIENT_STACK_SUFFIX] = HA(data[CLIENT_IP], data[CLIENT_PORT])
return cliNodeReg
def custom_tdir_with_pool_txns(pool_txn_data, tdir_for_pool_txns, pool_transactions_file_name):
ledger = create_genesis_txn_init_ledger(tdir_for_pool_txns, pool_transactions_file_name)
for item in pool_txn_data["txns"]:
if get_type(item) == NODE:
ledger.add(item)
ledger.stop()
return tdir_for_pool_txns
def handleConfigTxn(self, txn) -> None:
"""
Handles transaction of type POOL_CONFIG
:param txn:
"""
if get_type(txn) == POOL_CONFIG:
self.writes = get_payload_data(txn)[WRITES]
def getNodesServices(self):
# Returns services for each node
srvs = dict()
for _, txn in self.ledger.getAllTxn():
txn_data = get_payload_data(txn)
if get_type(txn) == NODE and \
txn_data.get(DATA, {}).get(SERVICES) is not None:
srvs.update({txn_data[TARGET_NYM]: txn_data[DATA][SERVICES]})
return srvs
def tdirWithDomainTxns(config_helper_class, poolTxnData, tdir, tconf, domainTxnOrderedFields):
config_helper = config_helper_class(tconf, chroot=tdir)
ledger = create_genesis_txn_init_ledger(config_helper.genesis_dir, tconf.domainTransactionsFile)
for item in poolTxnData["txns"]:
if get_type(item) == NYM:
ledger.add(item)
ledger.stop()
return config_helper.genesis_dir
def transform_txn_for_ledger(txn):
"""
Some transactions need to be transformed before they can be stored in the
ledger, eg. storing certain payload in another data store and only its
hash in the ledger
"""
if get_type(txn) == ATTRIB:
txn = DomainReqHandler.transform_attrib_for_ledger(txn)
return txn
def custom_tdir_with_domain_txns(pool_txn_data, tdir_for_domain_txns,
domain_txn_ordered_fields, domain_transactions_file_name):
ledger = create_genesis_txn_init_ledger(tdir_for_domain_txns, domain_transactions_file_name)
for item in pool_txn_data["txns"]:
if get_type(item) == NYM:
ledger.add(item)
ledger.stop()
return tdir_for_domain_txns
def getTxnsByType(self, txnType: str) -> List:
txns = []
for val in self.transactionLog.iterator(include_key=False,
include_value=True):
txn = self.serializer.deserialize(
val, fields=self.txnFieldOrdering)
if get_type(txn) == txnType:
txns.append(txn)
return txns
def _updateStateWithSingleTxn(self, txn, isCommitted=False):
typ = get_type(txn)
if typ == 'buy':
key, value = self.prepare_buy_for_state(txn)
self.state.set(key, value)
logger.trace('{} after adding to state, headhash is {}'.
format(self, self.state.headHash))
else:
super()._updateStateWithSingleTxn(txn, isCommitted=isCommitted)
def sendRepliesToClients(self, committedTxns, ppTime):
committedTxns = list(committedTxns)
req_handler = self.get_req_handler(DOMAIN_LEDGER_ID)
for txn in committedTxns:
if get_type(txn) == "buy":
key, value = req_handler.prepare_buy_for_state(txn)
_, proof = req_handler.get_value_from_state(key, with_proof=True)
if proof:
txn[STATE_PROOF] = proof
super().sendRepliesToClients(committedTxns, ppTime)
def test_genesis_nodes(looper, txnPoolNodeSet,
sdk_pool_handle,
sdk_wallet_client):
assert len(txnPoolNodeSet) == nodeCount
for node in txnPoolNodeSet:
assertEquality(node.poolLedger.size, nodeCount)
stw_count = sum(1 for _, txn in node.domainLedger.getAllTxn() if
(get_type(txn) == NYM) and (get_payload_data(txn).get(ROLE) == STEWARD))
assertEquality(stw_count, nodeCount)
sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle)
def _load_nodes_order_from_ledger(self):
self._ordered_node_ids = OrderedDict()
self._ordered_node_services = {}
for _, txn in self.ledger.getAllTxn():
if get_type(txn) == NODE:
txn_data = get_payload_data(txn)
self._set_node_ids_in_cash(txn_data[TARGET_NYM],
txn_data[DATA][ALIAS])
self._set_node_services_in_cash(txn_data[TARGET_NYM],
txn_data[DATA].get(SERVICES, None))
def processLedger(self) -> None:
"""
Checks ledger config txns and perfomes recent one
:return:
"""
logger.debug('{} processing config ledger for any POOL_CONFIGs'.format(
self), extra={"tags": ["pool-config"]})
for _, txn in self.ledger.getAllTxn():
if get_type(txn) == POOL_CONFIG:
self.handleConfigTxn(txn)
def _load_keys_for_root(self):
keys = {}
for _, txn in self._ledger.getAllTxn():
if get_type(txn) == NODE:
data = get_payload_data(txn)[DATA]
if not config.VALIDATE_BLS_SIGNATURE_WITHOUT_KEY_PROOF and \
data.get(BLS_KEY_PROOF, None) is None:
logger.warning("{} has no proof of possession for BLS public key.".format(data[ALIAS]))
keys[data[ALIAS]] = None
else:
keys[data[ALIAS]] = data.get(BLS_KEY, None)
return keys
def tdirWithPoolTxns(config_helper_class, poolTxnData, tdir, tconf):
import getpass
logging.debug("current user when creating new pool txn file: {}".
format(getpass.getuser()))
config_helper = config_helper_class(tconf, chroot=tdir)
ledger = create_genesis_txn_init_ledger(config_helper.genesis_dir, tconf.poolTransactionsFile)
for item in poolTxnData["txns"]:
if get_type(item) == NODE:
ledger.add(item)
ledger.stop()
return config_helper.genesis_dir
def _addAttr(self, txn, isCommitted=False) -> None:
"""
The state trie stores the hash of the whole attribute data at:
the did+attribute name if the data is plaintext (RAW)
the did+hash(attribute) if the data is encrypted (ENC)
If the attribute is HASH, then nothing is stored in attribute store,
the trie stores a blank value for the key did+hash
"""
assert get_type(txn) == ATTRIB
attr_type, path, value, hashed_value, value_bytes = domain.prepare_attr_for_state(txn)
self.state.set(path, value_bytes)
if attr_type != HASH:
self.attributeStore.set(hashed_value, value)
def prepare_attr_for_state(txn):
"""
Make key(path)-value pair for state from ATTRIB or GET_ATTR
:return: state path, state value, value for attribute store
"""
assert get_type(txn) == ATTRIB
txn_data = get_payload_data(txn)
nym = txn_data[TARGET_NYM]
attr_type, attr_key, value = parse_attr_txn(txn_data)
hashed_value = hash_of(value) if value else ''
seq_no = get_seq_no(txn)
txn_time = get_txn_time(txn)
value_bytes = encode_state_value(hashed_value, seq_no, txn_time)
path = make_state_path_for_attr(nym, attr_key, attr_type == HASH)
return attr_type, path, value, hashed_value, value_bytes
def update_txn_with_extra_data(self, txn):
"""
All the data of the transaction might not be stored in ledger so the
extra data that is omitted from ledger needs to be fetched from the
appropriate data store
:param txn:
:return:
"""
# For RAW and ENC attributes, only hash is stored in the ledger.
if get_type(txn) == ATTRIB:
txn_data = get_payload_data(txn)
# The key needs to be present and not None
key = RAW if (RAW in txn_data and txn_data[RAW] is not None) else \
ENC if (ENC in txn_data and txn_data[ENC] is not None) else \
None
if key:
txn_data[key] = self.attributeStore.get(txn_data[key])
return txn
def put_into_seq_no_db(txn):
# If there is no reqId, then it's genesis txn
if get_req_id(txn) is None:
return
txn_new = copy.deepcopy(txn)
operation = get_payload_data(txn_new)
operation[TXN_TYPE] = get_type(txn_new)
dct = {
f.IDENTIFIER.nm: get_from(txn_new),
f.REQ_ID.nm: get_req_id(txn_new),
OPERATION: operation,
}
if get_protocol_version(txn_new) is not None:
dct[f.PROTOCOL_VERSION.nm] = get_protocol_version(txn_new)
digest = sha256(serialize_msg_for_signing(dct)).hexdigest()
seq_no = get_seq_no(txn_new)
ledger_id = get_ledger_id_by_txn_type(operation[TXN_TYPE])
line_to_record = str(ledger_id) + ReqIdrToTxn.delimiter + str(seq_no)
dest_seq_no_db_storage.put(digest, line_to_record)
return digest
def _parse_pool_transaction_file(
ledger, nodeReg, cliNodeReg, nodeKeys, activeValidators,
ledger_size=None):
"""
helper function for parseLedgerForHaAndKeys
"""
for _, txn in ledger.getAllTxn(to=ledger_size):
if get_type(txn) == NODE:
txn_data = get_payload_data(txn)
nodeName = txn_data[DATA][ALIAS]
clientStackName = nodeName + CLIENT_STACK_SUFFIX
nHa = (txn_data[DATA][NODE_IP], txn_data[DATA][NODE_PORT]) \
if (NODE_IP in txn_data[DATA] and NODE_PORT in txn_data[DATA]) \
else None
cHa = (txn_data[DATA][CLIENT_IP], txn_data[DATA][CLIENT_PORT]) \
if (CLIENT_IP in txn_data[DATA] and CLIENT_PORT in txn_data[DATA]) \
else None
if nHa:
nodeReg[nodeName] = HA(*nHa)
if cHa:
cliNodeReg[clientStackName] = HA(*cHa)
try:
# TODO: Need to handle abbreviated verkey
key_type = 'verkey'
verkey = cryptonymToHex(str(txn_data[TARGET_NYM]))
key_type = 'identifier'
cryptonymToHex(get_from(txn))
except ValueError:
logger.exception(
'Invalid {}. Rebuild pool transactions.'.format(key_type))
exit('Invalid {}. Rebuild pool transactions.'.format(key_type))
nodeKeys[nodeName] = verkey
services = txn_data[DATA].get(SERVICES)
if isinstance(services, list):
if VALIDATOR in services:
activeValidators.add(nodeName)
else:
activeValidators.discard(nodeName)
def _sendIncorrectTxns(self, req, frm):
ledgerId = getattr(req, f.LEDGER_ID.nm)
if ledgerId == DOMAIN_LEDGER_ID:
logger.info("{} being malicious and sending incorrect transactions"
" for catchup request {} from {}".
format(self, req, frm))
start, end = getattr(req, f.SEQ_NO_START.nm), \
getattr(req, f.SEQ_NO_END.nm)
ledger = self.getLedgerForMsg(req)
txns = {}
for seqNo, txn in ledger.getAllTxn(start, end):
# Since the type of random request is `buy`
if get_type(txn) == "buy":
set_type(txn, "randombuy")
txns[seqNo] = txn
consProof = [Ledger.hashToStr(p) for p in
ledger.tree.consistency_proof(end, ledger.size)]
self.sendTo(msg=CatchupRep(getattr(req, f.LEDGER_ID.nm), txns,
consProof), to=frm)
else:
self.processCatchupReq(req, frm)
def executeAndCheckGenTxn(cli, cmd, typ, nym, role=None, data=None):
checkCmdValid(cli, cmd)
nymCorrect = False
roleCorrect = False if role else True
dataCorrect = False if data else True
typeCorrect = False if typ else True
role = Roles[role].value if role else role
for txn in cli.genesisTransactions:
txn_data = get_payload_data(txn)
if txn_data.get(TARGET_NYM) == nym:
nymCorrect = True
if get_type(txn) == typ:
typeCorrect = True
if txn_data.get(ROLE) == role:
roleCorrect = True
if data and txn_data.get(DATA) == json.loads(data):
dataCorrect = True
assert typeCorrect and nymCorrect and roleCorrect and dataCorrect
assert "Genesis transaction added" in cli.lastCmdOutput
def test_pool_genesis_txns(bootstrap, pool_genesis_file):
serializer = JsonSerializer()
with open(pool_genesis_file) as f:
for line in store_utils.cleanLines(f.readlines()):
txn = serializer.deserialize(line)
assert get_seq_no(txn)
assert get_txn_id(txn)
assert get_payload_data(txn)
assert get_type(txn) == NODE
assert get_version(txn) == "1"
assert get_protocol_version(txn) is None
assert get_payload_data(txn)[TARGET_NYM]
data = get_payload_data(txn).get(DATA)
assert data
assert data[ALIAS]
assert data[CLIENT_IP]
assert data[CLIENT_PORT]
assert data[NODE_IP]
assert data[NODE_PORT]
assert data[SERVICES] == [VALIDATOR]
assert data[BLS_KEY]
assert data[BLS_KEY_PROOF]
def onPoolMembershipChange(self, txn):
# `onPoolMembershipChange` method can be called only after txn added to ledger
if get_type(txn) != NODE:
return
txn_data = get_payload_data(txn)
if DATA not in txn_data:
return
nodeName = txn_data[DATA][ALIAS]
nodeNym = txn_data[TARGET_NYM]
self._set_node_ids_in_cash(nodeNym, nodeName)
def _updateNode(txn_data):
if SERVICES in txn_data[DATA]:
self.nodeServicesChanged(txn_data)
if txn_data[DATA][ALIAS] in self.node.nodeReg:
if {NODE_IP, NODE_PORT, CLIENT_IP, CLIENT_PORT}. \
intersection(set(txn_data[DATA].keys())):
self.nodeHaChanged(txn_data)
if VERKEY in txn_data:
self.nodeKeysChanged(txn_data)
if BLS_KEY in txn_data[DATA]:
self.node_blskey_changed(txn_data)
# If nodeNym is never added in self._ordered_node_services,
# nodeNym is never added in ledger
if nodeNym not in self._ordered_node_services:
txn_data[DATA].setdefault(SERVICES, [])
if VALIDATOR in txn_data[DATA].get(SERVICES, []):
self.addNewNodeAndConnect(txn_data)
else:
_updateNode(txn_data)
self._set_node_services_in_cash(nodeNym, txn_data[DATA].get(SERVICES, None))
def _addClaimDef(self, txn, isCommitted=False) -> None:
assert get_type(txn) == CLAIM_DEF
path, value_bytes = domain.prepare_claim_def_for_state(txn)
self.state.set(path, value_bytes)
def prepare_for_state(result):
if get_type(result) == "buy":
key, value = GetBuyHandler.prepare_buy_for_state(result)
return key, value
def get_last_node_upgrade_txn(self, start_no: int = None):
return self.get_upgrade_txn(lambda txn: get_type(txn) == NODE_UPGRADE
and get_from(txn) == self.nodeId,
start_no=start_no,
reverse=True)
def filter_fees(ledger_id: int, txn: Any):
origin_token_post_added_clb(ledger_id, txn, get_type(txn) != FeesTransactions.FEES.value)
def handleUpgradeTxn(self, txn) -> None:
"""
Handles transaction of type POOL_UPGRADE
Can schedule or cancel upgrade to a newer
version at specified time
:param txn:
"""
FINALIZING_EVENT_TYPES = [
UpgradeLog.Events.succeeded, UpgradeLog.Events.failed
]
if get_type(txn) != POOL_UPGRADE:
return
logger.info("Node '{}' handles upgrade txn {}".format(
self.nodeName, txn))
txn_data = get_payload_data(txn)
action = txn_data[ACTION]
version = txn_data[VERSION]
justification = txn_data.get(JUSTIFICATION)
pkg_name = txn_data.get(PACKAGE, self.config.UPGRADE_ENTRY)
upgrade_id = self.get_action_id(txn)
# TODO test
try:
version = src_version_cls(pkg_name)(version)
except InvalidVersionError as exc:
logger.warning(
"{} can't handle upgrade txn with version {} for package {}: {}"
.format(self, version, pkg_name, exc))
return
if action == START:
# forced txn could have partial schedule list
if self.nodeId not in txn_data[SCHEDULE]:
logger.info("Node '{}' disregards upgrade txn {}".format(
self.nodeName, txn))
return
last_event = self.lastActionEventInfo
if last_event:
if (last_event.data.upgrade_id == upgrade_id
and last_event.ev_type in FINALIZING_EVENT_TYPES):
logger.info(
"Node '{}' has already performed an upgrade with upgrade_id {}. "
"Last recorded event is {}".format(
self.nodeName, upgrade_id, last_event.data))
return
when = txn_data[SCHEDULE][self.nodeId]
failTimeout = txn_data.get(TIMEOUT, self.defaultActionTimeout)
if isinstance(when, str):
when = dateutil.parser.parse(when)
new_ev_data = UpgradeLogData(when, version, upgrade_id, pkg_name)
if self.scheduledAction:
if self.scheduledAction == new_ev_data:
logger.debug(
"Node {} already scheduled upgrade to version '{}' ".
format(self.nodeName, version))
return
else:
logger.info(
"Node '{}' cancels previous upgrade and schedules a new one to {}"
.format(self.nodeName, version))
self._cancelScheduledUpgrade(justification)
logger.info("Node '{}' schedules upgrade to {}".format(
self.nodeName, version))
self._scheduleUpgrade(new_ev_data, failTimeout)
return
if action == CANCEL:
if (self.scheduledAction
and self.scheduledAction.version == version):
self._cancelScheduledUpgrade(justification)
logger.info("Node '{}' cancels upgrade to {}".format(
self.nodeName, version))
return
logger.error("Got {} transaction with unsupported action {}".format(
POOL_UPGRADE, action))
def pool_node_txns(poolTxnData):
node_txns = []
for txn in poolTxnData["txns"]:
if get_type(txn) == NODE:
node_txns.append(txn)
return node_txns
def _updateStateWithSingleTxn(self, txn, isCommitted=False):
typ = get_type(txn)
if typ == WRITE_CONF:
conf = json.loads(get_payload_data(txn)[DATA])
key, val = conf.popitem()
self.state.set(key.encode(), val.encode())
def test_get_type(txn):
assert get_type(txn) == NYM
def _addSchema(self, txn, isCommitted=False) -> None:
assert get_type(txn) == SCHEMA
path, value_bytes = domain.prepare_schema_for_state(txn)
self.state.set(path, value_bytes)
def test_set_type(txn):
txn = set_type(txn, NODE)
assert get_type(txn) == NODE
def _addRevocDef(self, txn, isCommitted=False) -> None:
assert get_type(txn) == REVOC_REG_DEF
path, value_bytes = domain.prepare_revoc_def_for_state(txn)
self.state.set(path, value_bytes)
def _load_nodes_order_from_ledger(self):
self._ordered_node_ids = OrderedDict()
for _, txn in self.ledger.getAllTxn():
if get_type(txn) == NODE:
txn_data = get_payload_data(txn)
self._set_node_order(txn_data[TARGET_NYM], txn_data[DATA][ALIAS])
def _validate_txn_type(self, txn: Dict):
if get_type(txn) != self.txn_type:
raise LogicError
def hasNym(self, nym):
for txn in self.txnLog.getTxnsByType(NYM):
if get_type(txn) == NYM:
return True
return False
def prepare_for_state(self, result):
if get_type(result) == "buy":
from plenum.test.test_node import TestDomainRequestHandler
key, value = TestDomainRequestHandler.prepare_buy_for_state(result)
return key, value
def transform_txn_for_ledger(self, txn):
handlers = self.request_handlers.get(get_type(txn), None)
if handlers is None:
raise LogicError
return handlers[0].transform_txn_for_ledger(txn)
