def _check_consistency_after_commit(self, txn_root, state_root=None):
if self.ledger.root_hash != txn_root:
raise PlenumValueError(
'txnRoot', txn_root,
("equal to current ledger root hash {}".format(
self.ledger.root_hash)))
if state_root is not None and self.state is not None:
if self.state.committedHeadHash != state_root:
raise PlenumValueError(
'stateRoot', state_root,
("equal to current state root hash {}".format(
self.state.committedHeadHash)))
def apply_data(self, msg, sender):
if msg.msg_type != BATCH:
raise PlenumValueError('msg.msg_type', msg.msg_type, BATCH)
logger.debug("{} got BATCH {} from Observable Node {}".format(
OBSERVER_PREFIX, msg, sender))
# 1. make sure that the message is dict
batch = msg.msg
if not isinstance(batch, dict):
batch = self._node.toDict(batch)
# 2. check whether we can process the message (that it's not already processed and is the next expected seq_no)
if not self._can_process(batch):
return
# 3. stash the batch
self._add_batch(batch, sender)
# 4. check that the batch can be applied
# - this is the next expected batch (according to seq_nos)
# - either state proof is valid, or a consensus of f+1 equal batches from different nodes
if not self._can_apply(batch):
return
# 5. apply the batch (write to ledger and state)
self._do_apply_data(batch)
# 6. check whether we can apply stashed batches with next seq_no
self._process_stashed_messages()
def commit(self, txnCount, stateRoot, txnRoot, ppTime) -> List:
"""
:param txnCount: The number of requests to commit (The actual requests
are picked up from the uncommitted list from the ledger)
:param stateRoot: The state trie root after the txns are committed
:param txnRoot: The txn merkle root after the txns are committed
:return: list of committed transactions
"""
(seqNoStart, seqNoEnd), committedTxns = \
self.ledger.commitTxns(txnCount)
stateRoot = base58.b58decode(stateRoot.encode())
# TODO test for that
if self.ledger.root_hash != txnRoot:
# Probably the following fail should trigger catchup
# TODO add repr / str for Ledger class and dump it here as well
raise PlenumValueError(
'txnRoot', txnRoot,
("equal to current ledger root hash {}".format(
self.ledger.root_hash)))
self.state.commit(rootHash=stateRoot)
if self.ts_store:
self.ts_store.set(ppTime, stateRoot)
return committedTxns
def reconnectRemoteWithName(self, remoteName):
if remoteName not in self.remotes:
raise PlenumValueError(
'remoteName', remoteName,
"one of {}".format(self.remotes)
)
self.reconnectRemote(self.remotes[remoteName])
def __init__(self, config, *, chroot=None):
if config is None:
raise ValueUndefinedError('config')
if chroot is not None and not chroot.startswith("/"):
raise PlenumValueError('chroot', chroot, "starts with '/'")
self.config = config
self.chroot = chroot
def __init__(self, max_size=None, **kwargs):
if max_size is not None:
if not isinstance(max_size, int):
raise PlenumTypeError('max_size', max_size, int)
if not max_size > 0:
raise PlenumValueError('max_size', max_size, '> 0')
self.max_size = max_size
super().__init__(**kwargs)
def apply_batch(self, state_root, ledger_size):
"""
:param state_root: uncommitted state root
:param ledger_size: ledger size, after committing (future ledger size)
:return:
"""
if not state_root:
raise PlenumValueError('state_root', state_root,
"No state root given")
if ledger_size < 0:
raise PlenumValueError('ledger_size', ledger_size,
"Incorrect size of ledger")
self.un_committed.append((state_root, ledger_size))
def randomStr(size):
if not isinstance(size, int):
raise PlenumTypeError('size', size, int)
if not size > 0:
raise PlenumValueError('size', size, '> 0')
# Approach 1
rv = randombytes(size // 2).hex()
return rv if size % 2 == 0 else rv + hex(randombytes_uniform(15))[-1]
def __init__(self,
max_length=GENERAL_LIMIT_SIZE,
can_be_empty=False,
**kwargs):
if not max_length > 0:
raise PlenumValueError('max_length', max_length, '> 0')
super().__init__(**kwargs)
self._max_length = max_length
self._can_be_empty = can_be_empty
def merkleInfo(self, seqNo):
seqNo = int(seqNo)
if seqNo <= 0:
raise PlenumValueError('seqNo', seqNo, '> 0')
rootHash = self.tree.merkle_tree_hash(0, seqNo)
auditPath = self.tree.inclusion_proof(seqNo - 1, seqNo)
return {
F.rootHash.name: self.hashToStr(rootHash),
F.auditPath.name: [self.hashToStr(h) for h in auditPath]
}
def disconnectByName(self, remoteName: str):
if not remoteName:
raise PlenumValueError('remoteName', remoteName,
"non-empty string")
remote = self.remotes.get(remoteName)
if not remote:
logger.debug('{} did not find any remote '
'by name {} to disconnect'.format(self, remoteName))
return None
remote.disconnect()
return remote
def addRemote(self, name, ha, remoteVerkey, remotePublicKey):
if not name:
raise PlenumValueError('name', name, 'non-empty')
remote = self._RemoteClass(name, ha, remoteVerkey, remotePublicKey, self.queue_size, self.ha[0])
self.remotes[name] = remote
# TODO: Use weakref to remote below instead
self.remotesByKeys[remotePublicKey] = remote
if remoteVerkey:
self.addVerifier(remoteVerkey)
else:
logger.display('{} adding a remote {}({}) without a verkey'.format(self, name, ha))
return remote
def test_PlenumValueError():
aaa = 1
expected = "= 0"
exc = PlenumValueError('aaa', aaa, expected, prefix="PREFIX")
assert isinstance(exc, PlenumError)
assert isinstance(exc, ValueError)
exc_str = str(exc)
assert exc_str.startswith("PREFIX: ")
assert "variable 'aaa'" in exc_str
assert "value {}".format(aaa) in exc_str
assert "expected: {}".format(expected) in exc_str
def on_future_view_vchd_msg(self, view_no, frm, from_current_state: bool = False):
if not ((view_no > self.view_no) or
(self.view_no == 0 and from_current_state)):
raise PlenumValueError(
'view_no', view_no,
("= 0 or > {}" if from_current_state
else "> {}").format(self.view_no),
prefix=self
)
if view_no not in self._next_view_indications:
self._next_view_indications[view_no] = set()
self._next_view_indications[view_no].add(frm)
self._start_view_change_if_possible(view_no)
def cleanSeed(seed=None):
if seed:
bts = seedFromHex(seed)
if not bts:
if isinstance(seed, str):
seed = seed.encode('utf-8')
bts = bytes(seed)
if len(seed) != 32:
raise PlenumValueError(
'seed', seed,
'64-length hexadecimal string or 32-length in bytes representation'
)
return bts
def _commit(ledger, state, txn_count, state_root, txn_root):
_, committedTxns = ledger.commitTxns(txn_count)
state_root = state_roots_serializer.deserialize(
state_root.encode()) if isinstance(state_root, str) else state_root
# TODO test for that
if ledger.root_hash != txn_root:
# Probably the following fail should trigger catchup
# TODO add repr / str for Ledger class and dump it here as well
raise PlenumValueError(
'txnRoot', txn_root,
("equal to current ledger root hash {}".format(
ledger.root_hash)))
state.commit(rootHash=state_root)
return committedTxns
def commit_batch(self):
"""
:param state_root: committed state root
:param ledger_size: committed ledger size
:return: tuple of next committed state and count of committed transactions
"""
if len(self.un_committed) == 0:
raise PlenumValueError(
"un_committed", self.un_committed,
"commit_batch was called, but there is no tracked uncommitted states"
)
uncommitted_hash, uncommitted_size = self.un_committed.popleft()
self.last_committed = (uncommitted_hash, uncommitted_size)
def set_root_hash(self, root_hash):
if not is_string(root_hash):
raise PlenumTypeError('root_hash', root_hash, bytes)
if not (len(root_hash) in [0, 32]):
raise PlenumValueError('root_hash', root_hash,
'length in range [0, 32]')
if self.transient:
self.transient_root_hash = root_hash
return
if root_hash == BLANK_ROOT:
self.root_node = BLANK_NODE
return
# print(repr(root_hash))
self.root_node = self._decode_to_node(root_hash)
def __init__(self, name, encoder, decoder):
if not isinstance(name, str):
raise PlenumTypeError('name', name, str)
if not name:
raise PlenumValueError('name', name, 'a non-empty string')
if not callable(encoder):
raise PlenumTypeError('encoder', encoder, 'callable')
if not callable(decoder):
raise PlenumTypeError('decoder', decoder, 'callable')
self.name = name
self.encoder = encoder
self.decoder = decoder
def __init__(self, windowSize, delayFunction=None):
"""
Limits rate of actions performed in a unit of time (window)
:param windowSize: size (in seconds) of the time window events counted in
:param delayFunction: function from **number of actions** to **time to wait after the last one**
"""
if not isinstance(windowSize, int):
raise PlenumTypeError('windowSize', windowSize, int)
if not windowSize > 0:
raise PlenumValueError('windowSize', windowSize, '> 0')
self.windowSize = windowSize
self.delayFunction = delayFunction if delayFunction else self._defaultDelayFunction
self.actionsLog = []
def __init__(self, inner_field_type: FieldValidator, min_length=None,
max_length=None, **kwargs):
if not isinstance(inner_field_type, FieldValidator):
raise PlenumTypeError(
'inner_field_type', inner_field_type, FieldValidator)
for k in ('min_length', 'max_length'):
m = locals()[k]
if m is not None:
if not isinstance(m, int):
raise PlenumTypeError(k, m, int)
if not m > 0:
raise PlenumValueError(k, m, '> 0')
self.inner_field_type = inner_field_type
self.min_length = min_length
self.max_length = max_length
super().__init__(**kwargs)
def appendTxns(self, txns: List):
# These transactions are not yet committed so they do not go to
# the ledger
_no_seq_no_txns = [txn for txn in txns if get_seq_no(txn) is None]
if _no_seq_no_txns:
raise PlenumValueError(
'txns', txns,
("all txns should have defined seq_no, undefined in {}".format(
_no_seq_no_txns)))
uncommittedSize = self.size + len(self.uncommittedTxns)
self.uncommittedTree = self.treeWithAppliedTxns(
txns, self.uncommittedTree)
self.uncommittedRootHash = self.uncommittedTree.root_hash
self.uncommittedTxns.extend(txns)
if txns:
return (uncommittedSize + 1, uncommittedSize + len(txns)), txns
else:
return (uncommittedSize, uncommittedSize), txns
def bootstrapTestNodes(cls,
config,
startingPort,
nodeParamsFileName,
domainTxnFieldOrder,
config_helper_class=PConfigHelper,
node_config_helper_class=PNodeConfigHelper,
chroot: str = None):
parser = argparse.ArgumentParser(
description="Generate pool transactions for testing")
parser.add_argument('--nodes',
required=True,
help='node count should be less than 100',
type=cls._bootstrapArgsTypeNodeCount)
parser.add_argument('--clients',
required=True,
type=int,
help='client count')
parser.add_argument('--nodeNum',
type=int,
nargs='+',
help='the number of the node that will '
'run on this machine')
parser.add_argument('--ips',
help='IPs/hostnames of the nodes, provide comma '
'separated IPs, if no of IPs provided are less'
' than number of nodes then the remaining '
'nodes are assigned the loopback IP, '
'i.e 127.0.0.1',
type=cls._bootstrap_args_type_ips_hosts)
parser.add_argument('--network',
help='Network name (default sandbox)',
type=str,
default="sandbox",
required=False)
parser.add_argument(
'--appendToLedgers',
help="Determine if ledger files needs to be erased "
"before writing new information or not.",
action='store_true')
args = parser.parse_args()
if isinstance(args.nodeNum, int):
if not (1 <= args.nodeNum <= args.nodes):
raise PlenumValueError(
'args.nodeNum', args.nodeNum,
">= 1 && <= args.nodes {}".format(args.nodes))
elif isinstance(args.nodeNum, list):
if any([True for x in args.nodeNum if not (1 <= x <= args.nodes)]):
raise PlenumValueError(
'some items in nodeNum list', args.nodeNum,
">= 1 && <= args.nodes {}".format(args.nodes))
node_num = [args.nodeNum, None] if args.nodeNum else [None]
steward_defs, node_defs = cls.gen_defs(args.ips, args.nodes,
startingPort)
client_defs = cls.gen_client_defs(args.clients)
trustee_def = cls.gen_trustee_def(1)
if args.nodeNum:
# update network during node generation only
# edit NETWORK_NAME in config
for line in fileinput.input(['/etc/indy/indy_config.py'],
inplace=True):
if 'NETWORK_NAME' not in line:
print(line, end="")
with open('/etc/indy/indy_config.py', 'a') as cfgfile:
cfgfile.write("NETWORK_NAME = '{}'".format(args.network))
for n_num in node_num:
cls.bootstrapTestNodesCore(
config, args.network, args.appendToLedgers,
domainTxnFieldOrder, trustee_def, steward_defs, node_defs,
client_defs, n_num, nodeParamsFileName, config_helper_class,
node_config_helper_class)
# delete unnecessary key dir in client folder
key_dir = cls.setup_clibase_dir(config, args.network)
key_dir = os.path.join(key_dir, "keys")
if os.path.isdir(key_dir):
shutil.rmtree(key_dir, ignore_errors=True)
def validate_value(seq_no):
if seq_no <= 0:
# Also, this field will be caught on static validation phase and Reply with this message will be sent to client
raise PlenumValueError("seq_no should be number and more then 0")
def __init__(self, max_length: int, **kwargs):
if not max_length > 0:
raise PlenumValueError('max_length', max_length, '> 0')
super().__init__(**kwargs)
self._max_length = max_length
async def eventually(coroFunc: FlexFunc,
*args,
retryWait: float = 0.1,
timeout: Optional[float] = None,
ratchetSteps: Optional[int] = None,
acceptableExceptions=None,
verbose=True,
override_timeout_limit=False) -> T:
if timeout is None:
timeout = 5
if not timeout > 0:
raise PlenumValueError('timeout', timeout, '> 0')
if not override_timeout_limit:
if timeout > 240:
raise PlenumValueError('timeout', timeout,
"< 240 or override_timeout_limit=True")
else:
logger.debug('Overriding timeout limit to {} for evaluating {}'.format(
timeout, coroFunc))
if acceptableExceptions and not isinstance(acceptableExceptions, Iterable):
acceptableExceptions = [acceptableExceptions]
start = time.perf_counter()
ratchet = Ratchet.fromGoalDuration(retryWait * slowFactor,
ratchetSteps,
timeout * slowFactor).gen() \
if ratchetSteps else None
fname = get_func_name(coroFunc)
while True:
remain = 0
try:
remain = start + timeout * slowFactor - time.perf_counter()
if remain < 0:
# this provides a convenient breakpoint for a debugger
logger.debug("{} last try...".format(fname),
extra={"cli": False})
# noinspection PyCallingNonCallable
res = coroFunc(*args)
if isawaitable(res):
result = await res
else:
result = res
if verbose:
recordSuccess(fname, timeout, timeout * slowFactor, remain)
logger.debug(
"{} succeeded with {:.2f} seconds to spare".format(
fname, remain))
return result
except Exception as ex:
if acceptableExceptions and type(ex) not in acceptableExceptions:
raise
if remain >= 0:
sleep_dur = next(ratchet) if ratchet else retryWait
if verbose:
logger.trace("{} not succeeded yet, {:.2f} seconds "
"remaining..., will sleep for {}".format(
fname, remain, sleep_dur))
await asyncio.sleep(sleep_dur)
else:
recordFail(fname, timeout)
logger.error("{} failed; not trying any more because {} "
"seconds have passed; args were {}".format(
fname, timeout, args))
raise ex
def bootstrap_pool_ledger(cls,
config,
nodeParamsFileName,
config_helper_class=PConfigHelper,
node_config_helper_class=PNodeConfigHelper,
chroot: str = None):
parser = argparse.ArgumentParser(
description="Generate pool transactions")
parser.add_argument(
'--nodeVerkeys',
required=True,
help='Node Verkey, provide comma separated verification keys',
type=cls._bootstrap_args_type_verkeys)
parser.add_argument(
'--nodeBlskeys',
required=True,
help='Node BLS keys, provide comma separated Bls keys',
type=cls._bootstrap_args_type_bls)
parser.add_argument(
'--nodeBlsProofs',
required=True,
help=
'Node Proof of possession for BLS key, provide comma separated proof',
type=cls._bootstrap_args_type_bls)
parser.add_argument('--nodeName',
required=True,
help='Node name, provide comma separated name',
type=cls._bootstrap_args_type_list)
parser.add_argument('--nodePort',
required=True,
help='Node port, provide comma separated port',
type=cls._bootstrap_args_type_port)
parser.add_argument('--clientPort',
required=True,
help='Client port, provide comma separated port',
type=cls._bootstrap_args_type_port)
parser.add_argument('--stewardDids',
required=True,
help='Stewards Dids, provide comma separated dids',
type=cls._bootstrap_args_type_dids)
parser.add_argument('--nodeNum',
type=int,
nargs='+',
help='the number of the node that will '
'run on this machine')
parser.add_argument(
'--ips',
help='IPs/hostnames of the nodes, provide comma '
'separated IPs, if no of IPS provided are less than number of nodes then the remaining'
'nodes are assigned the loopback IP, '
'i.e 127.0.0.1',
type=cls._bootstrap_args_type_ips_hosts)
parser.add_argument('--network',
required=False,
help='Network name (default sandbox)',
type=str,
default="sandbox")
parser.add_argument(
'--appendToLedgers',
help="Determine if ledger files needs to be erased "
"before writing new information or not",
action='store_true')
args = parser.parse_args()
if ((len(args.nodeName) != len(args.nodeVerkeys))
or (len(args.nodeBlskeys) != len(args.nodeVerkeys))
or (len(args.nodeBlskeys) != len(args.nodeBlsProofs))
or (len(args.nodePort) != len(args.nodeBlsProofs))
or (len(args.nodePort) != len(args.clientPort))
or (len(args.stewardDids) != len(args.clientPort))):
raise argparse.ArgumentTypeError("Some arguments are missing.")
if isinstance(args.nodeNum, int):
if not (1 <= args.nodeNum <= len(args.nodeName)):
raise PlenumValueError(
'args.nodeNum', args.nodeNum,
">= 1 && <= len(args.nodeName) {}".format(
len(args.nodeName)))
elif isinstance(args.nodeNum, list):
if any([
True for x in args.nodeNum
if not (1 <= x <= len(args.nodeName))
]):
raise PlenumValueError(
'some items in nodeNum list',
args.nodeNum, ">=1 && <= len(args.nodeName) {}".format(
len(args.nodeName)))
node_num = [args.nodeNum, None] if args.nodeNum else [None]
node_defs = cls.gen_node_def(args.nodeName, args.ips, args.nodePort,
args.clientPort, args.nodeBlskeys,
args.nodeBlsProofs, args.nodeVerkeys,
args.stewardDids)
if args.nodeNum:
for line in fileinput.input(['/etc/indy/indy_config.py'],
inplace=True):
if 'NETWORK_NAME' not in line:
print(line, end="")
with open('/etc/indy/indy_config.py', 'a') as cfgfile:
cfgfile.write("NETWORK_NAME = '{}'".format(args.network))
for n_num in node_num:
cls.bootstrap_pool_ledger_core(config, args.network,
args.appendToLedgers, node_defs,
n_num, nodeParamsFileName,
config_helper_class,
node_config_helper_class)
