def test_node_load_after_disconnect(looper, txnPoolNodeSet, tconf,
allPluginsPath,
tdirWithPoolTxns,
sdk_pool_handle,
sdk_wallet_client,
capsys):
nodes = txnPoolNodeSet
x = nodes[-1]
with capsys.disabled():
print("Stopping node {} with pool ledger size {}".
format(x, x.poolManager.txnSeqNo))
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, x)
looper.removeProdable(x)
client_batches = 80
txns_per_batch = 10
for i in range(client_batches):
s = perf_counter()
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, txns_per_batch)
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.
format(i + 1, txns_per_batch, perf_counter() - s))
nodeHa, nodeCHa = HA(*x.nodestack.ha), HA(*x.clientstack.ha)
newNode = TestNode(x.name, basedirpath=tdirWithPoolTxns, base_data_dir=tdirWithPoolTxns, config=tconf,
ha=nodeHa, cliha=nodeCHa, pluginPaths=allPluginsPath)
looper.add(newNode)
txnPoolNodeSet[-1] = newNode
looper.run(checkNodesConnected(txnPoolNodeSet))
def testCreateAgentDoesNotAllocatePort(tdirWithPoolTxns):
for i in range(2):
checkPortAvailable(HA("0.0.0.0", agentPort))
agent = getNewAgent("Agent0", tdirWithPoolTxns, agentPort,
agentWallet())
checkPortAvailable(HA("0.0.0.0", agentPort))
agent.stop()
def onPoolMembershipChange(self, txn):
if txn[TXN_TYPE] == NODE:
nodeName = txn[DATA][ALIAS]
nodeNym = txn[TARGET_NYM]
def _updateNode(txn):
if {NODE_IP, NODE_PORT, CLIENT_IP, CLIENT_PORT}. \
intersection(set(txn[DATA].keys())):
self.nodeHaChanged(txn)
if VERKEY in txn:
self.nodeKeysChanged(txn)
if SERVICES in txn[DATA]:
self.nodeServicesChanged(txn)
if nodeName in self.nodeReg:
# The node was already part of the pool so update
_updateNode(txn)
else:
seqNos, info = self.getNodeInfoFromLedger(nodeNym)
if len(seqNos) == 1:
# Since only one transaction has been made, this is a new
# node transaction
self.addNewNodeAndConnect(txn)
else:
self.node.nodeReg[nodeName] = HA(info[DATA][NODE_IP],
info[DATA][NODE_PORT])
self.node.cliNodeReg[nodeName] = HA(
info[DATA][CLIENT_IP], info[DATA][CLIENT_PORT])
_updateNode(txn)
self.node.sendPoolInfoToClients(txn)
if self.config.UpdateGenesisPoolTxnFile:
updateGenesisPoolTxnFile(self.config.baseDir,
self.config.poolTransactionsFile, txn)
def connectNewRemote(self,
txn,
remoteName,
nodeOrClientObj,
addRemote=True):
verkey = cryptonymToHex(txn[TARGET_NYM])
nodeHa = (txn[DATA][NODE_IP], txn[DATA][NODE_PORT])
cliHa = (txn[DATA][CLIENT_IP], txn[DATA][CLIENT_PORT])
if addRemote:
try:
# Override any keys found, reason being the scenario where
# before this node comes to know about the other node, the other
# node tries to connect to it.
initRemoteKeep(self.name,
remoteName,
self.basedirpath,
verkey,
override=True)
except Exception as ex:
logger.error(
"Exception while initializing keep for remote {}".format(
ex))
if self.isNode:
nodeOrClientObj.nodeReg[remoteName] = HA(*nodeHa)
nodeOrClientObj.cliNodeReg[remoteName +
CLIENT_STACK_SUFFIX] = HA(*cliHa)
logger.debug("{} adding new node {} with HA {}".format(
self.name, remoteName, nodeHa))
else:
nodeOrClientObj.nodeReg[remoteName] = HA(*cliHa)
logger.debug("{} adding new node {} with HA {}".format(
self.name, remoteName, cliHa))
def getNodeStackParams(name,
nodeRegistry: Dict[str, HA],
ha: HA = None,
basedirpath: str = None) -> Tuple[dict, dict, dict]:
"""
Return tuple(nodeStack params, nodeReg)
"""
me = nodeRegistry[name]
if isinstance(me, NodeDetail):
sha = me.ha
nodeReg = {k: v.ha for k, v in nodeRegistry.items()}
else:
sha = me if isinstance(me, HA) else HA(*me[0])
nodeReg = {
k: v if isinstance(v, HA) else HA(*v[0])
for k, v in nodeRegistry.items()
}
if not ha: # pull it from the registry
ha = sha
cliNodeReg = {r.cliname: r.cliha for r in nodeRegistry.values()}
nstack = dict(name=name, ha=ha, main=True, auto=AutoMode.never)
if basedirpath:
nstack['basedirpath'] = basedirpath
return nstack, nodeReg, cliNodeReg
def getClientStackParams(name, nodeRegistry: Dict[str, HA], cliname, cliha,
basedirpath) -> dict:
"""
Return clientStack params
"""
me = nodeRegistry[name]
if isinstance(me, NodeDetail):
sha = me.ha
scliname = me.cliname
scliha = me.cliha
else:
sha = me if isinstance(me, HA) else HA(*me[0])
scliname = None
scliha = None
if not cliname: # default to the name plus the suffix
cliname = scliname if scliname else name + CLIENT_STACK_SUFFIX
if not cliha: # default to same ip, port + 1
cliha = scliha if scliha else HA(sha[0], sha[1] + 1)
cstack = dict(name=cliname, ha=cliha, main=True, auto=AutoMode.always)
if basedirpath:
cstack['basedirpath'] = basedirpath
return cstack
def getStackParamsAndNodeReg(self,
name,
basedirpath,
nodeRegistry=None,
ha=None,
cliname=None,
cliha=None):
nodeReg, cliNodeReg, nodeKeys = self.parseLedgerForHaAndKeys()
self.addRemoteKeysFromLedger(nodeKeys)
# If node name was not found in the pool transactions file
if not ha:
ha = nodeReg[name]
nstack = dict(name=name,
ha=HA('0.0.0.0', ha[1]),
main=True,
auto=AutoMode.never)
nodeReg[name] = HA(*ha)
cliname = cliname or (name + CLIENT_STACK_SUFFIX)
if not cliha:
cliha = cliNodeReg[cliname]
cstack = dict(name=cliname or (name + CLIENT_STACK_SUFFIX),
ha=HA('0.0.0.0', cliha[1]),
main=True,
auto=AutoMode.always)
cliNodeReg[cliname] = HA(*cliha)
if basedirpath:
nstack['basedirpath'] = basedirpath
cstack['basedirpath'] = basedirpath
return nstack, cstack, nodeReg, cliNodeReg
def test_node_load_after_add_then_disconnect(sdk_new_node_caught_up, txnPoolNodeSet,
tconf, looper, sdk_pool_handle,
sdk_wallet_client,
tdirWithPoolTxns, allPluginsPath,
capsys):
"""
A node that restarts after some transactions should eventually get the
transactions which happened while it was down
:return:
"""
new_node = sdk_new_node_caught_up
with capsys.disabled():
print("Stopping node {} with pool ledger size {}".
format(new_node, new_node.poolManager.txnSeqNo))
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, new_node)
looper.removeProdable(new_node)
client_batches = 80
txns_per_batch = 10
for i in range(client_batches):
s = perf_counter()
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, txns_per_batch)
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.
format(i + 1, txns_per_batch, perf_counter() - s))
with capsys.disabled():
print("Starting the stopped node, {}".format(new_node))
nodeHa, nodeCHa = HA(*new_node.nodestack.ha), HA(*new_node.clientstack.ha)
new_node = TestNode(
new_node.name,
basedirpath=tdirWithPoolTxns,
base_data_dir=tdirWithPoolTxns,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
looper.add(new_node)
txnPoolNodeSet[-1] = new_node
# Delay catchup reply processing so LedgerState does not change
delay_catchup_reply = 5
new_node.nodeIbStasher.delay(cr_delay(delay_catchup_reply))
looper.run(checkNodesConnected(txnPoolNodeSet))
# Make sure ledger starts syncing (sufficient consistency proofs received)
looper.run(eventually(check_ledger_state, new_node, DOMAIN_LEDGER_ID,
LedgerState.syncing, retryWait=.5, timeout=5))
# Not accurate timeout but a conservative one
timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) + \
2 * delay_catchup_reply
waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4],
customTimeout=timeout)
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
sdk_wallet_client, 5)
waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4])
def overlapNodePorts(nodeReg):
"""
From the given node registry, make Alpha and Beta run on the same port.
"""
A = nodeReg['Alpha']
betaPort = nodeReg['Beta'].ha.port
nodeReg['Alpha'] = NodeDetail(HA(A.ha.host, betaPort), A.cliname,
HA(A.cliha.host, A.cliha.port))
def stackHaChanged(self, txn, remoteName, nodeOrClientObj):
nodeHa = (txn[DATA][NODE_IP], txn[DATA][NODE_PORT])
cliHa = (txn[DATA][CLIENT_IP], txn[DATA][CLIENT_PORT])
rid = self.removeRemote(nodeOrClientObj.nodestack, remoteName)
if self.isNode:
nodeOrClientObj.nodeReg[remoteName] = HA(*nodeHa)
nodeOrClientObj.cliNodeReg[remoteName +
CLIENT_STACK_SUFFIX] = HA(*cliHa)
else:
nodeOrClientObj.nodeReg[remoteName] = HA(*cliHa)
return rid
def test_node_catchup_after_restart_no_txns(
newNodeCaughtUp,
txnPoolNodeSet,
tdir,
tconf,
nodeSetWithNodeAddedAfterSomeTxns,
tdirWithPoolTxns,
allPluginsPath):
"""
A node restarts but no transactions have happened while it was down.
It would then use the `LedgerStatus` to catchup
"""
looper, new_node, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns
waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1])
logger.debug("Stopping node {} with pool ledger size {}".
format(new_node, new_node.poolManager.txnSeqNo))
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, new_node)
looper.removeProdable(name=new_node.name)
logger.debug("Starting the stopped node, {}".format(new_node))
nodeHa, nodeCHa = HA(*new_node.nodestack.ha), HA(*new_node.clientstack.ha)
config_helper = PNodeConfigHelper(new_node.name, tconf, chroot=tdir)
new_node = TestNode(
new_node.name,
ledger_dir=config_helper.ledger_dir,
keys_dir=config_helper.keys_dir,
genesis_dir=config_helper.genesis_dir,
plugins_dir=config_helper.plugins_dir,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
looper.add(new_node)
txnPoolNodeSet[-1] = new_node
looper.run(checkNodesConnected(txnPoolNodeSet))
def chk():
for node in txnPoolNodeSet[:-1]:
check_last_ordered_3pc(new_node, node)
looper.run(eventually(chk, retryWait=1))
# sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5)
waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1])
# Did not receive any consistency proofs
assert get_count(new_node.ledgerManager,
new_node.ledgerManager.processConsistencyProof) == 0
def __init__(self,
name: str = None,
basedirpath: str = None,
client: Client = None,
port: int = None,
loop=None,
config=None,
endpointArgs=None):
self.endpoint = None
if port:
checkPortAvailable(HA("0.0.0.0", port))
Motor.__init__(self)
self.loop = loop or asyncio.get_event_loop()
self._eventListeners = {} # Dict[str, set(Callable)]
self._name = name or 'Agent'
self._port = port
self.config = config or getConfig()
self.basedirpath = basedirpath or os.path.expanduser(
self.config.baseDir)
self.endpointArgs = endpointArgs
# Client used to connect to Sovrin and forward on owner's txns
self._client = client # type: Client
# known identifiers of this agent's owner
self.ownerIdentifiers = {} # type: Dict[Identifier, Identity]
def __init__(self,
name: str,
nodeReg: Dict[str, HA] = None,
ha: Union[HA, Tuple[str, int]] = None,
peerHA: Union[HA, Tuple[str, int]] = None,
basedirpath: str = None,
config=None,
sighex: str = None):
config = config or getConfig()
super().__init__(name, nodeReg, ha, basedirpath, config, sighex)
self.graphStore = self.getGraphStore()
self.autoDiscloseAttributes = False
self.requestedPendingTxns = False
self.hasAnonCreds = bool(peerHA)
if self.hasAnonCreds:
self.peerHA = peerHA if isinstance(peerHA, HA) else HA(*peerHA)
stackargs = dict(name=self.stackName,
ha=peerHA,
main=True,
auto=AutoMode.always)
self.peerMsgRoutes = []
self.peerMsgRouter = Router(*self.peerMsgRoutes)
self.peerStack = SimpleStack(stackargs,
msgHandler=self.handlePeerMessage)
self.peerStack.sign = self.sign
self.peerInbox = deque()
self._observers = {} # type Dict[str, Callable]
self._observerSet = set(
) # makes it easier to guard against duplicates
def changeHA(looper,
config,
nodeName,
nodeSeed,
newNodeHA,
stewardName,
stewardsSeed,
newClientHA=None):
if not newClientHA:
newClientHA = HA(newNodeHA.host, newNodeHA.port + 1)
# prepare steward wallet
stewardSigner = SimpleSigner(seed=stewardsSeed)
stewardWallet = Wallet(stewardName)
stewardWallet.addIdentifier(signer=stewardSigner)
# prepare client to submit change ha request to sovrin
randomClientPort = random.randint(9700, 9799)
client = Client(stewardName,
ha=('0.0.0.0', randomClientPort),
config=config)
looper.add(client)
looper.run(
eventually(__checkClientConnected, client, retryWait=1, timeout=5))
nodeVerKey = SimpleSigner(seed=nodeSeed).verkey
# send request
req = submitNodeIpChange(client, stewardWallet, nodeName, nodeVerKey,
newNodeHA, newClientHA)
return client, req
def parseLedgerForHaAndKeys(ledger, returnActive=True):
"""
Returns validator ip, ports and keys
:param ledger:
:param returnActive: If returnActive is True, return only those
validators which are not out of service
:return:
"""
nodeReg = OrderedDict()
cliNodeReg = OrderedDict()
nodeKeys = {}
activeValidators = set()
for _, txn in ledger.getAllTxn().items():
if txn[TXN_TYPE] == NODE:
nodeName = txn[DATA][ALIAS]
clientStackName = nodeName + CLIENT_STACK_SUFFIX
nHa = (txn[DATA][NODE_IP], txn[DATA][NODE_PORT]) \
if (NODE_IP in txn[DATA] and NODE_PORT in txn[DATA]) \
else None
cHa = (txn[DATA][CLIENT_IP], txn[DATA][CLIENT_PORT]) \
if (CLIENT_IP in txn[DATA] and CLIENT_PORT in txn[DATA]) \
else None
if nHa:
nodeReg[nodeName] = HA(*nHa)
if cHa:
cliNodeReg[clientStackName] = HA(*cHa)
verkey = cryptonymToHex(txn[TARGET_NYM])
nodeKeys[nodeName] = verkey
services = txn[DATA].get(SERVICES)
if isinstance(services, list):
if VALIDATOR in services:
activeValidators.add(nodeName)
else:
activeValidators.discard(nodeName)
if returnActive:
allNodes = tuple(nodeReg.keys())
for nodeName in allNodes:
if nodeName not in activeValidators:
nodeReg.pop(nodeName, None)
cliNodeReg.pop(nodeName + CLIENT_STACK_SUFFIX, None)
nodeKeys.pop(nodeName, None)
return nodeReg, cliNodeReg, nodeKeys
else:
return nodeReg, cliNodeReg, nodeKeys, activeValidators
def txnPoolCliNodeReg(poolTxnData):
cliNodeReg = {}
for txn in poolTxnData["txns"]:
if txn[TXN_TYPE] == NEW_NODE:
data = txn[DATA]
cliNodeReg[data[ALIAS] + CLIENT_STACK_SUFFIX] = HA(
data[CLIENT_IP], data[CLIENT_PORT])
return cliNodeReg
def test_node_load_after_disconnect(looper, txnPoolNodeSet, tconf,
tdirWithPoolTxns, allPluginsPath,
poolTxnStewardData, capsys):
client, wallet = buildPoolClientAndWallet(poolTxnStewardData,
tdirWithPoolTxns,
clientClass=TestClient)
looper.add(client)
looper.run(client.ensureConnectedToNodes())
nodes = txnPoolNodeSet
x = nodes[-1]
with capsys.disabled():
print("Stopping node {} with pool ledger size {}".format(
x, x.poolManager.txnSeqNo))
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, x)
looper.removeProdable(x)
client_batches = 80
txns_per_batch = 10
for i in range(client_batches):
s = perf_counter()
sendReqsToNodesAndVerifySuffReplies(looper,
wallet,
client,
txns_per_batch,
override_timeout_limit=True)
with capsys.disabled():
print('{} executed {} client txns in {:.2f} seconds'.format(
i + 1, txns_per_batch,
perf_counter() - s))
nodeHa, nodeCHa = HA(*x.nodestack.ha), HA(*x.clientstack.ha)
newNode = TestNode(x.name,
basedirpath=tdirWithPoolTxns,
base_data_dir=tdirWithPoolTxns,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
looper.add(newNode)
txnPoolNodeSet[-1] = newNode
looper.run(checkNodesConnected(txnPoolNodeSet))
def testNodeKeysChanged(looper,
txnPoolNodeSet,
tdirWithPoolTxns,
tconf,
steward1,
nodeThetaAdded,
allPluginsPath=None):
newSteward, newStewardWallet, newNode = nodeThetaAdded
# Since the node returned by fixture `nodeThetaAdded` was abandoned in the
# previous test, so getting node `Theta` from `txnPoolNodeSet`
newNode = getNodeWithName(txnPoolNodeSet, newNode.name)
newNode.stop()
nodeHa, nodeCHa = HA(*newNode.nodestack.ha), HA(*newNode.clientstack.ha)
sigseed = randomString(32).encode()
verkey = SimpleSigner(seed=sigseed).naclSigner.verhex.decode()
changeNodeKeys(looper, newSteward, newStewardWallet, newNode, verkey)
initLocalKeep(newNode.name, tdirWithPoolTxns, sigseed)
initLocalKeep(newNode.name + CLIENT_STACK_SUFFIX, tdirWithPoolTxns,
sigseed)
looper.removeProdable(name=newNode.name)
logger.debug("{} starting with HAs {} {}".format(newNode, nodeHa, nodeCHa))
node = TestNode(newNode.name,
basedirpath=tdirWithPoolTxns,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
looper.add(node)
# The last element of `txnPoolNodeSet` is the node Theta that was just
# stopped
txnPoolNodeSet[-1] = node
looper.run(checkNodesConnected(txnPoolNodeSet))
looper.run(
eventually(checkNodeLedgersForEquality,
node,
*txnPoolNodeSet[:-1],
retryWait=1,
timeout=10))
ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1,
*txnPoolNodeSet)
ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward,
*txnPoolNodeSet)
def createClientAndWalletWithSeed(name, seed, ha=None):
if isinstance(seed, str):
seed = seed.encode()
if not ha:
port = genHa()[1]
ha = HA('0.0.0.0', port)
wallet = Wallet(name)
wallet.addIdentifier(signer=DidSigner(seed=seed))
client = Client(name, ha=ha)
return client, wallet
def spawnClient(clientName, port, signerSeed, host='0.0.0.0'):
clientAddress = HA(host, port)
# from plenum.client.request_id_store import FileRequestIdStore
# walletFilePath = os.path.join(config.baseDir, "wallet")
# print("Storing request ids in {}".format(walletFilePath))
# store = FileRequestIdStore(walletFilePath)
# wallet = Wallet(clientName, store)
wallet = Wallet(clientName)
wallet.addIdentifier(signer=DidSigner(seed=signerSeed))
client = Client(clientName, ha=clientAddress)
return client, wallet
def addNewNodeAndConnect(self, txn):
"""
Add a new node to remote keep and connect to it.
"""
verkey, pubkey = hexlify(base64_decode(txn[TARGET_NYM].encode())), \
txn[DATA][PUBKEY]
nodeName = txn[DATA][ALIAS]
nodeHa = (txn[DATA][NODE_IP], txn[DATA][NODE_PORT])
try:
initRemoteKeep(self.name, nodeName, self.basedirpath, pubkey,
verkey)
except Exception as ex:
logger.debug("Exception while initializing keep for remote {}".
format(ex))
self.node.nodestack.nodeReg[nodeName] = HA(*nodeHa)
def getStackParamsAndNodeReg(self, name, basedirpath, nodeRegistry=None,
ha=None, cliname=None, cliha=None):
nstack = None
cstack = None
nodeReg = OrderedDict()
for _, txn in self.poolTxnStore.getAllTxn().items():
if txn[TXN_TYPE] == NEW_NODE:
verkey, pubkey = hexlify(
base64_decode(txn[TARGET_NYM].encode())), \
txn[DATA][PUBKEY]
nodeName = txn[DATA][ALIAS]
nodeHa = (txn[DATA][NODE_IP], txn[DATA][NODE_PORT])
nodeReg[nodeName] = HA(*nodeHa)
if nodeName == name:
nstack = dict(name=name,
ha=HA('0.0.0.0', txn[DATA][NODE_PORT]),
main=True,
auto=AutoMode.never)
cstack = dict(name=nodeName + CLIENT_STACK_SUFFIX,
ha=HA('0.0.0.0', txn[DATA][CLIENT_PORT]),
main=True,
auto=AutoMode.always)
if basedirpath:
nstack['basedirpath'] = basedirpath
cstack['basedirpath'] = basedirpath
else:
try:
initRemoteKeep(name, nodeName, basedirpath, pubkey,
verkey)
except Exception as ex:
print(ex)
elif txn[TXN_TYPE] in (NEW_STEWARD, NEW_CLIENT) \
and self.config.clientBootStrategy == \
ClientBootStrategy.PoolTxn:
self.addNewRole(txn)
return nstack, cstack, nodeReg
def checkIfConnectedToAll(self):
connected = 0
# TODO refactor to not use values
for address in self.nodeReg.values():
for remote in self.nodestack.remotes.values():
if HA(*remote.ha) == address:
if Stack.isRemoteConnected(remote):
connected += 1
break
totalNodes = len(self.nodeReg)
if connected == 0:
raise NotConnectedToAny()
elif connected < totalNodes:
raise NotFullyConnected()
else:
assert connected == totalNodes
def __init__(self, port: int, msgHandler: Callable,
name: str=None, basedirpath: str=None):
if name and basedirpath:
ha = getHaFromLocalEstate(name, basedirpath)
if ha and ha[1] != port:
port = ha[1]
stackParams = {
"name": name or randomString(8),
"ha": HA("0.0.0.0", port),
"main": True,
"auto": AutoMode.always,
"mutable": "mutable"
}
if basedirpath:
stackParams["basedirpath"] = basedirpath
super().__init__(stackParams, self.baseMsgHandler)
self.msgHandler = msgHandler
def processPoolTxn(self, txn):
logger.debug("{} processing pool txn {} ".format(self, txn))
typ = get_type(txn)
txn_data = get_payload_data(txn)
if typ == NODE:
remoteName = txn_data[DATA][ALIAS] + CLIENT_STACK_SUFFIX
nodeName = txn_data[DATA][ALIAS]
nodeNym = txn_data[TARGET_NYM]
def _update(txn_data):
if {NODE_IP, NODE_PORT, CLIENT_IP, CLIENT_PORT}.\
intersection(set(txn_data[DATA].keys())):
self.stackHaChanged(txn_data, remoteName, self)
if VERKEY in txn_data:
self.stackKeysChanged(txn_data, remoteName, self)
if SERVICES in txn_data[DATA]:
self.nodeServicesChanged(txn_data)
self.setPoolParams()
if nodeName in self.nodeReg:
# The node was already part of the pool so update
_update(txn_data)
else:
seqNos, info = self.getNodeInfoFromLedger(nodeNym)
if len(seqNos) == 1:
# Since only one transaction has been made, this is a new
# node transactions
self.connectNewRemote(txn_data, remoteName, self)
self.setPoolParams()
else:
self.nodeReg[nodeName + CLIENT_STACK_SUFFIX] = HA(
info[DATA][CLIENT_IP], info[DATA][CLIENT_PORT])
_update(txn_data)
else:
logger.error("{} received unknown txn type {} in txn {}"
.format(self.name, typ, txn))
return
def __init__(self,
name: str,
nodeReg: Dict[str, HA] = None,
ha: Union[HA, Tuple[str, int]] = None,
lastReqId: int = 0,
signer: Signer = None,
signers: Dict[str, Signer] = None,
basedirpath: str = None):
"""
Creates a new client.
:param name: unique identifier for the client
:param nodeReg: names and host addresses of all nodes in the pool
:param ha: tuple of host and port
:param lastReqId: Request Id of the last request sent by client
:param signer: Signer; mutually exclusive of signers
:param signers: Dict of identifier -> Signer; useful for clients that
need to support multiple signers
"""
self.lastReqId = lastReqId
self._clientStack = None
self.minimumNodes = getMaxFailures(len(nodeReg)) + 1
cliNodeReg = OrderedDict()
for nm in nodeReg:
val = nodeReg[nm]
if len(val) == 3:
((ip, port), verkey, pubkey) = val
else:
ip, port = val
cliNodeReg[nm] = HA(ip, port)
nodeReg = cliNodeReg
cha = ha if isinstance(ha, HA) else HA(*ha)
stackargs = dict(
name=name,
ha=cha,
main=False, # stops incoming vacuous joins
auto=AutoMode.always)
if basedirpath:
stackargs['basedirpath'] = basedirpath
self.created = time.perf_counter()
NodeStacked.__init__(self, stackParams=stackargs, nodeReg=nodeReg)
logger.info("Client initialized with the following node registry:")
lengths = [
max(x) for x in zip(*[(len(name), len(host), len(str(port)))
for name, (host, port) in nodeReg.items()])
]
fmt = " {{:<{}}} listens at {{:<{}}} on port {{:>{}}}".format(
*lengths)
for name, (host, port) in nodeReg.items():
logger.info(fmt.format(name, host, port))
Motor.__init__(self)
self.inBox = deque()
if signer and signers:
raise ValueError("only one of 'signer' or 'signers' can be used")
self.signers = None
self.defaultIdentifier = None
if signer:
self.signers = {signer.identifier: signer}
self.defaultIdentifier = signer.identifier
elif signers:
self.signers = signers
else:
self.setupDefaultSigner()
self.connectNicelyUntil = 0 # don't need to connect nicely as a client
def test_node_catchup_after_restart_with_txns(
sdk_new_node_caught_up, txnPoolNodeSet, tdir, tconf,
sdk_node_set_with_node_added_after_some_txns, allPluginsPath):
"""
A node that restarts after some transactions should eventually get the
transactions which happened while it was down
:return:
"""
looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \
sdk_node_set_with_node_added_after_some_txns
logger.debug("Stopping node {} with pool ledger size {}".format(
new_node, new_node.poolManager.txnSeqNo))
disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, new_node)
looper.removeProdable(new_node)
# for n in txnPoolNodeSet[:4]:
# for r in n.nodestack.remotes.values():
# if r.name == newNode.name:
# r.removeStaleCorrespondents()
# looper.run(eventually(checkNodeDisconnectedFrom, newNode.name,
# txnPoolNodeSet[:4], retryWait=1, timeout=5))
# TODO: Check if the node has really stopped processing requests?
logger.debug("Sending requests")
more_requests = 5
sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle,
new_steward_wallet_handle, more_requests)
logger.debug("Starting the stopped node, {}".format(new_node))
nodeHa, nodeCHa = HA(*new_node.nodestack.ha), HA(*new_node.clientstack.ha)
config_helper = PNodeConfigHelper(new_node.name, tconf, chroot=tdir)
newNode = TestNode(new_node.name,
config_helper=config_helper,
config=tconf,
ha=nodeHa,
cliha=nodeCHa,
pluginPaths=allPluginsPath)
looper.add(newNode)
txnPoolNodeSet[-1] = newNode
# Make sure ledger is not synced initially
check_ledger_state(newNode, DOMAIN_LEDGER_ID, LedgerState.not_synced)
# Delay catchup reply processing so LedgerState does not change
# TODO fix delay, sometimes it's not enough and lower 'check_ledger_state'
# fails because newNode's domain ledger state is 'synced'
delay_catchup_reply = 10
newNode.nodeIbStasher.delay(cr_delay(delay_catchup_reply))
looper.run(checkNodesConnected(txnPoolNodeSet))
# Make sure ledger starts syncing (sufficient consistency proofs received)
looper.run(
eventually(check_ledger_state,
newNode,
DOMAIN_LEDGER_ID,
LedgerState.syncing,
retryWait=.5,
timeout=5))
confused_node = txnPoolNodeSet[0]
new_node_ledger = newNode.ledgerManager.ledgerRegistry[DOMAIN_LEDGER_ID]
cp = new_node_ledger.catchUpTill
start, end = cp.seqNoStart, cp.seqNoEnd
cons_proof = confused_node.ledgerManager._buildConsistencyProof(
DOMAIN_LEDGER_ID, start, end)
bad_send_time = None
def chk():
nonlocal bad_send_time
entries = newNode.ledgerManager.spylog.getAll(
newNode.ledgerManager.canProcessConsistencyProof.__name__)
for entry in entries:
# `canProcessConsistencyProof` should return False after `syncing_time`
if entry.result == False and entry.starttime > bad_send_time:
return
assert False
def send_and_chk(ledger_state):
nonlocal bad_send_time, cons_proof
bad_send_time = perf_counter()
confused_node.ledgerManager.sendTo(cons_proof, newNode.name)
# Check that the ConsistencyProof messages rejected
looper.run(eventually(chk, retryWait=.5, timeout=5))
check_ledger_state(newNode, DOMAIN_LEDGER_ID, ledger_state)
send_and_chk(LedgerState.syncing)
# Not accurate timeout but a conservative one
timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) + \
2 * delay_catchup_reply
waitNodeDataEquality(looper,
newNode,
*txnPoolNodeSet[:-1],
customTimeout=timeout)
assert new_node_ledger.num_txns_caught_up == more_requests
send_and_chk(LedgerState.synced)
def ensureNewNodeConnectedClient(looper, client: TestClient, node: TestNode):
stackParams = node.clientStackParams
client.nodeReg[stackParams['name']] = HA('127.0.0.1', stackParams['ha'][1])
looper.run(client.ensureConnectedToNodes())
def __init__(self,
name: str,
nodeReg: Dict[str, HA] = None,
ha: Union[HA, Tuple[str, int]] = None,
basedirpath: str = None,
config=None):
"""
Creates a new client.
:param name: unique identifier for the client
:param nodeReg: names and host addresses of all nodes in the pool
:param ha: tuple of host and port
:param lastReqId: Request Id of the last request sent by client
"""
self.config = config or getConfig()
basedirpath = self.config.baseDir if not basedirpath else basedirpath
self.basedirpath = basedirpath
cha = None
# If client information already exists is RAET then use that
if self.exists(name, basedirpath):
logger.debug("Client {} ignoring given ha".format(ha))
cha = getHaFromLocalEstate(name, basedirpath)
if cha:
cha = HA(*cha)
if not cha:
cha = ha if isinstance(ha, HA) else HA(*ha)
self.name = name
self.reqRepStore = self.getReqRepStore()
self.txnLog = self.getTxnLogStore()
self.dataDir = self.config.clientDataDir or "data/clients"
HasFileStorage.__init__(self,
self.name,
baseDir=self.basedirpath,
dataDir=self.dataDir)
self._ledger = None
if not nodeReg:
self.mode = None
HasPoolManager.__init__(self)
self.ledgerManager = LedgerManager(self, ownedByNode=False)
self.ledgerManager.addLedger(
0,
self.ledger,
postCatchupCompleteClbk=self.postPoolLedgerCaughtUp,
postTxnAddedToLedgerClbk=self.postTxnFromCatchupAddedToLedger)
else:
cliNodeReg = OrderedDict()
for nm, (ip, port) in nodeReg.items():
cliNodeReg[nm] = HA(ip, port)
self.nodeReg = cliNodeReg
self.mode = Mode.discovered
self.setF()
stackargs = dict(
name=name,
ha=cha,
main=False, # stops incoming vacuous joins
auto=AutoMode.always)
stackargs['basedirpath'] = basedirpath
self.created = time.perf_counter()
# noinspection PyCallingNonCallable
self.nodestack = self.nodeStackClass(stackargs, self.handleOneNodeMsg,
self.nodeReg)
self.nodestack.onConnsChanged = self.onConnsChanged
logger.info(
"Client {} initialized with the following node registry:".format(
name))
lengths = [
max(x)
for x in zip(*[(len(name), len(host), len(str(port)))
for name, (host, port) in self.nodeReg.items()])
]
fmt = " {{:<{}}} listens at {{:<{}}} on port {{:>{}}}".format(
*lengths)
for name, (host, port) in self.nodeReg.items():
logger.info(fmt.format(name, host, port))
Motor.__init__(self)
self.inBox = deque()
self.nodestack.connectNicelyUntil = 0 # don't need to connect
# nicely as a client
# TODO: Need to have couple of tests around `reqsPendingConnection`
# where we check with and without pool ledger
# Stores the requests that need to be sent to the nodes when the client
# has made sufficient connections to the nodes.
self.reqsPendingConnection = deque()
tp = loadPlugins(self.basedirpath)
logger.debug("total plugins loaded in client: {}".format(tp))
def _spawnClient(self, name, seed, host='0.0.0.0'):
self.__startPort += randint(100, 1000)
address = HA(host, self.__startPort)
logger.info("Seed for client {} is {}, "
"its len is {}".format(name, seed, len(seed)))
return createClientAndWalletWithSeed(name, seed, address)