def testRescheduleUpgradeToLowerVersionThanPreviouslyScheduled(
looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet):
"""
A node starts at version 1.2 running has scheduled upgrade for version 1.5
but get a txn for upgrade 1.4, it will schedule it and cancel upgrade to 1.5.
"""
upgr1 = deepcopy(validUpgrade)
upgr2 = deepcopy(upgr1)
upgr2[VERSION] = bumpVersion(upgr1[VERSION])
upgr2[NAME] += randomString(3)
# upgr2[SHA256] = get_valid_code_hash()
upgr2[SHA256] = 'ef9c3984e7a31994d4f692139116120bd0dd1ff7e270b6a2d773f8f2f9214d4c'
# An upgrade for higher version scheduled, it should pass
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr2)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr2[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# An upgrade for lower version scheduled, the transaction should pass and
# the upgrade should be scheduled
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def testRescheduleUpgradeToLowerVersionThanPreviouslyScheduled(
looper, tconf, nodeSet, validUpgrade, sdk_pool_handle,
sdk_wallet_trustee):
"""
A node starts at version 1.2 running has scheduled upgrade for version 1.5
but get a txn for upgrade 1.4, it will schedule it and cancel upgrade to 1.5.
"""
upgr1 = deepcopy(validUpgrade)
upgr2 = deepcopy(upgr1)
upgr2[VERSION] = bumpVersion(upgr1[VERSION])
upgr2[NAME] += randomString(3)
# upgr2[SHA256] = get_valid_code_hash()
upgr2[
SHA256] = 'ef9c3984e7a31994d4f692139116120bd0dd1ff7e270b6a2d773f8f2f9214d4c'
# An upgrade for higher version scheduled, it should pass
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr2)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr2[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# An upgrade for lower version scheduled, the transaction should pass and
# the upgrade should be scheduled
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_forced_upgrade_handled_once_if_ordered_and_then_request_received(
looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee,
validUpgradeExpForceTrue):
"""
Verifies that POOL_UPGRADE force=true request is handled one time in case
the node commits the transaction to the ledger and only after that receives
the request directly from the client
"""
slow_node = getNonPrimaryReplicas(nodeSet, instId=0)[-1].node
slow_node.clientIbStasher.delay(req_delay())
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee,
validUpgradeExpForceTrue)
looper.run(
eventually(checkUpgradeScheduled, [slow_node],
validUpgradeExpForceTrue[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
slow_node.clientIbStasher.reset_delays_and_process_delayeds()
looper.runFor(waits.expectedUpgradeScheduled())
checkUpgradeScheduled([slow_node], validUpgradeExpForceTrue[VERSION])
assert count_action_log_package(list(slow_node.upgrader._actionLog),
validUpgradeExpForceTrue['package']) == 1
assert slow_node.upgrader._actionLog.last_event.ev_type == UpgradeLog.Events.scheduled
def test_node_handles_forced_upgrade_on_propagate(looper, nodeSet,
sdk_pool_handle,
sdk_wallet_trustee,
validUpgradeExpForceTrue):
"""
Verifies that POOL_UPGRADE force=true request is handled immediately when
the node receives it in a PROPAGATE from any other node
"""
slow_node = getNonPrimaryReplicas(nodeSet, instId=0)[-1].node
# Stash all except PROPAGATEs from Gamma
slow_node.clientIbStasher.delay(req_delay())
slow_node.nodeIbStasher.delay(ppgDelay(sender_filter='Alpha'))
slow_node.nodeIbStasher.delay(ppgDelay(sender_filter='Beta'))
slow_node.nodeIbStasher.delay(ppDelay())
slow_node.nodeIbStasher.delay(pDelay())
slow_node.nodeIbStasher.delay(cDelay())
sdk_send_upgrade(looper, sdk_pool_handle, sdk_wallet_trustee,
validUpgradeExpForceTrue)
looper.run(
eventually(checkUpgradeScheduled, [slow_node],
validUpgradeExpForceTrue[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def upgradeScheduled(validUpgradeSent, looper, nodeSet, validUpgrade):
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
validUpgrade[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_upgrade_does_not_get_into_loop_if_reinstall(
looper,
tconf,
nodeSet,
validUpgrade,
sdk_pool_handle,
sdk_wallet_trustee,
monkeypatch):
new_version = bumpedVersion()
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
upgr1[REINSTALL] = True
# An upgrade scheduled, it should pass
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# here we make nodes think they have upgraded successfully
monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version)
check_no_loop(nodeSet, UpgradeLog.SUCCEEDED)
def test_upgrade_does_not_get_into_loop_if_reinstall(looper, tconf, nodeSet,
validUpgrade,
sdk_pool_handle,
sdk_wallet_trustee,
monkeypatch):
new_version = bumpedVersion(validUpgrade['version'])
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
upgr1[REINSTALL] = True
clear_aq_stash(nodeSet)
# An upgrade scheduled, it should pass
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# here we make nodes think they have upgraded successfully
monkeypatch.setattr(NodeControlUtil, '_get_curr_info',
lambda *x: "Version: {}".format(new_version))
check_no_loop(nodeSet, UpgradeLog.Events.succeeded)
def test_forced_upgrade_no_consensus_on_single_node(validUpgradeExpForceTrue,
looper, nodeSet,
sdk_pool_handle,
sdk_wallet_trustee):
nup = validUpgradeExpForceTrue.copy()
nup.update({VERSION: bumpVersion(validUpgradeExpForceTrue[VERSION])})
for node in nodeSet:
if node.name != "Alpha" and node in looper.prodables:
node.cleanupOnStopping = False
looper.removeProdable(node)
node.stop()
else:
node.upgrader.scheduledUpgrade = None
_, did = sdk_wallet_trustee
req = get_req_from_update(looper, did, nup)
sdk_sign_and_send_prepared_request(looper, sdk_wallet_trustee,
sdk_pool_handle, req)
def testsched():
for node in nodeSet:
if node.name == "Alpha":
assert node.upgrader.scheduledAction
assert node.upgrader.scheduledAction[0] == nup[VERSION]
looper.run(
eventually(testsched,
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def testNodeSchedulesUpgradeAfterRestart(upgradeScheduled, looper, nodeSet,
validUpgrade, testNodeClass, tdir,
tconf, allPluginsPath):
names = []
while nodeSet:
node = nodeSet.pop()
names.append(node.name)
node.cleanupOnStopping = False
looper.removeProdable(node)
node.stop()
del node
for nm in names:
config_helper = NodeConfigHelper(nm, tconf, chroot=tdir)
node = testNodeClass(nm,
config_helper=config_helper,
config=tconf,
pluginPaths=allPluginsPath)
looper.add(node)
nodeSet.append(node)
looper.run(checkNodesConnected(nodeSet))
ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
validUpgrade[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def upgradeScheduled(validUpgradeSent, looper, nodeSet, validUpgrade):
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
validUpgrade[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_node_reschedules_upgrade_for_proper_datetime(
looper, tconf, nodeSet, validUpgrade,
sdk_pool_handle, sdk_wallet_trustee):
upgr1 = deepcopy(validUpgrade)
# Upgrade 1 of each node will be scheduled for tomorrow
for nodeId in upgr1[SCHEDULE]:
nodeUpgradeDateTime = dateutil.parser.parse(upgr1[SCHEDULE][nodeId])
nodeUpgradeDateTime += timedelta(days=1)
upgr1[SCHEDULE][nodeId] = nodeUpgradeDateTime.isoformat()
upgr2 = deepcopy(upgr1)
upgr2[VERSION] = bumpVersion(upgr1[VERSION])
upgr2[NAME] += randomString(32)
upgr2[SHA256] = 'ef9c3984e7a31994d4f692139116120bd0dd1ff7e270b6a2d773f8f2f9214d4c'
# Upgrade 2 of each node will be scheduled for its own day
# (since today with one day step)
deltaDays = -1
for nodeId in upgr2[SCHEDULE]:
nodeUpgradeDateTime = dateutil.parser.parse(upgr2[SCHEDULE][nodeId])
nodeUpgradeDateTime += timedelta(days=deltaDays)
upgr2[SCHEDULE][nodeId] = nodeUpgradeDateTime.isoformat()
deltaDays += 1
# Upgrade 1 is scheduled
sdk_ensure_upgrade_sent(looper, sdk_pool_handle,
sdk_wallet_trustee, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# Upgrade 2 cancels Upgrade 1 and is scheduled itself
# according its own schedule
sdk_ensure_upgrade_sent(looper, sdk_pool_handle,
sdk_wallet_trustee, upgr2)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr2[VERSION],
upgr2[SCHEDULE],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def poolUpgradeScheduled(poolUpgradeSubmitted, poolNodesStarted, validUpgrade):
nodes = poolNodesStarted.nodes.values()
timeout = waits.expectedUpgradeScheduled()
poolNodesStarted.looper.run(
eventually(checkUpgradeScheduled,
nodes,
validUpgrade[VERSION],
retryWait=1,
timeout=timeout))
def test_node_reschedules_upgrade_for_proper_datetime(looper, tconf, nodeSet,
validUpgrade,
sdk_pool_handle,
sdk_wallet_trustee):
upgr1 = deepcopy(validUpgrade)
# Upgrade 1 of each node will be scheduled for tomorrow
for nodeId in upgr1[SCHEDULE]:
nodeUpgradeDateTime = dateutil.parser.parse(upgr1[SCHEDULE][nodeId])
nodeUpgradeDateTime += timedelta(days=1)
upgr1[SCHEDULE][nodeId] = nodeUpgradeDateTime.isoformat()
upgr2 = deepcopy(upgr1)
upgr2[VERSION] = bumpVersion(upgr1[VERSION])
upgr2[NAME] += randomString(32)
upgr2[
SHA256] = 'ef9c3984e7a31994d4f692139116120bd0dd1ff7e270b6a2d773f8f2f9214d4c'
# Upgrade 2 of each node will be scheduled for its own day
# (since today with one day step)
deltaDays = -1
for nodeId in upgr2[SCHEDULE]:
nodeUpgradeDateTime = dateutil.parser.parse(upgr2[SCHEDULE][nodeId])
nodeUpgradeDateTime += timedelta(days=deltaDays)
upgr2[SCHEDULE][nodeId] = nodeUpgradeDateTime.isoformat()
deltaDays += 1
# Upgrade 1 is scheduled
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# Upgrade 2 cancels Upgrade 1 and is scheduled itself
# according its own schedule
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr2)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr2[VERSION],
upgr2[SCHEDULE],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_do_not_upgrade_to_the_same_version(looper, tconf, nodeSet,
validUpgrade, sdk_pool_handle,
sdk_wallet_trustee):
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = codeVersion()
# An upgrade is not scheduled
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
with pytest.raises(AssertionError):
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_do_not_upgrade_to_the_same_version(looper, tconf, nodeSet,
validUpgrade, trustee,
trusteeWallet):
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = codeVersion()
# An upgrade scheduled, it should pass
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
with pytest.raises(AssertionError):
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_do_not_upgrade_to_the_same_version(looper, tconf, nodeSet,
validUpgrade, sdk_pool_handle,
sdk_wallet_trustee):
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = codeVersion()
# An upgrade is not scheduled
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
with pytest.raises(AssertionError):
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_do_upgrade_to_the_same_version_if_reinstall(looper, tconf, nodeSet,
validUpgrade, trustee,
trusteeWallet):
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = codeVersion()
upgr1[REINSTALL] = True
# An upgrade scheduled, it should pass
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_do_upgrade_to_the_same_version_if_reinstall(looper, tconf, nodeSet,
validUpgrade,
sdk_pool_handle,
sdk_wallet_trustee):
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = lowerVersion(validUpgrade['version'])
upgr1[REINSTALL] = True
# An upgrade scheduled, it should pass
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_forced_upgrade_handled_once_if_request_received_after_propagate(
looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee,
validUpgradeExpForceTrue):
"""
Verifies that POOL_UPGRADE force=true request is handled one time in case
the node commits the transaction to the ledger but during the 3PC-process
receives the request directly from the client after a PROPAGATE from some
other node
"""
slow_node = getNonPrimaryReplicas(nodeSet, instId=0)[-1].node
slow_node.clientIbStasher.delay(req_delay())
slow_node.nodeIbStasher.delay(ppgDelay(sender_filter='Beta'))
slow_node.nodeIbStasher.delay(ppgDelay(sender_filter='Gamma'))
original_process_propagate = slow_node.nodeMsgRouter.routes[Propagate]
original_process_request = slow_node.clientMsgRouter.routes[Request]
def patched_process_propagate(msg: Propagate, frm: str):
original_process_propagate(msg, frm)
slow_node.clientIbStasher.reset_delays_and_process_delayeds()
slow_node.nodeMsgRouter.routes[Propagate] = original_process_propagate
def patched_process_request(request: Request, frm: str):
original_process_request(request, frm)
slow_node.nodeIbStasher.reset_delays_and_process_delayeds()
slow_node.clientMsgRouter.routes[Request] = original_process_request
slow_node.nodeMsgRouter.routes[Propagate] = patched_process_propagate
slow_node.clientMsgRouter.routes[Request] = patched_process_request
init_len = len(list(slow_node.upgrader._actionLog))
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee,
validUpgradeExpForceTrue)
looper.runFor(waits.expectedUpgradeScheduled())
checkUpgradeScheduled([slow_node], validUpgradeExpForceTrue[VERSION])
if init_len == 0:
# first upgrade - should be only one scheduled
assert len(list(slow_node.upgrader._actionLog)) == 1
else:
# one upgrade were already scheduled. we should cancel it and schedule new one
# so action log should be increased by 2
assert len(list(slow_node.upgrader._actionLog)) == init_len + 2
assert slow_node.upgrader._actionLog.lastEvent[1] == UpgradeLog.SCHEDULED
def test_update_with_demoted_node(looper, nodeSet, validUpgrade,
sdk_pool_handle, sdk_wallet_stewards,
sdk_wallet_trustee):
# demote one node
demote_node(looper, sdk_wallet_stewards[3], sdk_pool_handle, nodeSet[3])
# remove demoted node from upgrade schedule
upgr = validUpgrade
del upgr[SCHEDULE][nodeSet[3].id]
# send upgrade
sdk_ensure_upgrade_sent(looper, sdk_pool_handle,
sdk_wallet_trustee, upgr)
# check upg scheduled
looper.run(eventually(checkUpgradeScheduled, nodeSet[:3], upgr[VERSION], retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_upgrade_does_not_get_into_loop(looper, tconf, nodeSet, validUpgrade,
trustee, trusteeWallet, monkeypatch):
new_version = bumpedVersion()
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
# An upgrade scheduled, it should pass
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# here we make nodes think they have upgraded successfully
monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version)
check_no_loop(nodeSet, UpgradeLog.SUCCEEDED)
def test_upgrade_does_not_get_into_loop_if_failed(looper, tconf, nodeSet,
validUpgrade, trustee,
trusteeWallet, monkeypatch):
new_version = bumpedVersion()
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
# An upgrade scheduled, it should pass
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# we have not patched indy_node version so nodes think the upgrade had
# failed
check_no_loop(nodeSet, UpgradeLog.FAILED)
def test_upgrade_does_not_get_into_loop_if_failed(looper, tconf, nodeSet,
validUpgrade, sdk_pool_handle,
sdk_wallet_trustee, monkeypatch):
new_version = bumpedVersion()
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
# An upgrade scheduled, it should pass
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# we have not patched indy_node version so nodes think the upgrade had
# failed
check_no_loop(nodeSet, UpgradeLog.FAILED)
def test_update_with_demoted_node(looper, nodeSet, validUpgrade,
sdk_pool_handle, sdk_wallet_stewards,
sdk_wallet_trustee):
# demote one node
demote_node(looper, sdk_wallet_stewards[3], sdk_pool_handle, nodeSet[3])
# remove demoted node from upgrade schedule
upgr = validUpgrade
del upgr[SCHEDULE][nodeSet[3].id]
# send upgrade
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr)
# check upg scheduled
looper.run(
eventually(checkUpgradeScheduled,
nodeSet[:3],
upgr[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_update_with_demoted_node(looper, nodeSet, validUpgrade,
stewards_and_wallets, trustee, trusteeWallet):
# demote one node
node_steward_cl, steward_wallet = stewards_and_wallets[3]
node_data = {
ALIAS: nodeSet[3].name,
SERVICES: []
}
updateNodeData(looper, node_steward_cl, steward_wallet, nodeSet[3], node_data)
# remove demoted node from upgrade schedule
upgr = validUpgrade
del upgr[SCHEDULE][nodeSet[3].id]
# send upgrade
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr)
# check upg scheduled
looper.run(eventually(checkUpgradeScheduled, nodeSet[:3], upgr[VERSION], retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_upgrade_does_not_get_into_loop_force_if_failed(
looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet, monkeypatch):
new_version = bumpedVersion()
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
upgr1[FORCE] = True
# An upgrade scheduled, it should pass
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# we have not patched indy_node version so nodes think the upgrade had
# failed
check_no_loop(nodeSet, UpgradeLog.UPGRADE_FAILED)
def test_upgrade_does_not_get_into_loop(looper, tconf, nodeSet,
validUpgrade, trustee,
trusteeWallet, monkeypatch):
new_version = bumpedVersion()
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
# An upgrade scheduled, it should pass
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# here we make nodes think they have upgraded successfully
monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version)
check_no_loop(nodeSet, UpgradeLog.UPGRADE_SUCCEEDED)
def test_forced_upgrade_no_consensus_on_single_node(
validUpgradeExpForceTrue, looper, nodeSet, trustee, trusteeWallet):
nup = validUpgradeExpForceTrue.copy()
nup.update({VERSION: bumpVersion(validUpgradeExpForceTrue[VERSION])})
for node in nodeSet:
if node.name != "Alpha":
node.cleanupOnStopping = False
looper.removeProdable(node)
node.stop()
else:
node.upgrader.scheduledUpgrade = None
sendUpgrade(trustee, trusteeWallet, nup)
def testsched():
for node in nodeSet:
if node.name == "Alpha":
assert node.upgrader.scheduledUpgrade
assert node.upgrader.scheduledUpgrade[0] == nup[VERSION]
looper.run(eventually(testsched, retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_upgrade_does_not_get_into_loop_if_reinstall_and_failed(
looper, tconf, nodeSet, validUpgrade, sdk_pool_handle,
sdk_wallet_trustee, monkeypatch):
new_version = bumpedVersion(validUpgrade['version'])
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
upgr1[REINSTALL] = True
clear_aq_stash(nodeSet)
# An upgrade scheduled, it should pass
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# we have not patched indy_node version so nodes think the upgrade had
# failed
check_no_loop(nodeSet, UpgradeLog.Events.failed)
def test_upgrade_does_not_get_into_loop_force(looper, tconf, nodeSet,
validUpgrade, sdk_pool_handle,
sdk_wallet_trustee, monkeypatch):
new_version = bumpedVersion(validUpgrade['version'])
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
upgr1[FORCE] = True
clear_aq_stash(nodeSet)
# An upgrade scheduled, it should pass
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# here we make nodes think they have upgraded successfully
monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version)
check_no_loop(nodeSet, UpgradeLog.SUCCEEDED)
def test_node_handles_forced_upgrade_on_client_request(
looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee,
validUpgradeExpForceTrue):
"""
Verifies that POOL_UPGRADE force=true request is handled immediately when
the node receives it directly from the client
"""
slow_node = getNonPrimaryReplicas(nodeSet, instId=0)[-1].node
# Stash all except client requests
slow_node.nodeIbStasher.delay(ppgDelay())
slow_node.nodeIbStasher.delay(ppDelay())
slow_node.nodeIbStasher.delay(pDelay())
slow_node.nodeIbStasher.delay(cDelay())
sdk_send_upgrade(looper, sdk_pool_handle, sdk_wallet_trustee,
validUpgradeExpForceTrue)
looper.run(eventually(checkUpgradeScheduled,
[slow_node],
validUpgradeExpForceTrue[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_forced_upgrade_no_consensus_on_single_node(
validUpgradeExpForceTrue, looper, nodeSet, sdk_pool_handle, sdk_wallet_trustee):
nup = validUpgradeExpForceTrue.copy()
nup.update({VERSION: bumpVersion(validUpgradeExpForceTrue[VERSION])})
for node in nodeSet:
if node.name != "Alpha":
node.cleanupOnStopping = False
looper.removeProdable(node)
node.stop()
else:
node.upgrader.scheduledUpgrade = None
_, did = sdk_wallet_trustee
req = get_req_from_update(looper, did, nup)
sdk_sign_and_send_prepared_request(looper, sdk_wallet_trustee, sdk_pool_handle, req)
def testsched():
for node in nodeSet:
if node.name == "Alpha":
assert node.upgrader.scheduledAction
assert node.upgrader.scheduledAction[0] == nup[VERSION]
looper.run(eventually(testsched, retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def testForcedUpgradeNoConsensusOnSingleNode(validUpgradeExpForceTrue, looper,
nodeSet, trustee, trusteeWallet):
nup = validUpgradeExpForceTrue.copy()
nup.update({VERSION: bumpVersion(validUpgradeExpForceTrue[VERSION])})
for node in nodeSet:
if node.name != "Alpha":
node.cleanupOnStopping = False
looper.removeProdable(node)
node.stop()
else:
node.upgrader.scheduledUpgrade = None
sendUpgrade(trustee, trusteeWallet, nup)
def testsched():
for node in nodeSet:
if node.name == "Alpha":
assert node.upgrader.scheduledUpgrade
assert node.upgrader.scheduledUpgrade[0] == nup[VERSION]
looper.run(
eventually(testsched,
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_node_schedules_upgrade_for_proper_datetime(
looper, tconf, nodeSet, validUpgrade, sdk_pool_handle, sdk_wallet_trustee):
upgr1 = deepcopy(validUpgrade)
# Upgrade of each node will be scheduled for its own day
# (since today with one day step)
deltaDays = 0
for nodeId in upgr1[SCHEDULE]:
nodeUpgradeDateTime = dateutil.parser.parse(upgr1[SCHEDULE][nodeId])
nodeUpgradeDateTime += timedelta(days=deltaDays)
upgr1[SCHEDULE][nodeId] = nodeUpgradeDateTime.isoformat()
deltaDays += 1
# Upgrade is scheduled for the proper datetime for each node
sdk_ensure_upgrade_sent(looper, sdk_pool_handle,
sdk_wallet_trustee, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
upgr1[SCHEDULE],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_update_with_demoted_node(looper, nodeSet, validUpgrade,
stewards_and_wallets, trustee,
trusteeWallet):
# demote one node
node_steward_cl, steward_wallet = stewards_and_wallets[3]
node_data = {ALIAS: nodeSet[3].name, SERVICES: []}
updateNodeData(looper, node_steward_cl, steward_wallet, nodeSet[3],
node_data)
# remove demoted node from upgrade schedule
upgr = validUpgrade
del upgr[SCHEDULE][nodeSet[3].id]
# send upgrade
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr)
# check upg scheduled
looper.run(
eventually(checkUpgradeScheduled,
nodeSet[:3],
upgr[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def test_node_schedules_upgrade_for_proper_datetime(looper, tconf, nodeSet,
validUpgrade,
sdk_pool_handle,
sdk_wallet_trustee):
upgr1 = deepcopy(validUpgrade)
# Upgrade of each node will be scheduled for its own day
# (since today with one day step)
deltaDays = 0
for nodeId in upgr1[SCHEDULE]:
nodeUpgradeDateTime = dateutil.parser.parse(upgr1[SCHEDULE][nodeId])
nodeUpgradeDateTime += timedelta(days=deltaDays)
upgr1[SCHEDULE][nodeId] = nodeUpgradeDateTime.isoformat()
deltaDays += 1
# Upgrade is scheduled for the proper datetime for each node
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
upgr1[SCHEDULE],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def testUpgradeLatestUncancelledVersion(looper, txnPoolNodeSet, tconf,
nodeThetaAdded, validUpgrade, trustee,
trusteeWallet, tdirWithPoolTxns,
allPluginsPath):
"""
A node starts and finds several upgrades but selects the latest one which
is not cancelled, eg node is on version 1.2 but finds 1.3, 1.4 and 1.5 but
since 1.5 is cancelled, it selects 1.4
"""
nodeSet = txnPoolNodeSet
newSteward, newStewardWallet, newNode = nodeThetaAdded
for node in nodeSet[:-1]:
node.nodestack.removeRemoteByName(newNode.nodestack.name)
newNode.nodestack.removeRemoteByName(node.nodestack.name)
newNode.stop()
nodeSet = nodeSet[:-1]
looper.removeProdable(newNode)
upgr1 = deepcopy(validUpgrade)
upgr2 = deepcopy(upgr1)
upgr2[VERSION] = bumpVersion(upgr1[VERSION])
upgr2[NAME] += randomString(3)
# upgr2[SHA256] = get_valid_code_hash()
upgr2[
SHA256] = 'db34a72a90d026dae49c3b3f0436c8d3963476c77468ad955845a1ccf7b03f55'
upgr3 = deepcopy(upgr2)
upgr3[VERSION] = bumpVersion(upgr2[VERSION])
upgr3[NAME] += randomString(3)
# upgr3[SHA256] = get_valid_code_hash()
upgr3[
SHA256] = '112c060527e8cecfafe64dcb5bdabc4010cc7b64e0bf9bc2a43d23c37d927128'
upgr4 = deepcopy(upgr3)
upgr4[ACTION] = CANCEL
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet[:-1],
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr2)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet[:-1],
upgr2[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr3)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet[:-1],
upgr3[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr4)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet[:-1],
upgr2[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
trustee.stopRetrying()
newNode = TestNode(newNode.name,
basedirpath=tdirWithPoolTxns,
base_data_dir=tdirWithPoolTxns,
config=tconf,
pluginPaths=allPluginsPath,
ha=newNode.nodestack.ha,
cliha=newNode.clientstack.ha)
looper.add(newNode)
nodeSet.append(newNode)
looper.run(checkNodesConnected(nodeSet))
looper.run(
eventually(checkUpgradeScheduled, [
newNode,
],
upgr2[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
def testUpgradeLatestUncancelledVersion(looper,
txnPoolNodeSet, tconf, sdk_node_theta_added,
validUpgrade, sdk_pool_handle, sdk_wallet_trustee,
tdirWithPoolTxns, allPluginsPath):
"""
A node starts and finds several upgrades but selects the latest one which
is not cancelled, eg node is on version 1.2 but finds 1.3, 1.4 and 1.5 but
since 1.5 is cancelled, it selects 1.4
"""
nodeSet = txnPoolNodeSet
_, newNode = sdk_node_theta_added
for node in nodeSet[:-1]:
node.nodestack.removeRemoteByName(newNode.nodestack.name)
newNode.nodestack.removeRemoteByName(node.nodestack.name)
newNode.stop()
nodeSet = nodeSet[:-1]
looper.removeProdable(newNode)
upgr1 = deepcopy(validUpgrade)
upgr2 = deepcopy(upgr1)
upgr2[VERSION] = bumpVersion(upgr1[VERSION])
upgr2[NAME] += randomString(3)
# upgr2[SHA256] = get_valid_code_hash()
upgr2[SHA256] = 'db34a72a90d026dae49c3b3f0436c8d3963476c77468ad955845a1ccf7b03f55'
upgr3 = deepcopy(upgr2)
upgr3[VERSION] = bumpVersion(upgr2[VERSION])
upgr3[NAME] += randomString(3)
# upgr3[SHA256] = get_valid_code_hash()
upgr3[SHA256] = '112c060527e8cecfafe64dcb5bdabc4010cc7b64e0bf9bc2a43d23c37d927128'
upgr4 = deepcopy(upgr3)
upgr4[ACTION] = CANCEL
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(eventually(checkUpgradeScheduled,
nodeSet[:-1],
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr2)
looper.run(eventually(checkUpgradeScheduled,
nodeSet[:-1],
upgr2[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr3)
looper.run(eventually(checkUpgradeScheduled,
nodeSet[:-1],
upgr3[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr4)
looper.run(eventually(checkUpgradeScheduled,
nodeSet[:-1],
upgr2[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
newNode = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, base_data_dir=tdirWithPoolTxns,
config=tconf, pluginPaths=allPluginsPath,
ha=newNode.nodestack.ha, cliha=newNode.clientstack.ha)
looper.add(newNode)
nodeSet.append(newNode)
looper.run(checkNodesConnected(nodeSet))
looper.run(eventually(checkUpgradeScheduled, [newNode, ], upgr2[VERSION],
retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def poolUpgradeScheduled(poolUpgradeSubmitted, poolNodesStarted, validUpgrade):
nodes = poolNodesStarted.nodes.values()
timeout = waits.expectedUpgradeScheduled()
poolNodesStarted.looper.run(
eventually(checkUpgradeScheduled, nodes,
validUpgrade[VERSION], retryWait=1, timeout=timeout))
