def testRescheduleUpgradeToLowerVersionThanPreviouslyScheduled( looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet): """ A node starts at version 1.2 running has scheduled upgrade for version 1.5 but get a txn for upgrade 1.4, it will schedule it and cancel upgrade to 1.5. """ upgr1 = deepcopy(validUpgrade) upgr2 = deepcopy(upgr1) upgr2[VERSION] = bumpVersion(upgr1[VERSION]) upgr2[NAME] += randomString(3) # upgr2[SHA256] = get_valid_code_hash() upgr2[SHA256] = 'ef9c3984e7a31994d4f692139116120bd0dd1ff7e270b6a2d773f8f2f9214d4c' # An upgrade for higher version scheduled, it should pass ensureUpgradeSent(looper, trustee, trusteeWallet, upgr2) looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr2[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # An upgrade for lower version scheduled, the transaction should pass and # the upgrade should be scheduled ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def test_upgrade_does_not_get_into_loop_if_reinstall( looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet, monkeypatch): new_version = bumpedVersion() upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = new_version upgr1[REINSTALL] = True # An upgrade scheduled, it should pass ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # here we make nodes think they have upgraded successfully monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version) check_no_loop(nodeSet, UpgradeLog.UPGRADE_SUCCEEDED)
def testScheduleUpgradeForALowerVersion(looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet): """ A node starts at version 1.2 running has scheduled upgrade for version 1.5 but get a txn for upgrade 1.4, it will schedule it and cancel upgrade to 1.5. """ upgr1 = deepcopy(validUpgrade) upgr2 = deepcopy(upgr1) upgr2[VERSION] = bumpVersion(upgr1[VERSION]) upgr2[NAME] += randomString(3) # upgr2[SHA256] = get_valid_code_hash() upgr2[ SHA256] = 'ef9c3984e7a31994d4f692139116120bd0dd1ff7e270b6a2d773f8f2f9214d4c' # An upgrade for higher version scheduled, it should pass ensureUpgradeSent(looper, trustee, trusteeWallet, upgr2) looper.run( eventually(checkUpgradeScheduled, nodeSet, upgr2[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # An upgrade for lower version scheduled, the transaction should pass and # the upgrade should be scheduled ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually(checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def validUpgradeSentExpForceFalse( looper, nodeSet, tdir, trustee, trusteeWallet, validUpgradeExpForceFalse): ensureUpgradeSent(looper, trustee, trusteeWallet, validUpgradeExpForceFalse)
def validUpgradeSentExpForceFalse( looper, nodeSet, tdir, trustee, trusteeWallet, validUpgradeExpForceFalse): ensureUpgradeSent(looper, trustee, trusteeWallet, validUpgradeExpForceFalse)
def test_do_not_upgrade_to_the_same_version(looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet): upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = codeVersion() # An upgrade scheduled, it should pass ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) with pytest.raises(AssertionError): looper.run( eventually(checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def test_do_upgrade_to_the_same_version_if_reinstall(looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet): upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = codeVersion() upgr1[REINSTALL] = True # An upgrade scheduled, it should pass ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def test_do_not_upgrade_to_the_same_version(looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet): upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = codeVersion() # An upgrade is not scheduled ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) with pytest.raises(AssertionError): looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def test_node_does_not_reschedule_cancelled_upgrade_after_restart( upgradeScheduled, looper, nodeSet, validUpgrade, testNodeClass, tdir, tconf, allPluginsPath, trustee, trusteeWallet): # Cancel the scheduled upgrade valid_upgrade_cancel = deepcopy(validUpgrade) valid_upgrade_cancel[ACTION] = CANCEL del valid_upgrade_cancel[SCHEDULE] ensureUpgradeSent(looper, trustee, trusteeWallet, valid_upgrade_cancel) # Verify that no upgrade is scheduled now looper.run( eventually( checkNoUpgradeScheduled, nodeSet, retryWait=1, timeout=waits.expectedNoUpgradeScheduled())) # Restart all the nodes names = [] while nodeSet: node = nodeSet.pop() names.append(node.name) node.cleanupOnStopping = False looper.removeProdable(node) node.stop() del node for nm in names: config_helper = NodeConfigHelper(nm, tconf, chroot=tdir) node = testNodeClass(nm, config_helper=config_helper, config=tconf, pluginPaths=allPluginsPath) looper.add(node) nodeSet.append(node) looper.run(checkNodesConnected(nodeSet)) ensureElectionsDone(looper=looper, nodes=nodeSet, retryWait=1) # Verify that still no upgrade is scheduled looper.run( eventually( checkNoUpgradeScheduled, nodeSet, retryWait=1, timeout=waits.expectedNoUpgradeScheduled()))
def test_upgrade_does_not_get_into_loop(looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet, monkeypatch): new_version = bumpedVersion() upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = new_version # An upgrade scheduled, it should pass ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually(checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # here we make nodes think they have upgraded successfully monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version) check_no_loop(nodeSet, UpgradeLog.SUCCEEDED)
def test_node_reschedules_upgrade_for_proper_datetime(looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet): upgr1 = deepcopy(validUpgrade) # Upgrade 1 of each node will be scheduled for tomorrow for nodeId in upgr1[SCHEDULE]: nodeUpgradeDateTime = dateutil.parser.parse(upgr1[SCHEDULE][nodeId]) nodeUpgradeDateTime += timedelta(days=1) upgr1[SCHEDULE][nodeId] = nodeUpgradeDateTime.isoformat() upgr2 = deepcopy(upgr1) upgr2[VERSION] = bumpVersion(upgr1[VERSION]) upgr2[NAME] += randomString(32) upgr2[ SHA256] = 'ef9c3984e7a31994d4f692139116120bd0dd1ff7e270b6a2d773f8f2f9214d4c' # Upgrade 2 of each node will be scheduled for its own day # (since today with one day step) deltaDays = -1 for nodeId in upgr2[SCHEDULE]: nodeUpgradeDateTime = dateutil.parser.parse(upgr2[SCHEDULE][nodeId]) nodeUpgradeDateTime += timedelta(days=deltaDays) upgr2[SCHEDULE][nodeId] = nodeUpgradeDateTime.isoformat() deltaDays += 1 # Upgrade 1 is scheduled ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually(checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # Upgrade 2 cancels Upgrade 1 and is scheduled itself # according its own schedule ensureUpgradeSent(looper, trustee, trusteeWallet, upgr2) looper.run( eventually(checkUpgradeScheduled, nodeSet, upgr2[VERSION], upgr2[SCHEDULE], retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def test_node_reschedules_upgrade_for_proper_datetime( looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet): upgr1 = deepcopy(validUpgrade) # Upgrade 1 of each node will be scheduled for tomorrow for nodeId in upgr1[SCHEDULE]: nodeUpgradeDateTime = dateutil.parser.parse(upgr1[SCHEDULE][nodeId]) nodeUpgradeDateTime += timedelta(days=1) upgr1[SCHEDULE][nodeId] = nodeUpgradeDateTime.isoformat() upgr2 = deepcopy(upgr1) upgr2[VERSION] = bumpVersion(upgr1[VERSION]) upgr2[NAME] += randomString(32) upgr2[SHA256] = 'ef9c3984e7a31994d4f692139116120bd0dd1ff7e270b6a2d773f8f2f9214d4c' # Upgrade 2 of each node will be scheduled for its own day # (since today with one day step) deltaDays = -1 for nodeId in upgr2[SCHEDULE]: nodeUpgradeDateTime = dateutil.parser.parse(upgr2[SCHEDULE][nodeId]) nodeUpgradeDateTime += timedelta(days=deltaDays) upgr2[SCHEDULE][nodeId] = nodeUpgradeDateTime.isoformat() deltaDays += 1 # Upgrade 1 is scheduled ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # Upgrade 2 cancels Upgrade 1 and is scheduled itself # according its own schedule ensureUpgradeSent(looper, trustee, trusteeWallet, upgr2) looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr2[VERSION], upgr2[SCHEDULE], retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def test_upgrade_does_not_get_into_loop_if_failed(looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet, monkeypatch): new_version = bumpedVersion() upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = new_version # An upgrade scheduled, it should pass ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually(checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # we have not patched indy_node version so nodes think the upgrade had # failed check_no_loop(nodeSet, UpgradeLog.FAILED)
def test_update_with_demoted_node(looper, nodeSet, validUpgrade, stewards_and_wallets, trustee, trusteeWallet): # demote one node node_steward_cl, steward_wallet = stewards_and_wallets[3] node_data = { ALIAS: nodeSet[3].name, SERVICES: [] } updateNodeData(looper, node_steward_cl, steward_wallet, nodeSet[3], node_data) # remove demoted node from upgrade schedule upgr = validUpgrade del upgr[SCHEDULE][nodeSet[3].id] # send upgrade ensureUpgradeSent(looper, trustee, trusteeWallet, upgr) # check upg scheduled looper.run(eventually(checkUpgradeScheduled, nodeSet[:3], upgr[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def test_upgrade_does_not_get_into_loop_force_if_failed( looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet, monkeypatch): new_version = bumpedVersion() upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = new_version upgr1[FORCE] = True # An upgrade scheduled, it should pass ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # we have not patched indy_node version so nodes think the upgrade had # failed check_no_loop(nodeSet, UpgradeLog.UPGRADE_FAILED)
def test_node_schedules_upgrade_for_proper_datetime(looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet): upgr1 = deepcopy(validUpgrade) # Upgrade of each node will be scheduled for its own day # (since today with one day step) deltaDays = 0 for nodeId in upgr1[SCHEDULE]: nodeUpgradeDateTime = dateutil.parser.parse(upgr1[SCHEDULE][nodeId]) nodeUpgradeDateTime += timedelta(days=deltaDays) upgr1[SCHEDULE][nodeId] = nodeUpgradeDateTime.isoformat() deltaDays += 1 # Upgrade is scheduled for the proper datetime for each node ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually(checkUpgradeScheduled, nodeSet, upgr1[VERSION], upgr1[SCHEDULE], retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def test_node_schedules_upgrade_for_proper_datetime( looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet): upgr1 = deepcopy(validUpgrade) # Upgrade of each node will be scheduled for its own day # (since today with one day step) deltaDays = 0 for nodeId in upgr1[SCHEDULE]: nodeUpgradeDateTime = dateutil.parser.parse(upgr1[SCHEDULE][nodeId]) nodeUpgradeDateTime += timedelta(days=deltaDays) upgr1[SCHEDULE][nodeId] = nodeUpgradeDateTime.isoformat() deltaDays += 1 # Upgrade is scheduled for the proper datetime for each node ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr1[VERSION], upgr1[SCHEDULE], retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def test_update_with_demoted_node(looper, nodeSet, validUpgrade, stewards_and_wallets, trustee, trusteeWallet): # demote one node node_steward_cl, steward_wallet = stewards_and_wallets[3] node_data = {ALIAS: nodeSet[3].name, SERVICES: []} updateNodeData(looper, node_steward_cl, steward_wallet, nodeSet[3], node_data) # remove demoted node from upgrade schedule upgr = validUpgrade del upgr[SCHEDULE][nodeSet[3].id] # send upgrade ensureUpgradeSent(looper, trustee, trusteeWallet, upgr) # check upg scheduled looper.run( eventually(checkUpgradeScheduled, nodeSet[:3], upgr[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def validUpgradeSent(looper, nodeSet, tdir, trustee, trusteeWallet, validUpgrade): ensureUpgradeSent(looper, trustee, trusteeWallet, validUpgrade)
def validUpgradeSent(looper, nodeSet, tdir, trustee, trusteeWallet, validUpgrade): ensureUpgradeSent(looper, trustee, trusteeWallet, validUpgrade)
def testUpgradeLatestUncancelledVersion(looper, txnPoolNodeSet, tconf, nodeThetaAdded, validUpgrade, trustee, trusteeWallet, tdirWithPoolTxns, allPluginsPath): """ A node starts and finds several upgrades but selects the latest one which is not cancelled, eg node is on version 1.2 but finds 1.3, 1.4 and 1.5 but since 1.5 is cancelled, it selects 1.4 """ nodeSet = txnPoolNodeSet newSteward, newStewardWallet, newNode = nodeThetaAdded for node in nodeSet[:-1]: node.nodestack.removeRemoteByName(newNode.nodestack.name) newNode.nodestack.removeRemoteByName(node.nodestack.name) newNode.stop() nodeSet = nodeSet[:-1] looper.removeProdable(newNode) upgr1 = deepcopy(validUpgrade) upgr2 = deepcopy(upgr1) upgr2[VERSION] = bumpVersion(upgr1[VERSION]) upgr2[NAME] += randomString(3) # upgr2[SHA256] = get_valid_code_hash() upgr2[SHA256] = 'db34a72a90d026dae49c3b3f0436c8d3963476c77468ad955845a1ccf7b03f55' upgr3 = deepcopy(upgr2) upgr3[VERSION] = bumpVersion(upgr2[VERSION]) upgr3[NAME] += randomString(3) # upgr3[SHA256] = get_valid_code_hash() upgr3[SHA256] = '112c060527e8cecfafe64dcb5bdabc4010cc7b64e0bf9bc2a43d23c37d927128' upgr4 = deepcopy(upgr3) upgr4[ACTION] = CANCEL ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run(eventually(checkUpgradeScheduled, nodeSet[:-1], upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) ensureUpgradeSent(looper, trustee, trusteeWallet, upgr2) looper.run(eventually(checkUpgradeScheduled, nodeSet[:-1], upgr2[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) ensureUpgradeSent(looper, trustee, trusteeWallet, upgr3) looper.run(eventually(checkUpgradeScheduled, nodeSet[:-1], upgr3[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) ensureUpgradeSent(looper, trustee, trusteeWallet, upgr4) looper.run(eventually(checkUpgradeScheduled, nodeSet[:-1], upgr2[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) trustee.stopRetrying() newNode = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, base_data_dir=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath, ha=newNode.nodestack.ha, cliha=newNode.clientstack.ha) looper.add(newNode) nodeSet.append(newNode) looper.run(checkNodesConnected(nodeSet)) looper.run(eventually(checkUpgradeScheduled, [newNode, ], upgr2[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled()))
def testUpgradeLatestUncancelledVersion(looper, txnPoolNodeSet, tconf, nodeThetaAdded, validUpgrade, trustee, trusteeWallet, tdirWithPoolTxns, allPluginsPath): """ A node starts and finds several upgrades but selects the latest one which is not cancelled, eg node is on version 1.2 but finds 1.3, 1.4 and 1.5 but since 1.5 is cancelled, it selects 1.4 """ nodeSet = txnPoolNodeSet newSteward, newStewardWallet, newNode = nodeThetaAdded for node in nodeSet[:-1]: node.nodestack.removeRemoteByName(newNode.nodestack.name) newNode.nodestack.removeRemoteByName(node.nodestack.name) newNode.stop() nodeSet = nodeSet[:-1] looper.removeProdable(newNode) upgr1 = deepcopy(validUpgrade) upgr2 = deepcopy(upgr1) upgr2[VERSION] = bumpVersion(upgr1[VERSION]) upgr2[NAME] += randomString(3) # upgr2[SHA256] = get_valid_code_hash() upgr2[ SHA256] = 'db34a72a90d026dae49c3b3f0436c8d3963476c77468ad955845a1ccf7b03f55' upgr3 = deepcopy(upgr2) upgr3[VERSION] = bumpVersion(upgr2[VERSION]) upgr3[NAME] += randomString(3) # upgr3[SHA256] = get_valid_code_hash() upgr3[ SHA256] = '112c060527e8cecfafe64dcb5bdabc4010cc7b64e0bf9bc2a43d23c37d927128' upgr4 = deepcopy(upgr3) upgr4[ACTION] = CANCEL ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually(checkUpgradeScheduled, nodeSet[:-1], upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) ensureUpgradeSent(looper, trustee, trusteeWallet, upgr2) looper.run( eventually(checkUpgradeScheduled, nodeSet[:-1], upgr2[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) ensureUpgradeSent(looper, trustee, trusteeWallet, upgr3) looper.run( eventually(checkUpgradeScheduled, nodeSet[:-1], upgr3[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) ensureUpgradeSent(looper, trustee, trusteeWallet, upgr4) looper.run( eventually(checkUpgradeScheduled, nodeSet[:-1], upgr2[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) trustee.stopRetrying() newNode = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, base_data_dir=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath, ha=newNode.nodestack.ha, cliha=newNode.clientstack.ha) looper.add(newNode) nodeSet.append(newNode) looper.run(checkNodesConnected(nodeSet)) looper.run( eventually(checkUpgradeScheduled, [ newNode, ], upgr2[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled()))