def test_upgrade_does_not_get_into_loop_if_reinstall( looper, tconf, nodeSet, validUpgrade, sdk_pool_handle, sdk_wallet_trustee, monkeypatch): new_version = bumpedVersion() upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = new_version upgr1[REINSTALL] = True # An upgrade scheduled, it should pass sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1) looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # here we make nodes think they have upgraded successfully monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version) check_no_loop(nodeSet, UpgradeLog.SUCCEEDED)
def testScheduleNodeUpgrade(tconf, nodeSet): """ Tests that upgrade scheduling works. For that it starts mock control service, schedules upgrade for near future and then checks that service received notification. """ loop = asyncio.get_event_loop() server, indicator = loop.run_until_complete( _createServer( host=tconf.controlServiceHost, port=tconf.controlServicePort ) ) indicator.add_done_callback(_stopServer(server)) node = nodeSet[0] # ATTENTION! nodeId and ledger must not be None, but there # we do not call methods that use them, so we can pass None # We do it because node from nodeSet is some testable object, not real # node, so it has no nodeId and ledger that we can use upgrader = Upgrader(nodeId=None, nodeName=None, dataDir=node.dataLocation, config=tconf, ledger=None) version = bumpedVersion() ev_data = UpgradeLogData( datetime.utcnow(), version, 'some_id', tconf.UPGRADE_ENTRY) upgrader._callUpgradeAgent(ev_data, 1000) result = loop.run_until_complete(eventuallySoon(_checkFuture(indicator))) expectedResult = UpgradeMessage(version, tconf.UPGRADE_ENTRY) assert result == expectedResult.toJson()
def testOnlyTrusteeCanSendPoolUpgrade(looper, sdk_pool_handle, sdk_wallet_steward, validUpgrade): # A steward sending POOL_UPGRADE but txn fails validUpgrade = deepcopy(validUpgrade) validUpgrade[NAME] = 'upgrade-20' validUpgrade[VERSION] = bumpedVersion() req = sdk_send_upgrade(looper, sdk_pool_handle, sdk_wallet_steward, validUpgrade) sdk_get_bad_response(looper, [req], RequestRejectedException, 'cannot do')
def test_upgrade_does_not_get_into_loop_if_reinstall(looper, tconf, nodeSet, validUpgrade, sdk_pool_handle, sdk_wallet_trustee, monkeypatch): new_version = bumpedVersion(validUpgrade['version']) upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = new_version upgr1[REINSTALL] = True clear_aq_stash(nodeSet) # An upgrade scheduled, it should pass sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1) looper.run( eventually(checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # here we make nodes think they have upgraded successfully monkeypatch.setattr(NodeControlUtil, '_get_curr_info', lambda *x: "Version: {}".format(new_version)) check_no_loop(nodeSet, UpgradeLog.Events.succeeded)
def testNodeControlCreatesBackups(monkeypatch, tdir, looper, tconf): version = bumpedVersion() stdout = 'teststdout' curr_src_ver = Upgrader.get_src_version() def transform(tool): nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout) monkeypatch.setattr(tool, '_remove_old_backups', lambda *x: None) def checkBackup(tool): assert os.path.isfile('{}.{}'.format( tool._backup_name(curr_src_ver.release), tool.backup_format)) monkeypatch.setattr( NodeControlUtil, 'get_latest_pkg_version', lambda *x, **y: DebianVersion( version, upstream_cls=src_version_cls()) ) nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform, config=tconf) try: sendUpgradeMessage(version) looper.run(eventually(checkBackup, nct.tool)) finally: clean_dir(nct.tool.base_dir) nct.stop()
def testOnlyTrusteeCanSendPoolUpgrade(looper, steward, validUpgrade): # A steward sending POOL_UPGRADE but txn fails stClient, stWallet = steward validUpgrade = deepcopy(validUpgrade) validUpgrade[NAME] = 'upgrade-20' validUpgrade[VERSION] = bumpedVersion() _, req = sendUpgrade(stClient, stWallet, validUpgrade) timeout = plenumWaits.expectedReqNAckQuorumTime() looper.run(eventually(checkRejects, stClient, req.reqId, 'cannot do', retryWait=1, timeout=timeout))
def invalidUpgrade(nodeIds, tconf): schedule = {} unow = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc()) startAt = unow + timedelta(seconds=60) acceptableDiff = tconf.MinSepBetweenNodeUpgrades + 1 for i in nodeIds: schedule[i] = datetime.isoformat(startAt) startAt = startAt + timedelta(seconds=acceptableDiff - 3) return dict(name='upgrade-14', version=bumpedVersion(), action=START, schedule=schedule, # sha256=get_valid_code_hash(), sha256='46c715a90b1067142d548cb1f1405b0486b32b1a27d418ef3a52bd976e9fae50', timeout=10)
def validUpgrade(nodeIds, tconf): schedule = {} unow = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc()) startAt = unow + timedelta(seconds=100) acceptableDiff = tconf.MinSepBetweenNodeUpgrades + 1 for i in nodeIds: schedule[i] = datetime.isoformat(startAt) startAt = startAt + timedelta(seconds=acceptableDiff + 3) return dict(name='upgrade-13', version=bumpedVersion(), action=START, schedule=schedule, # sha256=get_valid_code_hash(), sha256='db34a72a90d026dae49c3b3f0436c8d3963476c77468ad955845a1ccf7b03f55', timeout=1)
def validUpgrade(nodeIds, tconf, monkeypatch, pckg): schedule = {} unow = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc()) startAt = unow + timedelta(seconds=100) acceptableDiff = tconf.MinSepBetweenNodeUpgrades + 1 for i in nodeIds: schedule[i] = datetime.isoformat(startAt) startAt = startAt + timedelta(seconds=acceptableDiff + 3) patch_packet_mgr_output(monkeypatch, pckg[0], pckg[1]) return dict(name='upgrade-{}'.format(randomText(3)), version=bumpedVersion(pckg[1]), action=START, schedule=schedule, timeout=1, package=pckg[0], sha256='db34a72a90d026dae49c3b3f0436c8d3963476c77468ad955845a1ccf7b03f55')
def testNodeControlRemovesBackups(monkeypatch, tdir, looper, tconf): version = bumpedVersion() stdout = 'teststdout' curr_src_ver = Upgrader.get_src_version() backupsWereRemoved = m.Value('b', False) def testRemoveOldBackups(tool): assert len(tool._get_backups()) == (tool.backup_num + 1) #looper = Looper(debug=True) tool._remove_old_backups_test() backupsWereRemoved.value = True def transform(tool): nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout) tool._remove_old_backups_test = tool._remove_old_backups monkeypatch.setattr(tool, '_remove_old_backups', functools.partial(testRemoveOldBackups, tool)) def checkOldBackupsRemoved(): assert backupsWereRemoved.value def check_backups_files_exists(): assert os.path.exists('{}.{}'.format( nct.tool._backup_name(curr_src_ver.release), nct.tool.backup_format)) monkeypatch.setattr( NodeControlUtil, 'get_latest_pkg_version', lambda *x, **y: DebianVersion(version, upstream_cls=src_version_cls())) nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform) try: assert len(nct.tool._get_backups()) == 0 for i in range(nct.tool.backup_num): file = os.path.join(nct.tool.base_dir, '{}{}'.format(nct.tool.backup_name_prefix, i)) with open(file, 'w') as f: f.write('random') assert len(nct.tool._get_backups()) == nct.tool.backup_num sendUpgradeMessage(version) looper.run(eventually(checkOldBackupsRemoved)) looper.run(eventually(check_backups_files_exists)) assert len(nct.tool._get_backups()) == nct.tool.backup_num finally: clean_dir(nct.tool.base_dir) nct.stop()
def testNodeControlPerformsMigrations(monkeypatch, tdir, looper, tconf): version = bumpedVersion() stdout = 'teststdout' migrationFile = 'migrationProof' migrationText = "{} {}".format(releaseVersion(), version) old_call_upgrade_script = None def mock_call_upgrade_script(*args, **kwargs): old_call_upgrade_script(*args, **kwargs) monkeypatch.setattr( NodeControlUtil, '_get_curr_info', lambda *x, **y: ( "Package: {}\nVersion: {}\n".format(APP_NAME, version) ) ) def mock_migrate(curr_src_ver: str, new_src_ver: str): with open(os.path.join(tdir, migrationFile), 'w') as f: f.write("{} {}".format(curr_src_ver, new_src_ver)) def transform(tool): nonlocal old_call_upgrade_script nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout) monkeypatch.setattr(tool, '_do_migration', mock_migrate) old_call_upgrade_script = getattr(tool, '_call_upgrade_script') monkeypatch.setattr(tool, '_call_upgrade_script', mock_call_upgrade_script) def checkMigration(): with open(os.path.join(tdir, migrationFile)) as f: assert f.read() == migrationText monkeypatch.setattr( NodeControlUtil, 'get_latest_pkg_version', lambda *x, **y: DebianVersion( version, upstream_cls=src_version_cls()) ) nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform, config=tconf) try: sendUpgradeMessage(version) looper.run(eventually(checkMigration)) finally: nct.stop()
def test_upgrade_does_not_get_into_loop(looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet, monkeypatch): new_version = bumpedVersion() upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = new_version # An upgrade scheduled, it should pass ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually(checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # here we make nodes think they have upgraded successfully monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version) check_no_loop(nodeSet, UpgradeLog.SUCCEEDED)
def testNodeControlRestoresFromBackups(monkeypatch, tdir, looper, tconf): version = bumpedVersion() stdout = 'teststdout' backupWasRestored = m.Value('b', False) testFile = 'testFile' original_text = '1' new_text = '2' def testRestoreBackup(tool, src_ver: str): tool._restore_from_backup_test(src_ver) backupWasRestored.value = True def mockMigrate(tool, *args): monkeypatch.setattr(tool, '_do_migration', lambda *args: None) with open(os.path.join(tool.indy_dir, testFile), 'w') as f: f.write(new_text) raise Exception('test') def transform(tool): nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout) tool._restore_from_backup_test = tool._restore_from_backup monkeypatch.setattr(tool, '_do_migration', functools.partial(mockMigrate, tool)) monkeypatch.setattr(tool, '_restore_from_backup', functools.partial(testRestoreBackup, tool)) def checkBackupRestored(tool): assert backupWasRestored.value monkeypatch.setattr( NodeControlUtil, 'get_latest_pkg_version', lambda *x, **y: DebianVersion(version, upstream_cls=src_version_cls())) nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform) try: with open(os.path.join(nct.tool.indy_dir, testFile), 'w') as f: f.write(original_text) sendUpgradeMessage(version) looper.run(eventually(checkBackupRestored, nct.tool)) with open(os.path.join(nct.tool.indy_dir, testFile)) as f: assert original_text == f.read() finally: clean_dir(nct.tool.base_dir) nct.stop()
def test_upgrade_does_not_get_into_loop_if_failed(looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet, monkeypatch): new_version = bumpedVersion() upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = new_version # An upgrade scheduled, it should pass ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually(checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # we have not patched indy_node version so nodes think the upgrade had # failed check_no_loop(nodeSet, UpgradeLog.FAILED)
def test_upgrade_does_not_get_into_loop_if_failed(looper, tconf, nodeSet, validUpgrade, sdk_pool_handle, sdk_wallet_trustee, monkeypatch): new_version = bumpedVersion() upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = new_version # An upgrade scheduled, it should pass sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1) looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # we have not patched indy_node version so nodes think the upgrade had # failed check_no_loop(nodeSet, UpgradeLog.FAILED)
def test_upgrade_does_not_get_into_loop(looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet, monkeypatch): new_version = bumpedVersion() upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = new_version # An upgrade scheduled, it should pass ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # here we make nodes think they have upgraded successfully monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version) check_no_loop(nodeSet, UpgradeLog.UPGRADE_SUCCEEDED)
def test_upgrade_does_not_get_into_loop_force_if_failed( looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet, monkeypatch): new_version = bumpedVersion() upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = new_version upgr1[FORCE] = True # An upgrade scheduled, it should pass ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1) looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # we have not patched indy_node version so nodes think the upgrade had # failed check_no_loop(nodeSet, UpgradeLog.UPGRADE_FAILED)
def test_upgrade_does_not_get_into_loop_if_reinstall_and_failed( looper, tconf, nodeSet, validUpgrade, sdk_pool_handle, sdk_wallet_trustee, monkeypatch): new_version = bumpedVersion(validUpgrade['version']) upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = new_version upgr1[REINSTALL] = True clear_aq_stash(nodeSet) # An upgrade scheduled, it should pass sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1) looper.run( eventually(checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # we have not patched indy_node version so nodes think the upgrade had # failed check_no_loop(nodeSet, UpgradeLog.Events.failed)
def test_upgrade_does_not_get_into_loop_force(looper, tconf, nodeSet, validUpgrade, sdk_pool_handle, sdk_wallet_trustee, monkeypatch): new_version = bumpedVersion(validUpgrade['version']) upgr1 = deepcopy(validUpgrade) upgr1[VERSION] = new_version upgr1[FORCE] = True clear_aq_stash(nodeSet) # An upgrade scheduled, it should pass sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1) looper.run( eventually( checkUpgradeScheduled, nodeSet, upgr1[VERSION], retryWait=1, timeout=waits.expectedUpgradeScheduled())) # here we make nodes think they have upgraded successfully monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version) check_no_loop(nodeSet, UpgradeLog.SUCCEEDED)
import pytest from indy_common.constants import FAIL from indy_node.test.upgrade.helper import populate_log_with_upgrade_events, \ bumpedVersion, check_node_sent_acknowledges_upgrade, check_node_do_not_sent_acknowledges_upgrade, \ emulate_restart_pool_for_upgrade from indy_node.server.upgrade_log import UpgradeLog INVALID_VERSION = bumpedVersion() whitelist = ['unable to send message', 'failed upgrade', 'This problem may have external reasons, check syslog for more information'] # TODO: Implement a client in node @pytest.fixture(scope="module") def tdirWithPoolTxns(tdirWithPoolTxns, poolTxnNodeNames, tdir, tconf): # For each node, adding a file with he current version number which makes the node # think that an upgrade has been performed populate_log_with_upgrade_events( poolTxnNodeNames, tdir, tconf, INVALID_VERSION) return tdirWithPoolTxns def test_node_detected_upgrade_failed(nodeSet): ''' Test that each node checks Upgrade Log on startup (after Upgrade restart), and writes FAIL to it because the current version differs from the one in Upgrade log. Upgrade log already created START event (see fixture above emulating real upgrade)
import pytest from indy_common.constants import IN_PROGRESS, FAIL from indy_node.test.upgrade.helper import populate_log_with_upgrade_events, check_node_set_acknowledges_upgrade, \ bumpedVersion from indy_node.server.upgrade_log import UpgradeLog INVALID_VERSION = bumpedVersion() whitelist = ['unable to send message', 'Failed to upgrade node'] # TODO: Implement a client in node @pytest.fixture(scope="module") def tdirWithPoolTxns(tdirWithPoolTxns, poolTxnNodeNames, tconf): # For each node, adding a file with he current version number which makes the node # think that an upgrade has been performed populate_log_with_upgrade_events(tdirWithPoolTxns, poolTxnNodeNames, tconf, INVALID_VERSION) return tdirWithPoolTxns def test_node_handles_unsuccessful_upgrade(looper, nodeSet, nodeIds): check_node_set_acknowledges_upgrade(looper, nodeSet, nodeIds, [IN_PROGRESS, FAIL], INVALID_VERSION) for node in nodeSet: assert node.upgrader.scheduledUpgrade is None assert node.upgrader.lastUpgradeEventInfo[ 0] == UpgradeLog.UPGRADE_FAILED