def test_node_control_tool_processes_invalid_version(monkeypatch, tdir, tconf, pkg_name, version): def patch(tool): monkeypatch.setattr(tool, '_listen', lambda *x, **y: None) monkeypatch.setattr(tool, '_upgrade', lambda *x, **y: None) monkeypatch.setattr(tool, '_restart', lambda *x, **y: None) tool = NodeControlToolPatched(patch, backup_dir=tdir, backup_target=tdir) with pytest.raises(InvalidVersionError): src_version_cls(pkg_name)(version) with pytest.raises(BlowUp, match='invalid version 1.2.3.4.5'): tool._process_data(composeUpgradeMessage(version, pkg_name))
def testNodeControlCreatesBackups(monkeypatch, tdir, looper, tconf): version = bumpedVersion() stdout = 'teststdout' curr_src_ver = Upgrader.get_src_version() def transform(tool): nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout) monkeypatch.setattr(tool, '_remove_old_backups', lambda *x: None) def checkBackup(tool): assert os.path.isfile('{}.{}'.format( tool._backup_name(curr_src_ver.release), tool.backup_format)) monkeypatch.setattr( NodeControlUtil, 'get_latest_pkg_version', lambda *x, **y: DebianVersion( version, upstream_cls=src_version_cls()) ) nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform, config=tconf) try: sendUpgradeMessage(version) looper.run(eventually(checkBackup, nct.tool)) finally: clean_dir(nct.tool.base_dir) nct.stop()
def test_get_latest_pkg_version_invalid_args(): pkg_name = 'any_package' with pytest.raises(TypeError) as excinfo: NodeControlUtil.get_latest_pkg_version( pkg_name, upstream=DigitDotVersion('1.2.3'), update_cache=False) assert ("should be instance of {}".format(src_version_cls(pkg_name)) in str(excinfo.value))
def patch_packet_mgr_output(monkeypatch, pkg_name, pkg_version, new_version): if pkg_name != APP_NAME: node_package = (APP_NAME, '0.0.1') EXT_TOP_PKT_DEPS = [("aa", "1.1.1"), ("bb", "2.2.2")] PACKAGE_MNG_EXT_PTK_OUTPUT = ( "Package: {}\nVersion: {}\n" "Depends: {}, {} (= {}), {} (= {})\n".format( pkg_name, pkg_version, APP_NAME, *EXT_TOP_PKT_DEPS[0], *EXT_TOP_PKT_DEPS[1])) top_level_package = (pkg_name, pkg_version) plenum_package = ('indy-plenum', '0.0.3') top_level_package_with_version = '{}={}'.format(*top_level_package) top_level_package_dep1_with_version = '{}={}'.format( *EXT_TOP_PKT_DEPS[0]) top_level_package_dep2_with_version = '{}={}'.format( *EXT_TOP_PKT_DEPS[1]) node_package_with_version = '{}={}'.format(*node_package) plenum_package_with_version = '{}={}'.format(*plenum_package) mock_info = { top_level_package_with_version: "{}{} (= {}) {} (= {}), {} (= {})".format(randomText(100), *node_package, *EXT_TOP_PKT_DEPS[0], *EXT_TOP_PKT_DEPS[1]), node_package_with_version: '{}{} (= {}){}{}'.format(randomText(100), *plenum_package, randomText(100), randomText(100)), plenum_package_with_version: '{}'.format(randomText(100)), top_level_package_dep1_with_version: '{}{} (= {})'.format(randomText(100), *plenum_package), top_level_package_dep2_with_version: '{}{} (= {})'.format(randomText(100), *node_package) } def mock_get_info_from_package_manager(package): if package.startswith(pkg_name): return mock_info.get(top_level_package_with_version, "") return mock_info.get(package, "") #monkeypatch.setattr(NodeControlUtil, '_get_info_from_package_manager', # lambda x: mock_get_info_from_package_manager(x)) monkeypatch.setattr(NodeControlUtil, '_get_curr_info', lambda *x: PACKAGE_MNG_EXT_PTK_OUTPUT) else: #monkeypatch.setattr( # NodeControlUtil, '_get_info_from_package_manager', # lambda package: "Package: {}\nVersion: {}\n".format(APP_NAME, pkg_version) if package == APP_NAME else "" #) monkeypatch.setattr( NodeControlUtil, '_get_curr_info', lambda *x: "Package: {}\nVersion: {}\n".format( APP_NAME, pkg_version)) monkeypatch.setattr(NodeControlUtil, 'update_package_cache', lambda *x: None) monkeypatch.setattr( NodeControlUtil, 'get_latest_pkg_version', lambda *x, **y: DebianVersion(new_version, upstream_cls=src_version_cls(pkg_name)))
def get_src_version(pkg_name: str = APP_NAME, nocache: bool = False) -> SourceVersion: if pkg_name == APP_NAME and not nocache: from indy_node.__metadata__ import __version__ return src_version_cls(APP_NAME)(__version__) curr_pkg_ver, _ = NodeControlUtil.curr_pkg_info(pkg_name) return curr_pkg_ver.upstream if curr_pkg_ver else None
def test_get_latest_pkg_version(monkeypatch, pkg_name, upstream, output, expected): def _f(command, *args, **kwargs): if not output: raise ShellError(1, command) else: return output if upstream is not None: upstream = src_version_cls(pkg_name)(upstream) expected = None if expected is None else src_version_cls(pkg_name)( expected) monkeypatch.setattr(NodeControlUtil, 'run_shell_script_extended', _f) res = NodeControlUtil.get_latest_pkg_version(pkg_name, upstream, update_cache=False) assert expected == res if expected is None else res.upstream
def get_latest_pkg_version( cls, pkg_name: str, upstream: SourceVersion = None, update_cache: bool = True) -> PackageVersion: upstream_cls = src_version_cls(pkg_name) if upstream and not isinstance(upstream, upstream_cls): raise TypeError( "'upstream' should be instance of {}, got {}" .format(upstream_cls, type(upstream)) ) if update_cache: cls.update_package_cache() try: cmd = compose_cmd( ['apt-cache', 'show', pkg_name, '|', 'grep', '-E', "'^Version: '"] ) output = cls.run_shell_script_extended(cmd).strip() except ShellError as exc: # will fail if either package not found or grep returns nothing # the latter is unexpected and treated as no-data as well logger.info( "no data for package '{}' found".format(pkg_name) ) else: if output: versions = [] for v in output.split('\n'): try: dv = DebianVersion(v.split()[1], upstream_cls=upstream_cls) except InvalidVersionError as exc: logger.warning( "ignoring invalid version from output {} for upstream class {}: {}" .format(v.split()[1], upstream_cls, exc) ) else: if not upstream or (dv.upstream == upstream): versions.append(dv) try: return sorted(versions)[-1] except IndexError: pass except ShellError: logger.warning( "version comparison failed unexpectedly for versions: {}" .format(versions) ) return None
def test_versions_comparison(lower_version, higher_version): assert Upgrader.compareVersions(higher_version, lower_version) == 1 assert Upgrader.compareVersions(lower_version, higher_version) == -1 assert Upgrader.compareVersions(higher_version, higher_version) == 0 version_cls = src_version_cls(APP_NAME) lower_version = version_cls(lower_version) higher_version = version_cls(higher_version) assert not Upgrader.is_version_upgradable(higher_version, higher_version) assert Upgrader.is_version_upgradable(higher_version, higher_version, reinstall=True) assert Upgrader.is_version_upgradable(lower_version, higher_version) assert not Upgrader.is_version_upgradable(higher_version, lower_version)
def _process_data(self, data): import json try: command = json.loads(data.decode("utf-8")) logger.debug("Decoded ", command) if command[MESSAGE_TYPE] == UPGRADE_MESSAGE: new_src_ver = command['version'] pkg_name = command['pkg_name'] self._upgrade(src_version_cls(pkg_name)(new_src_ver), pkg_name) elif command[MESSAGE_TYPE] == RESTART_MESSAGE: self._restart() except json.decoder.JSONDecodeError as e: logger.error("JSON decoding failed: {}".format(e)) except Exception as e: logger.error("Unexpected error in process_data {}".format(e))
def __init__(self, when: Union[datetime, str], version: Union[SourceVersion, str], upgrade_id: str, pkg_name: str): super().__init__(when) if isinstance(version, str): version = src_version_cls(pkg_name)(version) if not isinstance(version, SourceVersion): raise TypeError( "'version' should be 'SourceVersion' or 'str', got: {}".format( type(version))) self.version = version self.upgrade_id = upgrade_id self.pkg_name = pkg_name
def test_generated_cmd_get_latest_pkg_version(catch_generated_commands): pkg_name = 'some_package' NodeControlUtil.get_latest_pkg_version(pkg_name) assert len(generated_commands) == 2 assert generated_commands[0] == "apt update" assert generated_commands[1] == ( "apt-cache show {} | grep -E '^Version: '".format(pkg_name)) generated_commands[:] = [] upstream = src_version_cls(pkg_name)('1.2.3') NodeControlUtil.get_latest_pkg_version(pkg_name, upstream=upstream, update_cache=False) assert len(generated_commands) == 1 assert generated_commands[0] == ( "apt-cache show {} | grep -E '^Version: '".format(pkg_name))
def test_curr_pkg_info(monkeypatch): output = 'Version: 1.2.3\nDepends: aaa (= 1.2.4), bbb (>= 1.2.5), ccc, aaa' expected_deps = ['aaa=1.2.4', 'bbb=1.2.5', 'ccc'] monkeypatch.setattr(NodeControlUtil, 'run_shell_command', lambda *_: output) for pkg_name in [APP_NAME, 'any_package']: upstream_cls = src_version_cls(pkg_name) expected_version = DebianVersion('1.2.3', upstream_cls=upstream_cls) pkg_info = NodeControlUtil.curr_pkg_info(pkg_name) assert expected_version == pkg_info[0] assert isinstance(expected_version, type(pkg_info[0])) assert isinstance(expected_version.upstream, type(pkg_info[0].upstream)) assert expected_deps == pkg_info[1]
def testNodeControlRemovesBackups(monkeypatch, tdir, looper, tconf): version = bumpedVersion() stdout = 'teststdout' curr_src_ver = Upgrader.get_src_version() backupsWereRemoved = m.Value('b', False) def testRemoveOldBackups(tool): assert len(tool._get_backups()) == (tool.backup_num + 1) #looper = Looper(debug=True) tool._remove_old_backups_test() backupsWereRemoved.value = True def transform(tool): nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout) tool._remove_old_backups_test = tool._remove_old_backups monkeypatch.setattr(tool, '_remove_old_backups', functools.partial(testRemoveOldBackups, tool)) def checkOldBackupsRemoved(): assert backupsWereRemoved.value def check_backups_files_exists(): assert os.path.exists('{}.{}'.format( nct.tool._backup_name(curr_src_ver.release), nct.tool.backup_format)) monkeypatch.setattr( NodeControlUtil, 'get_latest_pkg_version', lambda *x, **y: DebianVersion(version, upstream_cls=src_version_cls())) nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform) try: assert len(nct.tool._get_backups()) == 0 for i in range(nct.tool.backup_num): file = os.path.join(nct.tool.base_dir, '{}{}'.format(nct.tool.backup_name_prefix, i)) with open(file, 'w') as f: f.write('random') assert len(nct.tool._get_backups()) == nct.tool.backup_num sendUpgradeMessage(version) looper.run(eventually(checkOldBackupsRemoved)) looper.run(eventually(check_backups_files_exists)) assert len(nct.tool._get_backups()) == nct.tool.backup_num finally: clean_dir(nct.tool.base_dir) nct.stop()
def testNodeControlPerformsMigrations(monkeypatch, tdir, looper, tconf): version = bumpedVersion() stdout = 'teststdout' migrationFile = 'migrationProof' migrationText = "{} {}".format(releaseVersion(), version) old_call_upgrade_script = None def mock_call_upgrade_script(*args, **kwargs): old_call_upgrade_script(*args, **kwargs) monkeypatch.setattr( NodeControlUtil, '_get_curr_info', lambda *x, **y: ( "Package: {}\nVersion: {}\n".format(APP_NAME, version) ) ) def mock_migrate(curr_src_ver: str, new_src_ver: str): with open(os.path.join(tdir, migrationFile), 'w') as f: f.write("{} {}".format(curr_src_ver, new_src_ver)) def transform(tool): nonlocal old_call_upgrade_script nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout) monkeypatch.setattr(tool, '_do_migration', mock_migrate) old_call_upgrade_script = getattr(tool, '_call_upgrade_script') monkeypatch.setattr(tool, '_call_upgrade_script', mock_call_upgrade_script) def checkMigration(): with open(os.path.join(tdir, migrationFile)) as f: assert f.read() == migrationText monkeypatch.setattr( NodeControlUtil, 'get_latest_pkg_version', lambda *x, **y: DebianVersion( version, upstream_cls=src_version_cls()) ) nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform, config=tconf) try: sendUpgradeMessage(version) looper.run(eventually(checkMigration)) finally: nct.stop()
def get_latest_pkg_version( cls, pkg_name: str, upstream: SourceVersion = None, update_cache: bool = True) -> PackageVersion: upstream_cls = src_version_cls(pkg_name) if upstream and not isinstance(upstream, upstream_cls): raise TypeError( "'upstream' should be instance of {}, got {}" .format(upstream_cls, type(upstream)) ) if update_cache: cls.update_package_cache() regex = "'^Version: ([0-9]+:)?{}(-|$)'".format( upstream.full if upstream else '.*') try: cmd = compose_cmd( ['apt-cache', 'show', pkg_name, '|', 'grep', '-E', regex]) output = cls.run_shell_script_extended(cmd).strip() except ShellError as exc: # will fail if either package not found or grep returns nothing # the latter is unexpected and treated as no-data as well logger.info( "no-data for package '{}' with upstream version '{}' found" .format(pkg_name, upstream)) else: if output: versions = [ DebianVersion(v.split()[1], upstream_cls=upstream_cls) for v in output.split('\n') ] try: return sorted(versions)[-1] except ShellError: logger.warning( "version comparison failed unexpectedly for versions: {}" .format(versions) ) return None
def testNodeControlRestoresFromBackups(monkeypatch, tdir, looper, tconf): version = bumpedVersion() stdout = 'teststdout' backupWasRestored = m.Value('b', False) testFile = 'testFile' original_text = '1' new_text = '2' def testRestoreBackup(tool, src_ver: str): tool._restore_from_backup_test(src_ver) backupWasRestored.value = True def mockMigrate(tool, *args): monkeypatch.setattr(tool, '_do_migration', lambda *args: None) with open(os.path.join(tool.indy_dir, testFile), 'w') as f: f.write(new_text) raise Exception('test') def transform(tool): nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout) tool._restore_from_backup_test = tool._restore_from_backup monkeypatch.setattr(tool, '_do_migration', functools.partial(mockMigrate, tool)) monkeypatch.setattr(tool, '_restore_from_backup', functools.partial(testRestoreBackup, tool)) def checkBackupRestored(tool): assert backupWasRestored.value monkeypatch.setattr( NodeControlUtil, 'get_latest_pkg_version', lambda *x, **y: DebianVersion(version, upstream_cls=src_version_cls())) nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform) try: with open(os.path.join(nct.tool.indy_dir, testFile), 'w') as f: f.write(original_text) sendUpgradeMessage(version) looper.run(eventually(checkBackupRestored, nct.tool)) with open(os.path.join(nct.tool.indy_dir, testFile)) as f: assert original_text == f.read() finally: clean_dir(nct.tool.base_dir) nct.stop()
def check_upgrade_possible(pkg_name: str, target_ver: str, reinstall: bool = False): version_cls = src_version_cls(pkg_name) try: target_ver = version_cls(target_ver) except InvalidVersionError: return ("invalid target version {} for version class {}: ".format( target_ver, version_cls)) # get current installed package version of pkg_name curr_pkg_ver, cur_deps = NodeControlUtil.curr_pkg_info(pkg_name) if not curr_pkg_ver: return ("package {} is not installed and cannot be upgraded". format(pkg_name)) # TODO weak check if APP_NAME not in pkg_name and all( [APP_NAME not in d for d in cur_deps]): return "Package {} doesn't belong to pool".format(pkg_name) # compare whether it makes sense to try (target >= current, = for reinstall) if not Upgrader.is_version_upgradable(curr_pkg_ver.upstream, target_ver, reinstall): return "Version {} is not upgradable".format(target_ver) # get the most recent version of the package for provided version # TODO request to NodeControlTool since Node likely runs under user # which doesn't have rights to update list of system packages available # target_pkg_ver = NodeControlUtil.get_latest_pkg_version( # pkg_name, upstream=target_ver) # if not target_pkg_ver: # return ("package {} for target version {} is not found" # .format(pkg_name, target_ver)) return None
def _process_data(self, data): import json try: command = json.loads(data.decode("utf-8")) logger.debug("Decoded ", command) if command[MESSAGE_TYPE] == UPGRADE_MESSAGE: pkg_name = command['pkg_name'] upstream_cls = src_version_cls(pkg_name) try: new_src_ver = upstream_cls(command['version']) except InvalidVersionError as exc: logger.error( "invalid version {} for package {} with upstream class {}: {}" .format(command['version'], pkg_name, upstream_cls, exc)) else: self._upgrade(new_src_ver, pkg_name) elif command[MESSAGE_TYPE] == RESTART_MESSAGE: self._restart() except json.decoder.JSONDecodeError as e: logger.error("JSON decoding failed: {}".format(e)) except Exception as e: logger.error("Unexpected error in _process_data {}".format(e))
def handleUpgradeTxn(self, txn) -> None: """ Handles transaction of type POOL_UPGRADE Can schedule or cancel upgrade to a newer version at specified time :param txn: """ FINALIZING_EVENT_TYPES = [ UpgradeLog.Events.succeeded, UpgradeLog.Events.failed ] if get_type(txn) != POOL_UPGRADE: return logger.info("Node '{}' handles upgrade txn {}".format( self.nodeName, txn)) txn_data = get_payload_data(txn) action = txn_data[ACTION] version = txn_data[VERSION] justification = txn_data.get(JUSTIFICATION) pkg_name = txn_data.get(PACKAGE, self.config.UPGRADE_ENTRY) upgrade_id = self.get_action_id(txn) # TODO test try: version = src_version_cls(pkg_name)(version) except InvalidVersionError as exc: logger.warning( "{} can't handle upgrade txn with version {} for package {}: {}" .format(self, version, pkg_name, exc)) return if action == START: # forced txn could have partial schedule list if self.nodeId not in txn_data[SCHEDULE]: logger.info("Node '{}' disregards upgrade txn {}".format( self.nodeName, txn)) return last_event = self.lastActionEventInfo if last_event: if last_event.data.upgrade_id == upgrade_id and last_event.ev_type in FINALIZING_EVENT_TYPES: logger.info( "Node '{}' has already performed an upgrade with upgrade_id {}. " "Last recorded event is {}".format( self.nodeName, upgrade_id, last_event.data)) return when = txn_data[SCHEDULE][self.nodeId] failTimeout = txn_data.get(TIMEOUT, self.defaultActionTimeout) if isinstance(when, str): when = dateutil.parser.parse(when) new_ev_data = UpgradeLogData(when, version, upgrade_id, pkg_name) if self.scheduledAction: if self.scheduledAction == new_ev_data: logger.debug( "Node {} already scheduled upgrade to version '{}' ". format(self.nodeName, version)) return else: logger.info( "Node '{}' cancels previous upgrade and schedules a new one to {}" .format(self.nodeName, version)) self._cancelScheduledUpgrade(justification) logger.info("Node '{}' schedules upgrade to {}".format( self.nodeName, version)) self._scheduleUpgrade(new_ev_data, failTimeout) return if action == CANCEL: if self.scheduledAction and self.scheduledAction.version == version: self._cancelScheduledUpgrade(justification) logger.info("Node '{}' cancels upgrade to {}".format( self.nodeName, version)) return logger.error("Got {} transaction with unsupported action {}".format( POOL_UPGRADE, action))
' '.join(packages)) def test_get_latest_pkg_version_invalid_args(): pkg_name = 'any_package' with pytest.raises(TypeError) as excinfo: NodeControlUtil.get_latest_pkg_version( pkg_name, upstream=DigitDotVersion('1.2.3'), update_cache=False) assert ("should be instance of {}".format(src_version_cls(pkg_name)) in str(excinfo.value)) @pytest.mark.parametrize('output,expected', [ ('', None), ('Version: 1.2.3\nVersion: 1.2.4', DebianVersion('1.2.4', upstream_cls=src_version_cls('any_package'))), ], ids=lambda s: s.replace('\n', '_').replace(' ', '_')) def test_get_latest_pkg_version(monkeypatch, output, expected): def _f(command, *args, **kwargs): if not output: raise ShellError(1, command) else: return output monkeypatch.setattr(NodeControlUtil, 'run_shell_script_extended', _f) assert expected == NodeControlUtil.get_latest_pkg_version( 'any_package', update_cache=False) def test_get_latest_pkg_version_for_unknown_package():
def curr_pkg_info(cls, pkg_name: str) -> Tuple[PackageVersion, List]: package_info = cls._get_curr_info(pkg_name) return cls._parse_version_deps_from_pkg_mgr_output( package_info, upstream_cls=src_version_cls(pkg_name))
def compareVersions(verA: str, verB: str) -> int: version_cls = src_version_cls(APP_NAME) return version_cls.cmp(version_cls(verA), version_cls(verB))
def test_get_src_version_for(monkeypatch): assert (Upgrader.get_src_version(some_pkg_name) == src_version_cls( some_pkg_name)('1.2.2'))
def pkg_version(pool_upgrade_request): return DebianVersion('1.1.1', upstream_cls=src_version_cls( pool_upgrade_request.operation[PACKAGE]))
def test_src_version_cls(): assert src_version_cls() == NodeVersion assert src_version_cls(APP_NAME) == NodeVersion assert src_version_cls('some_package') == TopPkgDefVersion