def test_generated_cmd_hold_packages(monkeypatch, catch_generated_commands): packages = ['package1', 'package2'] monkeypatch.setattr(shutil, 'which', lambda *_: 'path') NodeControlUtil.hold_packages(packages) assert len(generated_commands) == 1 assert generated_commands[0] == "apt-mark hold {}".format( ' '.join(packages))
def test_generated_cmd_get_info_from_package_manager(catch_generated_commands): packages = ['package1', 'package2'] # TODO not an API for now NodeControlUtil._get_info_from_package_manager(*packages) assert len(generated_commands) == 1 assert generated_commands[0] == "apt-cache show {}".format( " ".join(packages))
def test_get_latest_pkg_version_invalid_args(): pkg_name = 'any_package' with pytest.raises(TypeError) as excinfo: NodeControlUtil.get_latest_pkg_version( pkg_name, upstream=DigitDotVersion('1.2.3'), update_cache=False) assert ("should be instance of {}".format(src_version_cls(pkg_name)) in str(excinfo.value))
def test_deps_levels(patch_pkg_mgr, fltr_hld, res_dep): deps_list = NodeControlUtil.get_deps_tree_filtered('{}={}'.format(*pkg_a), filter_list=fltr_hld) flat_deps = [] NodeControlUtil.dep_tree_traverse(deps_list, flat_deps) assert len(flat_deps) == len(res_dep) for d in res_dep: assert "{}={}".format(*d) in flat_deps
def _upgrade( self, new_src_ver: SourceVersion, pkg_name: str, migrate=True, rollback=True ): curr_pkg_ver, _ = NodeControlUtil.curr_pkg_info(pkg_name) if not curr_pkg_ver: logger.error("package {} is not found".format(pkg_name)) return logger.info( "Current version of package '{}' is '{}'" .format(pkg_name, curr_pkg_ver) ) new_pkg_ver = NodeControlUtil.get_latest_pkg_version( pkg_name, upstream=new_src_ver) if not new_pkg_ver: logger.error( "Upgrade from {} to upstream version {} failed: package {} for" " upstream version is not found" .format(curr_pkg_ver, new_src_ver, pkg_name) ) return self._do_upgrade(pkg_name, curr_pkg_ver, new_pkg_ver, rollback)
def _ext_init(self): NodeControlUtil.update_package_cache() self.ext_ver, ext_deps = self._ext_info() self.deps = ext_deps + self.deps holds = set([self.upgrade_entry] + ext_deps + self.packages_to_hold.strip(" ").split(" ")) self.packages_to_hold = ' '.join(list(holds))
def _get_deps_list(self, package): logger.info('Getting dependencies for {}'.format(package)) NodeControlUtil.update_package_cache() dep_tree = NodeControlUtil.get_deps_tree(package, self.deps) ret = [] NodeControlUtil.dep_tree_traverse(dep_tree, ret) return " ".join(ret)
def _hold_packages(self): if shutil.which("apt-mark"): packages_to_hold = ' '.join(self.config.PACKAGES_TO_HOLD + self.hold_ext) cmd = compose_cmd(['apt-mark', 'hold', packages_to_hold]) NodeControlUtil.run_shell_script(cmd) logger.info('Successfully put {} packages on hold'.format(packages_to_hold)) else: logger.info('Skipping packages holding')
def test_deps_levels(patch_pkg_mgr, fltr_hld, res_dep): deps_list = NodeControlUtil.get_deps_tree_filtered('{}={}'.format(*pkg_a), hold_list=fltr_hld, deps_map={}) flat_deps = [] NodeControlUtil.dep_tree_traverse(deps_list, flat_deps) assert len(deps_list) == len(res_dep) for d in res_dep: assert "=".join(list(d)) in deps_list
def get_dependencies(self, pkg_name, version): base_deps = [APP_NAME, "indy-plenum"] if pkg_name == APP_NAME: return base_deps deps = [] NodeControlUtil.dep_tree_traverse( NodeControlUtil.get_deps_tree("{}={}".format(pkg_name, version), base_deps), deps) return deps
def _call_upgrade_script(self, pkg_name, version): logger.info('Upgrading indy node to version {}, test_mode {}'.format(version, int(self.test_mode))) deps = self._get_deps_list('{}={}'.format(pkg_name, version)) deps = '"{}"'.format(deps) cmd_file = 'upgrade_indy_node' if self.test_mode: cmd_file = 'upgrade_indy_node_test' cmd = compose_cmd([cmd_file, deps]) NodeControlUtil.run_shell_script(cmd, timeout=self.timeout)
def _call_upgrade_script(self, pkg_name: str, pkg_ver: PackageVersion): logger.info("Upgrading {} to package version {}, test_mode {}".format( pkg_name, pkg_ver, int(self.test_mode))) deps = self._get_deps_list('{}={}'.format(pkg_name, pkg_ver)) deps = '"{}"'.format(deps) cmd_file = 'upgrade_indy_node' if self.test_mode: cmd_file = 'upgrade_indy_node_test' cmd = compose_cmd([cmd_file, deps]) NodeControlUtil.run_shell_script(cmd, timeout=self.timeout)
def _get_deps_list(self, package): # We assume, that package looks like 'package_name=1.2.3' logger.info('Getting dependencies for {}'.format(package)) NodeControlUtil.update_package_cache() holded = NodeControlUtil.get_sys_holds() if not holded: return package app_holded = list(set(self.config.PACKAGES_TO_HOLD + self.hold_ext + holded)) deps_list = NodeControlUtil.get_deps_tree_filtered(package, hold_list=app_holded, deps_map={}) # we need to make sure, that all hold package should be presented # in result list if package not in deps_list: deps_list.append(package) return " ".join(deps_list)
def test_generated_cmd_get_latest_pkg_version(catch_generated_commands): pkg_name = 'some_package' NodeControlUtil.get_latest_pkg_version(pkg_name) assert len(generated_commands) == 2 assert generated_commands[0] == "apt update" assert generated_commands[1] == ( "apt-cache show {} | grep -E '^Version: '".format(pkg_name)) generated_commands[:] = [] upstream = src_version_cls(pkg_name)('1.2.3') NodeControlUtil.get_latest_pkg_version(pkg_name, upstream=upstream, update_cache=False) assert len(generated_commands) == 1 assert generated_commands[0] == ( "apt-cache show {} | grep -E '^Version: '".format(pkg_name))
def _call_restart_node_script(self): logger.info('Restarting indy') cmd = compose_cmd(['restart_indy_node']) ret = NodeControlUtil.run_shell_script(cmd, self.timeout) if ret.returncode != 0: raise Exception('restart failed: script returned {}'.format( ret.returncode))
def getVersion(pkg: str = None): if pkg and pkg != APP_NAME: ver, _ = NodeControlUtil.curr_pkt_info(pkg) return ver from indy_node.__metadata__ import __version__ return __version__
def get_src_version(pkg_name: str = APP_NAME, nocache: bool = False) -> SourceVersion: if pkg_name == APP_NAME and not nocache: from indy_node.__metadata__ import __version__ return src_version_cls(APP_NAME)(__version__) curr_pkg_ver, _ = NodeControlUtil.curr_pkg_info(pkg_name) return curr_pkg_ver.upstream if curr_pkg_ver else None
def test_get_latest_pkg_version(monkeypatch, output, expected): def _f(command, *args, **kwargs): if not output: raise ShellError(1, command) else: return output monkeypatch.setattr(NodeControlUtil, 'run_shell_script_extended', _f) assert expected == NodeControlUtil.get_latest_pkg_version( 'any_package', update_cache=False)
def _hold_packages(self): if shutil.which("apt-mark"): cmd = compose_cmd(['apt-mark', 'hold', self.packages_to_hold]) ret = NodeControlUtil.run_shell_command(cmd, TIMEOUT) if ret.returncode != 0: raise Exception('cannot mark {} packages for hold ' 'since {} returned {}'.format( self.packages_to_hold, cmd, ret.returncode)) logger.info('Successfully put {} packages on hold'.format( self.packages_to_hold)) else: logger.info('Skipping packages holding')
def _hold_packages(self): if shutil.which("apt-mark"): packages_to_hold = '{} {}'.format( ' '.join(self.config.PACKAGES_TO_HOLD), self.hold_ext) cmd = compose_cmd(['apt-mark', 'hold', packages_to_hold]) ret = NodeControlUtil.run_shell_command(cmd, TIMEOUT) if ret.returncode != 0: raise Exception( 'cannot mark {} packages for hold since {} returned {}'. format(packages_to_hold, cmd, ret.returncode)) logger.info('Successfully put {} packages on hold'.format( packages_to_hold)) else: logger.info('Skipping packages holding')
def start(self): NodeControlUtil.hold_packages(self.config.PACKAGES_TO_HOLD + self.hold_ext) # Sockets from which we expect to read readers = [self.server] # Sockets to which we expect to write writers = [] errs = [] while readers: # Wait for at least one of the sockets to be ready for processing logger.debug('Waiting for the next event') readable, writable, exceptional = select.select( readers, writers, errs) for s in readable: if s is self.server: # A "readable" server socket is ready to accept a # connection connection, client_address = s.accept() logger.debug('New connection from {} on fd {}' .format(client_address, connection.fileno())) connection.setblocking(0) readers.append(connection) else: data = s.recv(8192) if data: logger.debug( 'Received "{}" from {} on fd {}' .format(data, s.getpeername(), s.fileno())) self._process_data(data) else: logger.debug('Closing socket with fd {}' .format(s.fileno())) readers.remove(s) s.close()
def test_get_latest_pkg_version(monkeypatch, pkg_name, upstream, output, expected): def _f(command, *args, **kwargs): if not output: raise ShellError(1, command) else: return output expected = None if expected is None else src_version_cls(pkg_name)( expected) monkeypatch.setattr(NodeControlUtil, 'run_shell_script_extended', _f) res = NodeControlUtil.get_latest_pkg_version(pkg_name, upstream, update_cache=False) assert expected == res if expected is None else res.upstream
def _call_upgrade_script(self, pkg_name, version): logger.info('Upgrading indy node to version {}, test_mode {}'.format( version, int(self.test_mode))) deps = self._get_deps_list('{}={}'.format(pkg_name, version)) deps = '"{}"'.format(deps) cmd_file = 'upgrade_indy_node' if self.test_mode: cmd_file = 'upgrade_indy_node_test' cmd = compose_cmd([cmd_file, deps]) ret = NodeControlUtil.run_shell_script(cmd, self.timeout) if ret.returncode != 0: raise Exception('upgrade script failed, exit code is {}'.format( ret.returncode))
def test_curr_pkg_info(monkeypatch): output = 'Version: 1.2.3\nDepends: aaa (= 1.2.4), bbb (>= 1.2.5), ccc, aaa' expected_deps = ['aaa=1.2.4', 'bbb=1.2.5', 'ccc'] monkeypatch.setattr(NodeControlUtil, 'run_shell_command', lambda *_: output) for pkg_name in [APP_NAME, 'any_package']: upstream_cls = src_version_cls(pkg_name) expected_version = DebianVersion('1.2.3', upstream_cls=upstream_cls) pkg_info = NodeControlUtil.curr_pkg_info(pkg_name) assert expected_version == pkg_info[0] assert isinstance(expected_version, type(pkg_info[0])) assert isinstance(expected_version.upstream, type(pkg_info[0].upstream)) assert expected_deps == pkg_info[1]
def _upgrade(self, new_version, pkg_name, migrate=True, rollback=True): current_version, _ = NodeControlUtil.curr_pkt_info(pkg_name) try: from indy_node.server.upgrader import Upgrader node_cur_version = Upgrader.getVersion() logger.info('Trying to upgrade from {}={} to {}'.format(pkg_name, current_version, new_version)) self._call_upgrade_script(pkg_name, new_version) if migrate: node_new_version = Upgrader.getVersion() self._do_migration(node_cur_version, node_new_version) self._call_restart_node_script() except Exception as ex: self._declare_upgrade_failed(from_version=current_version, to_version=new_version, reason=str(ex)) logger.error("Trying to rollback to the previous version {}" .format(ex, current_version)) if rollback: self._upgrade(current_version, pkg_name, rollback=False)
def didLastExecutedUpgradeSucceeded(self) -> bool: """ Checks last record in upgrade log to find out whether it is about scheduling upgrade. If so - checks whether current version is equals to the one in that record :returns: upgrade execution result """ lastEventInfo = self.lastActionEventInfo if lastEventInfo: ev_data = lastEventInfo.data currentPkgVersion = NodeControlUtil.curr_pkg_info( ev_data.pkg_name)[0] if currentPkgVersion: return currentPkgVersion.upstream == ev_data.version else: logger.warning("{} failed to get information about package {} " "scheduled for last upgrade".format( self, ev_data.pkg_name)) return False
def check_upgrade_possible(pkg_name: str, target_ver: str, reinstall: bool = False): version_cls = src_version_cls(pkg_name) try: target_ver = version_cls(target_ver) except InvalidVersionError: return ("invalid target version {} for version class {}: ".format( target_ver, version_cls)) # get current installed package version of pkg_name curr_pkg_ver, cur_deps = NodeControlUtil.curr_pkg_info(pkg_name) if not curr_pkg_ver: return ("package {} is not installed and cannot be upgraded". format(pkg_name)) # TODO weak check if APP_NAME not in pkg_name and all( [APP_NAME not in d for d in cur_deps]): return "Package {} doesn't belong to pool".format(pkg_name) # compare whether it makes sense to try (target >= current, = for reinstall) if not Upgrader.is_version_upgradable(curr_pkg_ver.upstream, target_ver, reinstall): return "Version {} is not upgradable".format(target_ver) # get the most recent version of the package for provided version # TODO request to NodeControlTool since Node likely runs under user # which doesn't have rights to update list of system packages available # target_pkg_ver = NodeControlUtil.get_latest_pkg_version( # pkg_name, upstream=target_ver) # if not target_pkg_ver: # return ("package {} for target version {} is not found" # .format(pkg_name, target_ver)) return None
def _get_deps_list(self, package): logger.info('Getting dependencies for {}'.format(package)) NodeControlUtil.update_package_cache() dep_tree = NodeControlUtil.get_deps_tree(package) ret = [] NodeControlUtil.dep_tree_traverse(dep_tree, ret) # Filter deps according to system hold list # in case of hold empty return only package holded = NodeControlUtil.get_sys_holds() if not holded: return package else: ret_list = [] for rl in ret: name = rl.split("=", maxsplit=1)[0] if name in holded: ret_list.append(rl) if package not in ret_list: ret_list.append(package) return " ".join(ret_list)
def curr_pkt_info(self, pkg_name): if pkg_name == APP_NAME: return Upgrader.getVersion(), [APP_NAME] return NodeControlUtil.curr_pkt_info(pkg_name)
def _call_upgrade_node_script(self): logger.info('Upgrading indy') cmd = compose_cmd(['/opt/controller/upgrade_indy']) NodeControlUtil.run_shell_script(cmd, timeout=self.timeout)