def test_generated_cmd_hold_packages(monkeypatch, catch_generated_commands):
packages = ['package1', 'package2']
monkeypatch.setattr(shutil, 'which', lambda *_: 'path')
NodeControlUtil.hold_packages(packages)
assert len(generated_commands) == 1
assert generated_commands[0] == "apt-mark hold {}".format(
' '.join(packages))
def test_generated_cmd_get_info_from_package_manager(catch_generated_commands):
packages = ['package1', 'package2']
# TODO not an API for now
NodeControlUtil._get_info_from_package_manager(*packages)
assert len(generated_commands) == 1
assert generated_commands[0] == "apt-cache show {}".format(
" ".join(packages))
def test_get_latest_pkg_version_invalid_args():
pkg_name = 'any_package'
with pytest.raises(TypeError) as excinfo:
NodeControlUtil.get_latest_pkg_version(
pkg_name, upstream=DigitDotVersion('1.2.3'), update_cache=False)
assert ("should be instance of {}".format(src_version_cls(pkg_name))
in str(excinfo.value))
def test_deps_levels(patch_pkg_mgr, fltr_hld, res_dep):
deps_list = NodeControlUtil.get_deps_tree_filtered('{}={}'.format(*pkg_a), filter_list=fltr_hld)
flat_deps = []
NodeControlUtil.dep_tree_traverse(deps_list, flat_deps)
assert len(flat_deps) == len(res_dep)
for d in res_dep:
assert "{}={}".format(*d) in flat_deps
def _upgrade(
self,
new_src_ver: SourceVersion,
pkg_name: str,
migrate=True,
rollback=True
):
curr_pkg_ver, _ = NodeControlUtil.curr_pkg_info(pkg_name)
if not curr_pkg_ver:
logger.error("package {} is not found".format(pkg_name))
return
logger.info(
"Current version of package '{}' is '{}'"
.format(pkg_name, curr_pkg_ver)
)
new_pkg_ver = NodeControlUtil.get_latest_pkg_version(
pkg_name, upstream=new_src_ver)
if not new_pkg_ver:
logger.error(
"Upgrade from {} to upstream version {} failed: package {} for"
" upstream version is not found"
.format(curr_pkg_ver, new_src_ver, pkg_name)
)
return
self._do_upgrade(pkg_name, curr_pkg_ver, new_pkg_ver, rollback)
def _ext_init(self):
NodeControlUtil.update_package_cache()
self.ext_ver, ext_deps = self._ext_info()
self.deps = ext_deps + self.deps
holds = set([self.upgrade_entry] + ext_deps +
self.packages_to_hold.strip(" ").split(" "))
self.packages_to_hold = ' '.join(list(holds))
def _get_deps_list(self, package):
logger.info('Getting dependencies for {}'.format(package))
NodeControlUtil.update_package_cache()
dep_tree = NodeControlUtil.get_deps_tree(package, self.deps)
ret = []
NodeControlUtil.dep_tree_traverse(dep_tree, ret)
return " ".join(ret)
def _hold_packages(self):
if shutil.which("apt-mark"):
packages_to_hold = ' '.join(self.config.PACKAGES_TO_HOLD + self.hold_ext)
cmd = compose_cmd(['apt-mark', 'hold', packages_to_hold])
NodeControlUtil.run_shell_script(cmd)
logger.info('Successfully put {} packages on hold'.format(packages_to_hold))
else:
logger.info('Skipping packages holding')
def test_deps_levels(patch_pkg_mgr, fltr_hld, res_dep):
deps_list = NodeControlUtil.get_deps_tree_filtered('{}={}'.format(*pkg_a),
hold_list=fltr_hld,
deps_map={})
flat_deps = []
NodeControlUtil.dep_tree_traverse(deps_list, flat_deps)
assert len(deps_list) == len(res_dep)
for d in res_dep:
assert "=".join(list(d)) in deps_list
def get_dependencies(self, pkg_name, version):
base_deps = [APP_NAME, "indy-plenum"]
if pkg_name == APP_NAME:
return base_deps
deps = []
NodeControlUtil.dep_tree_traverse(
NodeControlUtil.get_deps_tree("{}={}".format(pkg_name, version),
base_deps), deps)
return deps
def _call_upgrade_script(self, pkg_name, version):
logger.info('Upgrading indy node to version {}, test_mode {}'.format(version, int(self.test_mode)))
deps = self._get_deps_list('{}={}'.format(pkg_name, version))
deps = '"{}"'.format(deps)
cmd_file = 'upgrade_indy_node'
if self.test_mode:
cmd_file = 'upgrade_indy_node_test'
cmd = compose_cmd([cmd_file, deps])
NodeControlUtil.run_shell_script(cmd, timeout=self.timeout)
def _call_upgrade_script(self, pkg_name: str, pkg_ver: PackageVersion):
logger.info("Upgrading {} to package version {}, test_mode {}".format(
pkg_name, pkg_ver, int(self.test_mode)))
deps = self._get_deps_list('{}={}'.format(pkg_name, pkg_ver))
deps = '"{}"'.format(deps)
cmd_file = 'upgrade_indy_node'
if self.test_mode:
cmd_file = 'upgrade_indy_node_test'
cmd = compose_cmd([cmd_file, deps])
NodeControlUtil.run_shell_script(cmd, timeout=self.timeout)
def _get_deps_list(self, package):
# We assume, that package looks like 'package_name=1.2.3'
logger.info('Getting dependencies for {}'.format(package))
NodeControlUtil.update_package_cache()
holded = NodeControlUtil.get_sys_holds()
if not holded:
return package
app_holded = list(set(self.config.PACKAGES_TO_HOLD + self.hold_ext + holded))
deps_list = NodeControlUtil.get_deps_tree_filtered(package, hold_list=app_holded, deps_map={})
# we need to make sure, that all hold package should be presented
# in result list
if package not in deps_list:
deps_list.append(package)
return " ".join(deps_list)
def test_generated_cmd_get_latest_pkg_version(catch_generated_commands):
pkg_name = 'some_package'
NodeControlUtil.get_latest_pkg_version(pkg_name)
assert len(generated_commands) == 2
assert generated_commands[0] == "apt update"
assert generated_commands[1] == (
"apt-cache show {} | grep -E '^Version: '".format(pkg_name))
generated_commands[:] = []
upstream = src_version_cls(pkg_name)('1.2.3')
NodeControlUtil.get_latest_pkg_version(pkg_name,
upstream=upstream,
update_cache=False)
assert len(generated_commands) == 1
assert generated_commands[0] == (
"apt-cache show {} | grep -E '^Version: '".format(pkg_name))
def _call_restart_node_script(self):
logger.info('Restarting indy')
cmd = compose_cmd(['restart_indy_node'])
ret = NodeControlUtil.run_shell_script(cmd, self.timeout)
if ret.returncode != 0:
raise Exception('restart failed: script returned {}'.format(
ret.returncode))
def getVersion(pkg: str = None):
if pkg and pkg != APP_NAME:
ver, _ = NodeControlUtil.curr_pkt_info(pkg)
return ver
from indy_node.__metadata__ import __version__
return __version__
def get_src_version(pkg_name: str = APP_NAME,
nocache: bool = False) -> SourceVersion:
if pkg_name == APP_NAME and not nocache:
from indy_node.__metadata__ import __version__
return src_version_cls(APP_NAME)(__version__)
curr_pkg_ver, _ = NodeControlUtil.curr_pkg_info(pkg_name)
return curr_pkg_ver.upstream if curr_pkg_ver else None
def test_get_latest_pkg_version(monkeypatch, output, expected):
def _f(command, *args, **kwargs):
if not output:
raise ShellError(1, command)
else:
return output
monkeypatch.setattr(NodeControlUtil, 'run_shell_script_extended', _f)
assert expected == NodeControlUtil.get_latest_pkg_version(
'any_package', update_cache=False)
def _hold_packages(self):
if shutil.which("apt-mark"):
cmd = compose_cmd(['apt-mark', 'hold', self.packages_to_hold])
ret = NodeControlUtil.run_shell_command(cmd, TIMEOUT)
if ret.returncode != 0:
raise Exception('cannot mark {} packages for hold '
'since {} returned {}'.format(
self.packages_to_hold, cmd,
ret.returncode))
logger.info('Successfully put {} packages on hold'.format(
self.packages_to_hold))
else:
logger.info('Skipping packages holding')
def _hold_packages(self):
if shutil.which("apt-mark"):
packages_to_hold = '{} {}'.format(
' '.join(self.config.PACKAGES_TO_HOLD), self.hold_ext)
cmd = compose_cmd(['apt-mark', 'hold', packages_to_hold])
ret = NodeControlUtil.run_shell_command(cmd, TIMEOUT)
if ret.returncode != 0:
raise Exception(
'cannot mark {} packages for hold since {} returned {}'.
format(packages_to_hold, cmd, ret.returncode))
logger.info('Successfully put {} packages on hold'.format(
packages_to_hold))
else:
logger.info('Skipping packages holding')
def start(self):
NodeControlUtil.hold_packages(self.config.PACKAGES_TO_HOLD + self.hold_ext)
# Sockets from which we expect to read
readers = [self.server]
# Sockets to which we expect to write
writers = []
errs = []
while readers:
# Wait for at least one of the sockets to be ready for processing
logger.debug('Waiting for the next event')
readable, writable, exceptional = select.select(
readers, writers, errs)
for s in readable:
if s is self.server:
# A "readable" server socket is ready to accept a
# connection
connection, client_address = s.accept()
logger.debug('New connection from {} on fd {}'
.format(client_address, connection.fileno()))
connection.setblocking(0)
readers.append(connection)
else:
data = s.recv(8192)
if data:
logger.debug(
'Received "{}" from {} on fd {}'
.format(data, s.getpeername(), s.fileno()))
self._process_data(data)
else:
logger.debug('Closing socket with fd {}'
.format(s.fileno()))
readers.remove(s)
s.close()
def test_get_latest_pkg_version(monkeypatch, pkg_name, upstream, output,
expected):
def _f(command, *args, **kwargs):
if not output:
raise ShellError(1, command)
else:
return output
expected = None if expected is None else src_version_cls(pkg_name)(
expected)
monkeypatch.setattr(NodeControlUtil, 'run_shell_script_extended', _f)
res = NodeControlUtil.get_latest_pkg_version(pkg_name,
upstream,
update_cache=False)
assert expected == res if expected is None else res.upstream
def _call_upgrade_script(self, pkg_name, version):
logger.info('Upgrading indy node to version {}, test_mode {}'.format(
version, int(self.test_mode)))
deps = self._get_deps_list('{}={}'.format(pkg_name, version))
deps = '"{}"'.format(deps)
cmd_file = 'upgrade_indy_node'
if self.test_mode:
cmd_file = 'upgrade_indy_node_test'
cmd = compose_cmd([cmd_file, deps])
ret = NodeControlUtil.run_shell_script(cmd, self.timeout)
if ret.returncode != 0:
raise Exception('upgrade script failed, exit code is {}'.format(
ret.returncode))
def test_curr_pkg_info(monkeypatch):
output = 'Version: 1.2.3\nDepends: aaa (= 1.2.4), bbb (>= 1.2.5), ccc, aaa'
expected_deps = ['aaa=1.2.4', 'bbb=1.2.5', 'ccc']
monkeypatch.setattr(NodeControlUtil, 'run_shell_command',
lambda *_: output)
for pkg_name in [APP_NAME, 'any_package']:
upstream_cls = src_version_cls(pkg_name)
expected_version = DebianVersion('1.2.3', upstream_cls=upstream_cls)
pkg_info = NodeControlUtil.curr_pkg_info(pkg_name)
assert expected_version == pkg_info[0]
assert isinstance(expected_version, type(pkg_info[0]))
assert isinstance(expected_version.upstream,
type(pkg_info[0].upstream))
assert expected_deps == pkg_info[1]
def _upgrade(self, new_version, pkg_name, migrate=True, rollback=True):
current_version, _ = NodeControlUtil.curr_pkt_info(pkg_name)
try:
from indy_node.server.upgrader import Upgrader
node_cur_version = Upgrader.getVersion()
logger.info('Trying to upgrade from {}={} to {}'.format(pkg_name, current_version, new_version))
self._call_upgrade_script(pkg_name, new_version)
if migrate:
node_new_version = Upgrader.getVersion()
self._do_migration(node_cur_version, node_new_version)
self._call_restart_node_script()
except Exception as ex:
self._declare_upgrade_failed(from_version=current_version,
to_version=new_version,
reason=str(ex))
logger.error("Trying to rollback to the previous version {}"
.format(ex, current_version))
if rollback:
self._upgrade(current_version, pkg_name, rollback=False)
def didLastExecutedUpgradeSucceeded(self) -> bool:
"""
Checks last record in upgrade log to find out whether it
is about scheduling upgrade. If so - checks whether current version
is equals to the one in that record
:returns: upgrade execution result
"""
lastEventInfo = self.lastActionEventInfo
if lastEventInfo:
ev_data = lastEventInfo.data
currentPkgVersion = NodeControlUtil.curr_pkg_info(
ev_data.pkg_name)[0]
if currentPkgVersion:
return currentPkgVersion.upstream == ev_data.version
else:
logger.warning("{} failed to get information about package {} "
"scheduled for last upgrade".format(
self, ev_data.pkg_name))
return False
def check_upgrade_possible(pkg_name: str,
target_ver: str,
reinstall: bool = False):
version_cls = src_version_cls(pkg_name)
try:
target_ver = version_cls(target_ver)
except InvalidVersionError:
return ("invalid target version {} for version class {}: ".format(
target_ver, version_cls))
# get current installed package version of pkg_name
curr_pkg_ver, cur_deps = NodeControlUtil.curr_pkg_info(pkg_name)
if not curr_pkg_ver:
return ("package {} is not installed and cannot be upgraded".
format(pkg_name))
# TODO weak check
if APP_NAME not in pkg_name and all(
[APP_NAME not in d for d in cur_deps]):
return "Package {} doesn't belong to pool".format(pkg_name)
# compare whether it makes sense to try (target >= current, = for reinstall)
if not Upgrader.is_version_upgradable(curr_pkg_ver.upstream,
target_ver, reinstall):
return "Version {} is not upgradable".format(target_ver)
# get the most recent version of the package for provided version
# TODO request to NodeControlTool since Node likely runs under user
# which doesn't have rights to update list of system packages available
# target_pkg_ver = NodeControlUtil.get_latest_pkg_version(
# pkg_name, upstream=target_ver)
# if not target_pkg_ver:
# return ("package {} for target version {} is not found"
# .format(pkg_name, target_ver))
return None
def _get_deps_list(self, package):
logger.info('Getting dependencies for {}'.format(package))
NodeControlUtil.update_package_cache()
dep_tree = NodeControlUtil.get_deps_tree(package)
ret = []
NodeControlUtil.dep_tree_traverse(dep_tree, ret)
# Filter deps according to system hold list
# in case of hold empty return only package
holded = NodeControlUtil.get_sys_holds()
if not holded:
return package
else:
ret_list = []
for rl in ret:
name = rl.split("=", maxsplit=1)[0]
if name in holded:
ret_list.append(rl)
if package not in ret_list:
ret_list.append(package)
return " ".join(ret_list)
def curr_pkt_info(self, pkg_name):
if pkg_name == APP_NAME:
return Upgrader.getVersion(), [APP_NAME]
return NodeControlUtil.curr_pkt_info(pkg_name)
def _call_upgrade_node_script(self):
logger.info('Upgrading indy')
cmd = compose_cmd(['/opt/controller/upgrade_indy'])
NodeControlUtil.run_shell_script(cmd, timeout=self.timeout)