def test_call_async_output_kwargs(test_file, mocker):
mock_callback_stdout = mock.Mock()
mock_callback_stdinfo = mock.Mock()
mock_callback_stderr = mock.Mock()
def stdinfo_callback(a):
mock_callback_stdinfo(a)
def stdout_callback(a):
mock_callback_stdout(a)
def stderr_callback(a):
mock_callback_stderr(a)
callbacks = (
lambda l: stdout_callback(l),
lambda l: stderr_callback(l),
lambda l: stdinfo_callback(l),
)
with pytest.raises(ValueError):
call_async_output(["cat", str(test_file)], callbacks, stdout=None)
mock_callback_stdout.assert_not_called()
mock_callback_stdinfo.assert_not_called()
mock_callback_stderr.assert_not_called()
mock_callback_stdout.reset_mock()
mock_callback_stdinfo.reset_mock()
mock_callback_stderr.reset_mock()
with pytest.raises(ValueError):
call_async_output(["cat", str(test_file)], callbacks, stderr=None)
mock_callback_stdout.assert_not_called()
mock_callback_stdinfo.assert_not_called()
mock_callback_stderr.assert_not_called()
mock_callback_stdout.reset_mock()
mock_callback_stdinfo.reset_mock()
mock_callback_stderr.reset_mock()
with pytest.raises(TypeError):
call_async_output(["cat", str(test_file)], callbacks, stdinfo=None)
mock_callback_stdout.assert_not_called()
mock_callback_stdinfo.assert_not_called()
mock_callback_stderr.assert_not_called()
mock_callback_stdout.reset_mock()
mock_callback_stdinfo.reset_mock()
mock_callback_stderr.reset_mock()
dirname = os.path.dirname(str(test_file))
os.mkdir(os.path.join(dirname, "testcwd"))
call_async_output(["cat", str(test_file)],
callbacks,
cwd=os.path.join(dirname, "testcwd"))
calls = [mock.call("foo"), mock.call("bar")]
mock_callback_stdout.assert_has_calls(calls)
mock_callback_stdinfo.assert_not_called()
mock_callback_stderr.assert_not_called()
def apt_install(self, cmd):
def is_relevant(l):
return "Reading database ..." not in l.rstrip()
callbacks = (
lambda l: logger.info("+ " + l.rstrip() + "\r")
if is_relevant(l) else logger.debug(l.rstrip() + "\r"),
lambda l: logger.warning(l.rstrip()),
)
cmd = "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --quiet -o=Dpkg::Use-Pty=0 --fix-broken --assume-yes " + cmd
logger.debug("Running: %s" % cmd)
call_async_output(cmd, callbacks, shell=True)
def test_call_async_output(test_file):
mock_callback_stdout = mock.Mock()
mock_callback_stderr = mock.Mock()
def stdout_callback(a):
mock_callback_stdout(a)
def stderr_callback(a):
mock_callback_stderr(a)
callbacks = (lambda l: stdout_callback(l), lambda l: stderr_callback(l))
call_async_output(["cat", str(test_file)], callbacks)
calls = [mock.call("foo"), mock.call("bar")]
mock_callback_stdout.assert_has_calls(calls)
mock_callback_stderr.assert_not_called()
mock_callback_stdout.reset_mock()
mock_callback_stderr.reset_mock()
with pytest.raises(TypeError):
call_async_output(["cat", str(test_file)], 1)
mock_callback_stdout.assert_not_called()
mock_callback_stderr.assert_not_called()
mock_callback_stdout.reset_mock()
mock_callback_stderr.reset_mock()
def callback_stdout(a):
mock_callback_stdout(a)
def callback_stderr(a):
mock_callback_stderr(a)
callback = (callback_stdout, callback_stderr)
call_async_output(["cat", str(test_file)], callback)
calls = [mock.call("foo"), mock.call("bar")]
mock_callback_stdout.assert_has_calls(calls)
mock_callback_stderr.assert_not_called()
mock_callback_stdout.reset_mock()
mock_callback_stderr.reset_mock()
env_var = {"LANG": "C"}
call_async_output(["cat", "doesntexists"], callback, env=env_var)
calls = [mock.call("cat: doesntexists: No such file or directory")]
mock_callback_stdout.assert_not_called()
mock_callback_stderr.assert_has_calls(calls)
def hook_exec(path, args=None, raise_on_error=False, no_trace=False,
chdir=None, env=None):
"""
Execute hook from a file with arguments
Keyword argument:
path -- Path of the script to execute
args -- Ordered list of arguments to pass to the script
raise_on_error -- Raise if the script returns a non-zero exit code
no_trace -- Do not print each command that will be executed
chdir -- The directory from where the script will be executed
env -- Dictionnary of environment variables to export
"""
from moulinette.utils.process import call_async_output
from yunohost.app import _value_for_locale
# Validate hook path
if path[0] != '/':
path = os.path.realpath(path)
if not os.path.isfile(path):
raise MoulinetteError(errno.EIO, m18n.g('file_not_exist'))
# Construct command variables
cmd_args = ''
if args and isinstance(args, list):
# Concatenate arguments and escape them with double quotes to prevent
# bash related issue if an argument is empty and is not the last
cmd_args = '"{:s}"'.format('" "'.join(str(s) for s in args))
if not chdir:
# use the script directory as current one
chdir, cmd_script = os.path.split(path)
cmd_script = './{0}'.format(cmd_script)
else:
cmd_script = path
# Construct command to execute
command = ['sudo', '-n', '-u', 'admin', '-H', 'sh', '-c']
if no_trace:
cmd = '/bin/bash "{script}" {args}'
else:
# use xtrace on fd 7 which is redirected to stdout
cmd = 'BASH_XTRACEFD=7 /bin/bash -x "{script}" {args} 7>&1'
if env:
# prepend environment variables
cmd = '{0} {1}'.format(
' '.join(['{0}="{1}"'.format(k, v) for k, v in env.items()]), cmd)
command.append(cmd.format(script=cmd_script, args=cmd_args))
if logger.isEnabledFor(log.DEBUG):
logger.info(m18n.n('executing_command', command=' '.join(command)))
else:
logger.info(m18n.n('executing_script', script=path))
# Define output callbacks and call command
callbacks = (
lambda l: logger.info(l.rstrip()),
lambda l: logger.warning(l.rstrip()),
)
returncode = call_async_output(
command, callbacks, shell=False, cwd=chdir
)
# Check and return process' return code
if returncode is None:
if raise_on_error:
raise MoulinetteError(m18n.n('hook_exec_not_terminated', path=path))
else:
logger.error(m18n.n('hook_exec_not_terminated', path=path))
return 1
elif raise_on_error and returncode != 0:
raise MoulinetteError(m18n.n('hook_exec_failed', path=path))
return returncode
def tools_upgrade(operation_logger,
apps=None,
system=False,
allow_yunohost_upgrade=True):
"""
Update apps & package cache, then display changelog
Keyword arguments:
apps -- List of apps to upgrade (or [] to update all apps)
system -- True to upgrade system
"""
from yunohost.utils import packages
if packages.dpkg_is_broken():
raise YunohostValidationError("dpkg_is_broken")
# Check for obvious conflict with other dpkg/apt commands already running in parallel
if not packages.dpkg_lock_available():
raise YunohostValidationError("dpkg_lock_not_available")
if system is not False and apps is not None:
raise YunohostValidationError("tools_upgrade_cant_both")
if system is False and apps is None:
raise YunohostValidationError("tools_upgrade_at_least_one")
#
# Apps
# This is basically just an alias to yunohost app upgrade ...
#
if apps is not None:
# Make sure there's actually something to upgrade
upgradable_apps = [app["id"] for app in _list_upgradable_apps()]
if not upgradable_apps or (len(apps) and all(app not in upgradable_apps
for app in apps)):
logger.info(m18n.n("apps_already_up_to_date"))
return
# Actually start the upgrades
try:
app_upgrade(app=apps)
except Exception as e:
logger.warning("unable to upgrade apps: %s" % str(e))
logger.error(m18n.n("app_upgrade_some_app_failed"))
return
#
# System
#
if system is True:
# Check that there's indeed some packages to upgrade
upgradables = list(_list_upgradable_apt_packages())
if not upgradables:
logger.info(m18n.n("already_up_to_date"))
logger.info(m18n.n("upgrading_packages"))
operation_logger.start()
# Critical packages are packages that we can't just upgrade
# randomly from yunohost itself... upgrading them is likely to
critical_packages = [
"moulinette", "yunohost", "yunohost-admin", "ssowat"
]
critical_packages_upgradable = [
p["name"] for p in upgradables if p["name"] in critical_packages
]
noncritical_packages_upgradable = [
p["name"] for p in upgradables
if p["name"] not in critical_packages
]
# Prepare dist-upgrade command
dist_upgrade = "DEBIAN_FRONTEND=noninteractive"
dist_upgrade += " APT_LISTCHANGES_FRONTEND=none"
dist_upgrade += " apt-get"
dist_upgrade += (
" --fix-broken --show-upgraded --assume-yes --quiet -o=Dpkg::Use-Pty=0"
)
for conf_flag in ["old", "miss", "def"]:
dist_upgrade += ' -o Dpkg::Options::="--force-conf{}"'.format(
conf_flag)
dist_upgrade += " dist-upgrade"
#
# "Regular" packages upgrade
#
if noncritical_packages_upgradable:
logger.info(m18n.n("tools_upgrade_regular_packages"))
# Mark all critical packages as held
for package in critical_packages:
check_output("apt-mark hold %s" % package)
# Doublecheck with apt-mark showhold that packages are indeed held ...
held_packages = check_output("apt-mark showhold").split("\n")
if any(p not in held_packages for p in critical_packages):
logger.warning(
m18n.n("tools_upgrade_cant_hold_critical_packages"))
operation_logger.error(m18n.n("packages_upgrade_failed"))
raise YunohostError(m18n.n("packages_upgrade_failed"))
logger.debug("Running apt command :\n{}".format(dist_upgrade))
def is_relevant(line):
irrelevants = [
"service sudo-ldap already provided",
"Reading database ...",
]
return all(i not in line.rstrip() for i in irrelevants)
callbacks = (
lambda l: logger.info("+ " + l.rstrip() + "\r")
if is_relevant(l) else logger.debug(l.rstrip() + "\r"),
lambda l: logger.warning(l.rstrip())
if is_relevant(l) else logger.debug(l.rstrip()),
)
returncode = call_async_output(dist_upgrade, callbacks, shell=True)
if returncode != 0:
upgradables = list(_list_upgradable_apt_packages())
noncritical_packages_upgradable = [
p["name"] for p in upgradables
if p["name"] not in critical_packages
]
logger.warning(
m18n.n(
"tools_upgrade_regular_packages_failed",
packages_list=", ".join(
noncritical_packages_upgradable),
))
operation_logger.error(m18n.n("packages_upgrade_failed"))
raise YunohostError(m18n.n("packages_upgrade_failed"))
#
# Critical packages upgrade
#
if critical_packages_upgradable and allow_yunohost_upgrade:
logger.info(m18n.n("tools_upgrade_special_packages"))
# Mark all critical packages as unheld
for package in critical_packages:
check_output("apt-mark unhold %s" % package)
# Doublecheck with apt-mark showhold that packages are indeed unheld ...
held_packages = check_output("apt-mark showhold").split("\n")
if any(p in held_packages for p in critical_packages):
logger.warning(
m18n.n("tools_upgrade_cant_unhold_critical_packages"))
operation_logger.error(m18n.n("packages_upgrade_failed"))
raise YunohostError(m18n.n("packages_upgrade_failed"))
#
# Here we use a dirty hack to run a command after the current
# "yunohost tools upgrade", because the upgrade of yunohost
# will also trigger other yunohost commands (e.g. "yunohost tools migrations run")
# (also the upgrade of the package, if executed from the webadmin, is
# likely to kill/restart the api which is in turn likely to kill this
# command before it ends...)
#
logfile = operation_logger.log_path
dist_upgrade = dist_upgrade + " 2>&1 | tee -a {}".format(logfile)
MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock"
wait_until_end_of_yunohost_command = (
"(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK))
mark_success = (
"(echo 'Done!' | tee -a {} && echo 'success: true' >> {})".
format(logfile, operation_logger.md_path))
mark_failure = (
"(echo 'Failed :(' | tee -a {} && echo 'success: false' >> {})"
.format(logfile, operation_logger.md_path))
update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}"
update_log_metadata = update_log_metadata.format(
operation_logger.md_path)
# Dirty hack such that the operation_logger does not add ended_at
# and success keys in the log metadata. (c.f. the code of the
# is_unit_operation + operation_logger.close()) We take care of
# this ourselves (c.f. the mark_success and updated_log_metadata in
# the huge command launched by os.system)
operation_logger.ended_at = "notyet"
upgrade_completed = "\n" + m18n.n(
"tools_upgrade_special_packages_completed")
command = "({wait} && {dist_upgrade}) && {mark_success} || {mark_failure}; {update_metadata}; echo '{done}'".format(
wait=wait_until_end_of_yunohost_command,
dist_upgrade=dist_upgrade,
mark_success=mark_success,
mark_failure=mark_failure,
update_metadata=update_log_metadata,
done=upgrade_completed,
)
logger.warning(
m18n.n("tools_upgrade_special_packages_explanation"))
logger.debug("Running command :\n{}".format(command))
open("/tmp/yunohost-selfupgrade",
"w").write("rm /tmp/yunohost-selfupgrade; " + command)
# Using systemd-run --scope is like nohup/disown and &, but more robust somehow
# (despite using nohup/disown and &, the self-upgrade process was still getting killed...)
# ref: https://unix.stackexchange.com/questions/420594/why-process-killed-with-nohup
# (though I still don't understand it 100%...)
os.system("systemd-run --scope bash /tmp/yunohost-selfupgrade &")
return
else:
logger.success(m18n.n("system_upgraded"))
operation_logger.success()
def tools_update(apps=False, system=False):
"""
Update apps & system package cache
Keyword arguments:
system -- Fetch available system packages upgrades (equivalent to apt update)
apps -- Fetch the application list to check which apps can be upgraded
"""
# If neither --apps nor --system specified, do both
if not apps and not system:
apps = True
system = True
upgradable_system_packages = []
if system:
# Update APT cache
# LC_ALL=C is here to make sure the results are in english
command = "LC_ALL=C apt-get update -o Acquire::Retries=3"
# Filter boring message about "apt not having a stable CLI interface"
# Also keep track of wether or not we encountered a warning...
warnings = []
def is_legit_warning(m):
legit_warning = (m.rstrip()
and "apt does not have a stable CLI interface"
not in m.rstrip())
if legit_warning:
warnings.append(m)
return legit_warning
callbacks = (
# stdout goes to debug
lambda l: logger.debug(l.rstrip()),
# stderr goes to warning except for the boring apt messages
lambda l: logger.warning(l.rstrip())
if is_legit_warning(l) else logger.debug(l.rstrip()),
)
logger.info(m18n.n("updating_apt_cache"))
returncode = call_async_output(command, callbacks, shell=True)
if returncode != 0:
raise YunohostError("update_apt_cache_failed",
sourceslist="\n".join(_dump_sources_list()))
elif warnings:
logger.error(
m18n.n(
"update_apt_cache_warning",
sourceslist="\n".join(_dump_sources_list()),
))
upgradable_system_packages = list(_list_upgradable_apt_packages())
logger.debug(m18n.n("done"))
upgradable_apps = []
if apps:
try:
_update_apps_catalog()
except YunohostError as e:
logger.error(str(e))
upgradable_apps = list(_list_upgradable_apps())
if len(upgradable_apps) == 0 and len(upgradable_system_packages) == 0:
logger.info(m18n.n("already_up_to_date"))
return {"system": upgradable_system_packages, "apps": upgradable_apps}
def tools_update(target=None, apps=False, system=False):
"""
Update apps & system package cache
"""
# Legacy options (--system, --apps)
if apps or system:
logger.warning(
"Using 'yunohost tools update' with --apps / --system is deprecated, just write 'yunohost tools update apps system' (no -- prefix anymore)"
)
if apps and system:
target = "all"
elif apps:
target = "apps"
else:
target = "system"
elif not target:
target = "all"
if target not in ["system", "apps", "all"]:
raise YunohostError(
"Unknown target %s, should be 'system', 'apps' or 'all'" % target,
raw_msg=True,
)
upgradable_system_packages = []
if target in ["system", "all"]:
# Update APT cache
# LC_ALL=C is here to make sure the results are in english
command = "LC_ALL=C apt-get update -o Acquire::Retries=3"
# Filter boring message about "apt not having a stable CLI interface"
# Also keep track of wether or not we encountered a warning...
warnings = []
def is_legit_warning(m):
legit_warning = (m.rstrip()
and "apt does not have a stable CLI interface"
not in m.rstrip())
if legit_warning:
warnings.append(m)
return legit_warning
callbacks = (
# stdout goes to debug
lambda l: logger.debug(l.rstrip()),
# stderr goes to warning except for the boring apt messages
lambda l: logger.warning(l.rstrip())
if is_legit_warning(l) else logger.debug(l.rstrip()),
)
logger.info(m18n.n("updating_apt_cache"))
returncode = call_async_output(command, callbacks, shell=True)
if returncode != 0:
raise YunohostError("update_apt_cache_failed",
sourceslist="\n".join(_dump_sources_list()))
elif warnings:
logger.error(
m18n.n(
"update_apt_cache_warning",
sourceslist="\n".join(_dump_sources_list()),
))
upgradable_system_packages = list(_list_upgradable_apt_packages())
logger.debug(m18n.n("done"))
upgradable_apps = []
if target in ["apps", "all"]:
try:
_update_apps_catalog()
except YunohostError as e:
logger.error(str(e))
upgradable_apps = list(_list_upgradable_apps())
if len(upgradable_apps) == 0 and len(upgradable_system_packages) == 0:
logger.info(m18n.n("already_up_to_date"))
return {"system": upgradable_system_packages, "apps": upgradable_apps}
def _hook_exec_bash(path, args, chdir, env, return_format, loggers):
from moulinette.utils.process import call_async_output
# Construct command variables
cmd_args = ""
if args and isinstance(args, list):
# Concatenate escaped arguments
cmd_args = " ".join(shell_quote(s) for s in args)
if not chdir:
# use the script directory as current one
chdir, cmd_script = os.path.split(path)
cmd_script = "./{0}".format(cmd_script)
else:
cmd_script = path
# Add Execution dir to environment var
if env is None:
env = {}
env["YNH_CWD"] = chdir
env["YNH_INTERFACE"] = msettings.get("interface")
stdreturn = os.path.join(tempfile.mkdtemp(), "stdreturn")
with open(stdreturn, "w") as f:
f.write("")
env["YNH_STDRETURN"] = stdreturn
# use xtrace on fd 7 which is redirected to stdout
env["BASH_XTRACEFD"] = "7"
cmd = '/bin/bash -x "{script}" {args} 7>&1'
cmd = cmd.format(script=cmd_script, args=cmd_args)
logger.debug("Executing command '%s'" % cmd)
_env = os.environ.copy()
_env.update(env)
returncode = call_async_output(cmd,
loggers,
shell=True,
cwd=chdir,
env=_env)
raw_content = None
try:
with open(stdreturn, "r") as f:
raw_content = f.read()
returncontent = {}
if return_format == "json":
if raw_content != "":
try:
returncontent = read_json(stdreturn)
except Exception as e:
raise YunohostError(
"hook_json_return_error",
path=path,
msg=str(e),
raw_content=raw_content,
)
elif return_format == "plain_dict":
for line in raw_content.split("\n"):
if "=" in line:
key, value = line.strip().split("=", 1)
returncontent[key] = value
else:
raise YunohostError(
"Expected value for return_format is either 'json' or 'plain_dict', got '%s'"
% return_format)
finally:
stdreturndir = os.path.split(stdreturn)[0]
os.remove(stdreturn)
os.rmdir(stdreturndir)
return returncode, returncontent
