def test_call_async_output_kwargs(test_file, mocker): mock_callback_stdout = mock.Mock() mock_callback_stdinfo = mock.Mock() mock_callback_stderr = mock.Mock() def stdinfo_callback(a): mock_callback_stdinfo(a) def stdout_callback(a): mock_callback_stdout(a) def stderr_callback(a): mock_callback_stderr(a) callbacks = ( lambda l: stdout_callback(l), lambda l: stderr_callback(l), lambda l: stdinfo_callback(l), ) with pytest.raises(ValueError): call_async_output(["cat", str(test_file)], callbacks, stdout=None) mock_callback_stdout.assert_not_called() mock_callback_stdinfo.assert_not_called() mock_callback_stderr.assert_not_called() mock_callback_stdout.reset_mock() mock_callback_stdinfo.reset_mock() mock_callback_stderr.reset_mock() with pytest.raises(ValueError): call_async_output(["cat", str(test_file)], callbacks, stderr=None) mock_callback_stdout.assert_not_called() mock_callback_stdinfo.assert_not_called() mock_callback_stderr.assert_not_called() mock_callback_stdout.reset_mock() mock_callback_stdinfo.reset_mock() mock_callback_stderr.reset_mock() with pytest.raises(TypeError): call_async_output(["cat", str(test_file)], callbacks, stdinfo=None) mock_callback_stdout.assert_not_called() mock_callback_stdinfo.assert_not_called() mock_callback_stderr.assert_not_called() mock_callback_stdout.reset_mock() mock_callback_stdinfo.reset_mock() mock_callback_stderr.reset_mock() dirname = os.path.dirname(str(test_file)) os.mkdir(os.path.join(dirname, "testcwd")) call_async_output(["cat", str(test_file)], callbacks, cwd=os.path.join(dirname, "testcwd")) calls = [mock.call("foo"), mock.call("bar")] mock_callback_stdout.assert_has_calls(calls) mock_callback_stdinfo.assert_not_called() mock_callback_stderr.assert_not_called()
def apt_install(self, cmd): def is_relevant(l): return "Reading database ..." not in l.rstrip() callbacks = ( lambda l: logger.info("+ " + l.rstrip() + "\r") if is_relevant(l) else logger.debug(l.rstrip() + "\r"), lambda l: logger.warning(l.rstrip()), ) cmd = "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --quiet -o=Dpkg::Use-Pty=0 --fix-broken --assume-yes " + cmd logger.debug("Running: %s" % cmd) call_async_output(cmd, callbacks, shell=True)
def test_call_async_output(test_file): mock_callback_stdout = mock.Mock() mock_callback_stderr = mock.Mock() def stdout_callback(a): mock_callback_stdout(a) def stderr_callback(a): mock_callback_stderr(a) callbacks = (lambda l: stdout_callback(l), lambda l: stderr_callback(l)) call_async_output(["cat", str(test_file)], callbacks) calls = [mock.call("foo"), mock.call("bar")] mock_callback_stdout.assert_has_calls(calls) mock_callback_stderr.assert_not_called() mock_callback_stdout.reset_mock() mock_callback_stderr.reset_mock() with pytest.raises(TypeError): call_async_output(["cat", str(test_file)], 1) mock_callback_stdout.assert_not_called() mock_callback_stderr.assert_not_called() mock_callback_stdout.reset_mock() mock_callback_stderr.reset_mock() def callback_stdout(a): mock_callback_stdout(a) def callback_stderr(a): mock_callback_stderr(a) callback = (callback_stdout, callback_stderr) call_async_output(["cat", str(test_file)], callback) calls = [mock.call("foo"), mock.call("bar")] mock_callback_stdout.assert_has_calls(calls) mock_callback_stderr.assert_not_called() mock_callback_stdout.reset_mock() mock_callback_stderr.reset_mock() env_var = {"LANG": "C"} call_async_output(["cat", "doesntexists"], callback, env=env_var) calls = [mock.call("cat: doesntexists: No such file or directory")] mock_callback_stdout.assert_not_called() mock_callback_stderr.assert_has_calls(calls)
def hook_exec(path, args=None, raise_on_error=False, no_trace=False, chdir=None, env=None): """ Execute hook from a file with arguments Keyword argument: path -- Path of the script to execute args -- Ordered list of arguments to pass to the script raise_on_error -- Raise if the script returns a non-zero exit code no_trace -- Do not print each command that will be executed chdir -- The directory from where the script will be executed env -- Dictionnary of environment variables to export """ from moulinette.utils.process import call_async_output from yunohost.app import _value_for_locale # Validate hook path if path[0] != '/': path = os.path.realpath(path) if not os.path.isfile(path): raise MoulinetteError(errno.EIO, m18n.g('file_not_exist')) # Construct command variables cmd_args = '' if args and isinstance(args, list): # Concatenate arguments and escape them with double quotes to prevent # bash related issue if an argument is empty and is not the last cmd_args = '"{:s}"'.format('" "'.join(str(s) for s in args)) if not chdir: # use the script directory as current one chdir, cmd_script = os.path.split(path) cmd_script = './{0}'.format(cmd_script) else: cmd_script = path # Construct command to execute command = ['sudo', '-n', '-u', 'admin', '-H', 'sh', '-c'] if no_trace: cmd = '/bin/bash "{script}" {args}' else: # use xtrace on fd 7 which is redirected to stdout cmd = 'BASH_XTRACEFD=7 /bin/bash -x "{script}" {args} 7>&1' if env: # prepend environment variables cmd = '{0} {1}'.format( ' '.join(['{0}="{1}"'.format(k, v) for k, v in env.items()]), cmd) command.append(cmd.format(script=cmd_script, args=cmd_args)) if logger.isEnabledFor(log.DEBUG): logger.info(m18n.n('executing_command', command=' '.join(command))) else: logger.info(m18n.n('executing_script', script=path)) # Define output callbacks and call command callbacks = ( lambda l: logger.info(l.rstrip()), lambda l: logger.warning(l.rstrip()), ) returncode = call_async_output( command, callbacks, shell=False, cwd=chdir ) # Check and return process' return code if returncode is None: if raise_on_error: raise MoulinetteError(m18n.n('hook_exec_not_terminated', path=path)) else: logger.error(m18n.n('hook_exec_not_terminated', path=path)) return 1 elif raise_on_error and returncode != 0: raise MoulinetteError(m18n.n('hook_exec_failed', path=path)) return returncode
def tools_upgrade(operation_logger, apps=None, system=False, allow_yunohost_upgrade=True): """ Update apps & package cache, then display changelog Keyword arguments: apps -- List of apps to upgrade (or [] to update all apps) system -- True to upgrade system """ from yunohost.utils import packages if packages.dpkg_is_broken(): raise YunohostValidationError("dpkg_is_broken") # Check for obvious conflict with other dpkg/apt commands already running in parallel if not packages.dpkg_lock_available(): raise YunohostValidationError("dpkg_lock_not_available") if system is not False and apps is not None: raise YunohostValidationError("tools_upgrade_cant_both") if system is False and apps is None: raise YunohostValidationError("tools_upgrade_at_least_one") # # Apps # This is basically just an alias to yunohost app upgrade ... # if apps is not None: # Make sure there's actually something to upgrade upgradable_apps = [app["id"] for app in _list_upgradable_apps()] if not upgradable_apps or (len(apps) and all(app not in upgradable_apps for app in apps)): logger.info(m18n.n("apps_already_up_to_date")) return # Actually start the upgrades try: app_upgrade(app=apps) except Exception as e: logger.warning("unable to upgrade apps: %s" % str(e)) logger.error(m18n.n("app_upgrade_some_app_failed")) return # # System # if system is True: # Check that there's indeed some packages to upgrade upgradables = list(_list_upgradable_apt_packages()) if not upgradables: logger.info(m18n.n("already_up_to_date")) logger.info(m18n.n("upgrading_packages")) operation_logger.start() # Critical packages are packages that we can't just upgrade # randomly from yunohost itself... upgrading them is likely to critical_packages = [ "moulinette", "yunohost", "yunohost-admin", "ssowat" ] critical_packages_upgradable = [ p["name"] for p in upgradables if p["name"] in critical_packages ] noncritical_packages_upgradable = [ p["name"] for p in upgradables if p["name"] not in critical_packages ] # Prepare dist-upgrade command dist_upgrade = "DEBIAN_FRONTEND=noninteractive" dist_upgrade += " APT_LISTCHANGES_FRONTEND=none" dist_upgrade += " apt-get" dist_upgrade += ( " --fix-broken --show-upgraded --assume-yes --quiet -o=Dpkg::Use-Pty=0" ) for conf_flag in ["old", "miss", "def"]: dist_upgrade += ' -o Dpkg::Options::="--force-conf{}"'.format( conf_flag) dist_upgrade += " dist-upgrade" # # "Regular" packages upgrade # if noncritical_packages_upgradable: logger.info(m18n.n("tools_upgrade_regular_packages")) # Mark all critical packages as held for package in critical_packages: check_output("apt-mark hold %s" % package) # Doublecheck with apt-mark showhold that packages are indeed held ... held_packages = check_output("apt-mark showhold").split("\n") if any(p not in held_packages for p in critical_packages): logger.warning( m18n.n("tools_upgrade_cant_hold_critical_packages")) operation_logger.error(m18n.n("packages_upgrade_failed")) raise YunohostError(m18n.n("packages_upgrade_failed")) logger.debug("Running apt command :\n{}".format(dist_upgrade)) def is_relevant(line): irrelevants = [ "service sudo-ldap already provided", "Reading database ...", ] return all(i not in line.rstrip() for i in irrelevants) callbacks = ( lambda l: logger.info("+ " + l.rstrip() + "\r") if is_relevant(l) else logger.debug(l.rstrip() + "\r"), lambda l: logger.warning(l.rstrip()) if is_relevant(l) else logger.debug(l.rstrip()), ) returncode = call_async_output(dist_upgrade, callbacks, shell=True) if returncode != 0: upgradables = list(_list_upgradable_apt_packages()) noncritical_packages_upgradable = [ p["name"] for p in upgradables if p["name"] not in critical_packages ] logger.warning( m18n.n( "tools_upgrade_regular_packages_failed", packages_list=", ".join( noncritical_packages_upgradable), )) operation_logger.error(m18n.n("packages_upgrade_failed")) raise YunohostError(m18n.n("packages_upgrade_failed")) # # Critical packages upgrade # if critical_packages_upgradable and allow_yunohost_upgrade: logger.info(m18n.n("tools_upgrade_special_packages")) # Mark all critical packages as unheld for package in critical_packages: check_output("apt-mark unhold %s" % package) # Doublecheck with apt-mark showhold that packages are indeed unheld ... held_packages = check_output("apt-mark showhold").split("\n") if any(p in held_packages for p in critical_packages): logger.warning( m18n.n("tools_upgrade_cant_unhold_critical_packages")) operation_logger.error(m18n.n("packages_upgrade_failed")) raise YunohostError(m18n.n("packages_upgrade_failed")) # # Here we use a dirty hack to run a command after the current # "yunohost tools upgrade", because the upgrade of yunohost # will also trigger other yunohost commands (e.g. "yunohost tools migrations run") # (also the upgrade of the package, if executed from the webadmin, is # likely to kill/restart the api which is in turn likely to kill this # command before it ends...) # logfile = operation_logger.log_path dist_upgrade = dist_upgrade + " 2>&1 | tee -a {}".format(logfile) MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock" wait_until_end_of_yunohost_command = ( "(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK)) mark_success = ( "(echo 'Done!' | tee -a {} && echo 'success: true' >> {})". format(logfile, operation_logger.md_path)) mark_failure = ( "(echo 'Failed :(' | tee -a {} && echo 'success: false' >> {})" .format(logfile, operation_logger.md_path)) update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}" update_log_metadata = update_log_metadata.format( operation_logger.md_path) # Dirty hack such that the operation_logger does not add ended_at # and success keys in the log metadata. (c.f. the code of the # is_unit_operation + operation_logger.close()) We take care of # this ourselves (c.f. the mark_success and updated_log_metadata in # the huge command launched by os.system) operation_logger.ended_at = "notyet" upgrade_completed = "\n" + m18n.n( "tools_upgrade_special_packages_completed") command = "({wait} && {dist_upgrade}) && {mark_success} || {mark_failure}; {update_metadata}; echo '{done}'".format( wait=wait_until_end_of_yunohost_command, dist_upgrade=dist_upgrade, mark_success=mark_success, mark_failure=mark_failure, update_metadata=update_log_metadata, done=upgrade_completed, ) logger.warning( m18n.n("tools_upgrade_special_packages_explanation")) logger.debug("Running command :\n{}".format(command)) open("/tmp/yunohost-selfupgrade", "w").write("rm /tmp/yunohost-selfupgrade; " + command) # Using systemd-run --scope is like nohup/disown and &, but more robust somehow # (despite using nohup/disown and &, the self-upgrade process was still getting killed...) # ref: https://unix.stackexchange.com/questions/420594/why-process-killed-with-nohup # (though I still don't understand it 100%...) os.system("systemd-run --scope bash /tmp/yunohost-selfupgrade &") return else: logger.success(m18n.n("system_upgraded")) operation_logger.success()
def tools_update(apps=False, system=False): """ Update apps & system package cache Keyword arguments: system -- Fetch available system packages upgrades (equivalent to apt update) apps -- Fetch the application list to check which apps can be upgraded """ # If neither --apps nor --system specified, do both if not apps and not system: apps = True system = True upgradable_system_packages = [] if system: # Update APT cache # LC_ALL=C is here to make sure the results are in english command = "LC_ALL=C apt-get update -o Acquire::Retries=3" # Filter boring message about "apt not having a stable CLI interface" # Also keep track of wether or not we encountered a warning... warnings = [] def is_legit_warning(m): legit_warning = (m.rstrip() and "apt does not have a stable CLI interface" not in m.rstrip()) if legit_warning: warnings.append(m) return legit_warning callbacks = ( # stdout goes to debug lambda l: logger.debug(l.rstrip()), # stderr goes to warning except for the boring apt messages lambda l: logger.warning(l.rstrip()) if is_legit_warning(l) else logger.debug(l.rstrip()), ) logger.info(m18n.n("updating_apt_cache")) returncode = call_async_output(command, callbacks, shell=True) if returncode != 0: raise YunohostError("update_apt_cache_failed", sourceslist="\n".join(_dump_sources_list())) elif warnings: logger.error( m18n.n( "update_apt_cache_warning", sourceslist="\n".join(_dump_sources_list()), )) upgradable_system_packages = list(_list_upgradable_apt_packages()) logger.debug(m18n.n("done")) upgradable_apps = [] if apps: try: _update_apps_catalog() except YunohostError as e: logger.error(str(e)) upgradable_apps = list(_list_upgradable_apps()) if len(upgradable_apps) == 0 and len(upgradable_system_packages) == 0: logger.info(m18n.n("already_up_to_date")) return {"system": upgradable_system_packages, "apps": upgradable_apps}
def tools_update(target=None, apps=False, system=False): """ Update apps & system package cache """ # Legacy options (--system, --apps) if apps or system: logger.warning( "Using 'yunohost tools update' with --apps / --system is deprecated, just write 'yunohost tools update apps system' (no -- prefix anymore)" ) if apps and system: target = "all" elif apps: target = "apps" else: target = "system" elif not target: target = "all" if target not in ["system", "apps", "all"]: raise YunohostError( "Unknown target %s, should be 'system', 'apps' or 'all'" % target, raw_msg=True, ) upgradable_system_packages = [] if target in ["system", "all"]: # Update APT cache # LC_ALL=C is here to make sure the results are in english command = "LC_ALL=C apt-get update -o Acquire::Retries=3" # Filter boring message about "apt not having a stable CLI interface" # Also keep track of wether or not we encountered a warning... warnings = [] def is_legit_warning(m): legit_warning = (m.rstrip() and "apt does not have a stable CLI interface" not in m.rstrip()) if legit_warning: warnings.append(m) return legit_warning callbacks = ( # stdout goes to debug lambda l: logger.debug(l.rstrip()), # stderr goes to warning except for the boring apt messages lambda l: logger.warning(l.rstrip()) if is_legit_warning(l) else logger.debug(l.rstrip()), ) logger.info(m18n.n("updating_apt_cache")) returncode = call_async_output(command, callbacks, shell=True) if returncode != 0: raise YunohostError("update_apt_cache_failed", sourceslist="\n".join(_dump_sources_list())) elif warnings: logger.error( m18n.n( "update_apt_cache_warning", sourceslist="\n".join(_dump_sources_list()), )) upgradable_system_packages = list(_list_upgradable_apt_packages()) logger.debug(m18n.n("done")) upgradable_apps = [] if target in ["apps", "all"]: try: _update_apps_catalog() except YunohostError as e: logger.error(str(e)) upgradable_apps = list(_list_upgradable_apps()) if len(upgradable_apps) == 0 and len(upgradable_system_packages) == 0: logger.info(m18n.n("already_up_to_date")) return {"system": upgradable_system_packages, "apps": upgradable_apps}
def _hook_exec_bash(path, args, chdir, env, return_format, loggers): from moulinette.utils.process import call_async_output # Construct command variables cmd_args = "" if args and isinstance(args, list): # Concatenate escaped arguments cmd_args = " ".join(shell_quote(s) for s in args) if not chdir: # use the script directory as current one chdir, cmd_script = os.path.split(path) cmd_script = "./{0}".format(cmd_script) else: cmd_script = path # Add Execution dir to environment var if env is None: env = {} env["YNH_CWD"] = chdir env["YNH_INTERFACE"] = msettings.get("interface") stdreturn = os.path.join(tempfile.mkdtemp(), "stdreturn") with open(stdreturn, "w") as f: f.write("") env["YNH_STDRETURN"] = stdreturn # use xtrace on fd 7 which is redirected to stdout env["BASH_XTRACEFD"] = "7" cmd = '/bin/bash -x "{script}" {args} 7>&1' cmd = cmd.format(script=cmd_script, args=cmd_args) logger.debug("Executing command '%s'" % cmd) _env = os.environ.copy() _env.update(env) returncode = call_async_output(cmd, loggers, shell=True, cwd=chdir, env=_env) raw_content = None try: with open(stdreturn, "r") as f: raw_content = f.read() returncontent = {} if return_format == "json": if raw_content != "": try: returncontent = read_json(stdreturn) except Exception as e: raise YunohostError( "hook_json_return_error", path=path, msg=str(e), raw_content=raw_content, ) elif return_format == "plain_dict": for line in raw_content.split("\n"): if "=" in line: key, value = line.strip().split("=", 1) returncontent[key] = value else: raise YunohostError( "Expected value for return_format is either 'json' or 'plain_dict', got '%s'" % return_format) finally: stdreturndir = os.path.split(stdreturn)[0] os.remove(stdreturn) os.rmdir(stdreturndir) return returncode, returncontent