def test_systemfuckedup_during_app_upgrade(mocker, secondary_domain):
install_break_yo_system(secondary_domain, breakwhat="upgrade")
with pytest.raises(YunohostError):
with message(mocker, "app_action_broke_system"):
app_upgrade("break_yo_system", file=os.path.join(get_test_apps_dir(), "break_yo_system_ynh"))
def test_failed_multiple_app_upgrade(mocker, secondary_domain):
install_legacy_app(secondary_domain, "/legacy")
install_break_yo_system(secondary_domain, breakwhat="upgrade")
with pytest.raises(YunohostError):
with message(mocker, "app_not_upgraded"):
app_upgrade(
["break_yo_system", "legacy_app"],
file={
"break_yo_system": os.path.join(get_test_apps_dir(), "break_yo_system_ynh"),
"legacy": os.path.join(get_test_apps_dir(), "legacy_app_ynh"),
},
)
def test_permission_protection_management_by_helper():
app_install(os.path.join(get_test_apps_dir(), "permissions_app_ynh"),
args="domain=%s&domain_2=%s&path=%s&admin=%s" %
(maindomain, other_domains[0], "/urlpermissionapp", "alice"),
force=True)
res = user_permission_list(full=True)['permissions']
assert res['permissions_app.main']['protected'] is False
assert res['permissions_app.admin']['protected'] is True
assert res['permissions_app.dev']['protected'] is False
app_upgrade(["permissions_app"],
file=os.path.join(get_test_apps_dir(), "permissions_app_ynh"))
res = user_permission_list(full=True)['permissions']
assert res['permissions_app.main']['protected'] is False
assert res['permissions_app.admin']['protected'] is False
assert res['permissions_app.dev']['protected'] is True
def tools_upgrade(operation_logger,
apps=None,
system=False,
allow_yunohost_upgrade=True):
"""
Update apps & package cache, then display changelog
Keyword arguments:
apps -- List of apps to upgrade (or [] to update all apps)
system -- True to upgrade system
"""
from yunohost.utils import packages
if packages.dpkg_is_broken():
raise YunohostValidationError("dpkg_is_broken")
# Check for obvious conflict with other dpkg/apt commands already running in parallel
if not packages.dpkg_lock_available():
raise YunohostValidationError("dpkg_lock_not_available")
if system is not False and apps is not None:
raise YunohostValidationError("tools_upgrade_cant_both")
if system is False and apps is None:
raise YunohostValidationError("tools_upgrade_at_least_one")
#
# Apps
# This is basically just an alias to yunohost app upgrade ...
#
if apps is not None:
# Make sure there's actually something to upgrade
upgradable_apps = [app["id"] for app in _list_upgradable_apps()]
if not upgradable_apps or (len(apps) and all(app not in upgradable_apps
for app in apps)):
logger.info(m18n.n("apps_already_up_to_date"))
return
# Actually start the upgrades
try:
app_upgrade(app=apps)
except Exception as e:
logger.warning("unable to upgrade apps: %s" % str(e))
logger.error(m18n.n("app_upgrade_some_app_failed"))
return
#
# System
#
if system is True:
# Check that there's indeed some packages to upgrade
upgradables = list(_list_upgradable_apt_packages())
if not upgradables:
logger.info(m18n.n("already_up_to_date"))
logger.info(m18n.n("upgrading_packages"))
operation_logger.start()
# Critical packages are packages that we can't just upgrade
# randomly from yunohost itself... upgrading them is likely to
critical_packages = [
"moulinette", "yunohost", "yunohost-admin", "ssowat"
]
critical_packages_upgradable = [
p["name"] for p in upgradables if p["name"] in critical_packages
]
noncritical_packages_upgradable = [
p["name"] for p in upgradables
if p["name"] not in critical_packages
]
# Prepare dist-upgrade command
dist_upgrade = "DEBIAN_FRONTEND=noninteractive"
dist_upgrade += " APT_LISTCHANGES_FRONTEND=none"
dist_upgrade += " apt-get"
dist_upgrade += (
" --fix-broken --show-upgraded --assume-yes --quiet -o=Dpkg::Use-Pty=0"
)
for conf_flag in ["old", "miss", "def"]:
dist_upgrade += ' -o Dpkg::Options::="--force-conf{}"'.format(
conf_flag)
dist_upgrade += " dist-upgrade"
#
# "Regular" packages upgrade
#
if noncritical_packages_upgradable:
logger.info(m18n.n("tools_upgrade_regular_packages"))
# Mark all critical packages as held
for package in critical_packages:
check_output("apt-mark hold %s" % package)
# Doublecheck with apt-mark showhold that packages are indeed held ...
held_packages = check_output("apt-mark showhold").split("\n")
if any(p not in held_packages for p in critical_packages):
logger.warning(
m18n.n("tools_upgrade_cant_hold_critical_packages"))
operation_logger.error(m18n.n("packages_upgrade_failed"))
raise YunohostError(m18n.n("packages_upgrade_failed"))
logger.debug("Running apt command :\n{}".format(dist_upgrade))
def is_relevant(line):
irrelevants = [
"service sudo-ldap already provided",
"Reading database ...",
]
return all(i not in line.rstrip() for i in irrelevants)
callbacks = (
lambda l: logger.info("+ " + l.rstrip() + "\r")
if is_relevant(l) else logger.debug(l.rstrip() + "\r"),
lambda l: logger.warning(l.rstrip())
if is_relevant(l) else logger.debug(l.rstrip()),
)
returncode = call_async_output(dist_upgrade, callbacks, shell=True)
if returncode != 0:
upgradables = list(_list_upgradable_apt_packages())
noncritical_packages_upgradable = [
p["name"] for p in upgradables
if p["name"] not in critical_packages
]
logger.warning(
m18n.n(
"tools_upgrade_regular_packages_failed",
packages_list=", ".join(
noncritical_packages_upgradable),
))
operation_logger.error(m18n.n("packages_upgrade_failed"))
raise YunohostError(m18n.n("packages_upgrade_failed"))
#
# Critical packages upgrade
#
if critical_packages_upgradable and allow_yunohost_upgrade:
logger.info(m18n.n("tools_upgrade_special_packages"))
# Mark all critical packages as unheld
for package in critical_packages:
check_output("apt-mark unhold %s" % package)
# Doublecheck with apt-mark showhold that packages are indeed unheld ...
held_packages = check_output("apt-mark showhold").split("\n")
if any(p in held_packages for p in critical_packages):
logger.warning(
m18n.n("tools_upgrade_cant_unhold_critical_packages"))
operation_logger.error(m18n.n("packages_upgrade_failed"))
raise YunohostError(m18n.n("packages_upgrade_failed"))
#
# Here we use a dirty hack to run a command after the current
# "yunohost tools upgrade", because the upgrade of yunohost
# will also trigger other yunohost commands (e.g. "yunohost tools migrations run")
# (also the upgrade of the package, if executed from the webadmin, is
# likely to kill/restart the api which is in turn likely to kill this
# command before it ends...)
#
logfile = operation_logger.log_path
dist_upgrade = dist_upgrade + " 2>&1 | tee -a {}".format(logfile)
MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock"
wait_until_end_of_yunohost_command = (
"(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK))
mark_success = (
"(echo 'Done!' | tee -a {} && echo 'success: true' >> {})".
format(logfile, operation_logger.md_path))
mark_failure = (
"(echo 'Failed :(' | tee -a {} && echo 'success: false' >> {})"
.format(logfile, operation_logger.md_path))
update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}"
update_log_metadata = update_log_metadata.format(
operation_logger.md_path)
# Dirty hack such that the operation_logger does not add ended_at
# and success keys in the log metadata. (c.f. the code of the
# is_unit_operation + operation_logger.close()) We take care of
# this ourselves (c.f. the mark_success and updated_log_metadata in
# the huge command launched by os.system)
operation_logger.ended_at = "notyet"
upgrade_completed = "\n" + m18n.n(
"tools_upgrade_special_packages_completed")
command = "({wait} && {dist_upgrade}) && {mark_success} || {mark_failure}; {update_metadata}; echo '{done}'".format(
wait=wait_until_end_of_yunohost_command,
dist_upgrade=dist_upgrade,
mark_success=mark_success,
mark_failure=mark_failure,
update_metadata=update_log_metadata,
done=upgrade_completed,
)
logger.warning(
m18n.n("tools_upgrade_special_packages_explanation"))
logger.debug("Running command :\n{}".format(command))
open("/tmp/yunohost-selfupgrade",
"w").write("rm /tmp/yunohost-selfupgrade; " + command)
# Using systemd-run --scope is like nohup/disown and &, but more robust somehow
# (despite using nohup/disown and &, the self-upgrade process was still getting killed...)
# ref: https://unix.stackexchange.com/questions/420594/why-process-killed-with-nohup
# (though I still don't understand it 100%...)
os.system("systemd-run --scope bash /tmp/yunohost-selfupgrade &")
return
else:
logger.success(m18n.n("system_upgraded"))
operation_logger.success()
def tools_upgrade(auth, ignore_apps=False, ignore_packages=False):
"""
Update apps & package cache, then display changelog
Keyword arguments:
ignore_apps -- Ignore apps upgrade
ignore_packages -- Ignore APT packages upgrade
"""
failure = False
# Retrieve interface
is_api = True if msettings.get('interface') == 'api' else False
if not ignore_packages:
cache = apt.Cache()
cache.open(None)
cache.upgrade(True)
# If API call
if is_api:
critical_packages = ("moulinette", "yunohost", "yunohost-admin",
"ssowat", "python")
critical_upgrades = set()
for pkg in cache.get_changes():
if pkg.name in critical_packages:
critical_upgrades.add(pkg.name)
# Temporarily keep package ...
pkg.mark_keep()
# ... and set a hourly cron up to upgrade critical packages
if critical_upgrades:
logger.info(
m18n.n('packages_upgrade_critical_later',
packages=', '.join(critical_upgrades)))
with open('/etc/cron.d/yunohost-upgrade', 'w+') as f:
f.write(
'00 * * * * root PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin apt-get install %s -y && rm -f /etc/cron.d/yunohost-upgrade\n'
% ' '.join(critical_upgrades))
if cache.get_changes():
logger.info(m18n.n('upgrading_packages'))
try:
# Apply APT changes
# TODO: Logs output for the API
cache.commit(apt.progress.text.AcquireProgress(),
apt.progress.base.InstallProgress())
except Exception as e:
failure = True
logger.warning('unable to upgrade packages: %s' % str(e))
logger.error(m18n.n('packages_upgrade_failed'))
else:
logger.info(m18n.n('done'))
else:
logger.info(m18n.n('packages_no_upgrade'))
if not ignore_apps:
try:
app_upgrade(auth)
except Exception as e:
failure = True
logger.warning('unable to upgrade apps: %s' % str(e))
logger.error(m18n.n('app_upgrade_failed'))
if not failure:
logger.success(m18n.n('system_upgraded'))
# Return API logs if it is an API call
if is_api:
return {"log": service_log('yunohost-api', number="100").values()[0]}
def tools_upgrade(ignore_apps=False, ignore_packages=False):
"""
Update apps & package cache, then display changelog
Keyword arguments:
ignore_apps -- Ignore apps upgrade
ignore_packages -- Ignore APT packages upgrade
"""
from yunohost.app import app_upgrade
is_api = True if msettings.get('interface') == 'api' else False
if not ignore_packages:
cache = apt.Cache()
cache.open(None)
cache.upgrade(True)
# If API call
if is_api:
critical_packages = ("moulinette", "moulinette-yunohost",
"yunohost-admin", "yunohost-config-nginx", "ssowat", "python")
critical_upgrades = set()
for pkg in cache.get_changes():
if pkg.name in critical_packages:
critical_upgrades.add(pkg.name)
# Temporarily keep package ...
pkg.mark_keep()
# ... and set a hourly cron up to upgrade critical packages
if critical_upgrades:
msignals.display(m18n.n('packages_upgrade_critical_later')
% ', '.join(critical_upgrades))
with open('/etc/cron.d/yunohost-upgrade', 'w+') as f:
f.write('00 * * * * root PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin apt-get install %s -y && rm -f /etc/cron.d/yunohost-upgrade' % ' '.join(critical_upgrades))
if cache.get_changes():
msignals.display(m18n.n('upgrading_packages'))
try:
# Apply APT changes
# TODO: Logs output for the API
cache.commit(apt.progress.text.AcquireProgress(),
apt.progress.base.InstallProgress())
except Exception as e:
logging.warning('unable to upgrade packages: %s' % str(e))
msignals.display(m18n.n('packages_upgrade_failed'), 'error')
else:
msignals.display(m18n.n('done'))
else:
msignals.display(m18n.n('packages_no_upgrade'))
if not ignore_apps:
try:
app_upgrade()
except: pass
msignals.display(m18n.n('system_upgraded'), 'success')
# Return API logs if it is an API call
if msettings.get('interface') == 'api':
from yunohost.service import service_log
return { "log": service_log('yunohost-api', number="100").values()[0] }
