def app_is_installed(app):
if app == "permissions_app":
return _is_installed(app)
# These are files we know should be installed by the app
app_files = []
app_files.append("/etc/nginx/conf.d/%s.d/%s.conf" % (maindomain, app))
app_files.append("/var/www/%s/index.html" % app)
app_files.append("/etc/importantfile")
return _is_installed(app) and all(os.path.exists(f) for f in app_files)
def test_restore_app_not_in_backup(mocker):
assert not _is_installed("wordpress")
assert not _is_installed("yoloswag")
with message(mocker, "backup_archive_app_not_found", app="yoloswag"):
with raiseYunohostError(mocker, "restore_nothings_done"):
backup_restore(
system=None, name=backup_list()["archives"][0], apps=["yoloswag"]
)
assert not _is_installed("wordpress")
assert not _is_installed("yoloswag")
def test_restore_app_not_enough_free_space(monkeypatch, mocker):
def custom_free_space_in_directory(dirpath):
return 0
monkeypatch.setattr("yunohost.backup.free_space_in_directory",
custom_free_space_in_directory)
assert not _is_installed("wordpress")
with raiseYunohostError(mocker, 'restore_not_enough_disk_space'):
backup_restore(system=None,
name=backup_list()["archives"][0],
apps=["wordpress"])
assert not _is_installed("wordpress")
def test_backup_app_not_installed(mocker):
assert not _is_installed("wordpress")
with message(mocker, "unbackup_app", app="wordpress"):
with raiseYunohostError(mocker, "backup_nothings_done"):
backup_create(system=None, apps=["wordpress"])
def test_restore_app_already_installed(mocker):
assert not _is_installed("wordpress")
with message(mocker, "restore_complete"):
backup_restore(
system=None, name=backup_list()["archives"][0], apps=["wordpress"]
)
assert _is_installed("wordpress")
with raiseYunohostError(mocker, "restore_already_installed_apps"):
backup_restore(
system=None, name=backup_list()["archives"][0], apps=["wordpress"]
)
assert _is_installed("wordpress")
def test_restore_app_script_failure_handling(monkeypatch, mocker):
def custom_hook_exec(name, *args, **kwargs):
if os.path.basename(name).startswith("restore"):
monkeypatch.undo()
raise Exception
monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec)
assert not _is_installed("wordpress")
with message(mocker, "restore_app_failed", app="wordpress"):
with raiseYunohostError(mocker, "restore_nothings_done"):
backup_restore(
system=None, name=backup_list()["archives"][0], apps=["wordpress"]
)
assert not _is_installed("wordpress")
def uninstall_test_apps_if_needed():
for app in [
"legacy_app", "backup_recommended_app", "wordpress",
"permissions_app"
]:
if _is_installed(app):
app_remove(app)
def run(self):
# Get list of php7.0 pool files
php70_pool_files = glob.glob("{}/*.conf".format(PHP70_POOLS))
# Keep only basenames
php70_pool_files = [os.path.basename(f) for f in php70_pool_files]
# Ignore the "www.conf" (default stuff, probably don't want to touch it ?)
php70_pool_files = [f for f in php70_pool_files if f != "www.conf"]
for f in php70_pool_files:
# Copy the files to the php7.3 pool
src = "{}/{}".format(PHP70_POOLS, f)
dest = "{}/{}".format(PHP73_POOLS, f)
copy2(src, dest)
# Replace the socket prefix if it's found
c = "sed -i -e 's@{}@{}@g' {}".format(PHP70_SOCKETS_PREFIX,
PHP73_SOCKETS_PREFIX, dest)
os.system(c)
# Also add a comment that it was automatically moved from php7.0
# (for human traceability and backward migration)
c = "sed -i '1i {}' {}".format(MIGRATION_COMMENT, dest)
os.system(c)
app_id = os.path.basename(f)[:-len(".conf")]
if _is_installed(app_id):
_patch_legacy_php_versions_in_settings(
"/etc/yunohost/apps/%s/" % app_id)
nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/%s.conf" %
app_id)
for f in nginx_conf_files:
# Replace the socket prefix if it's found
c = "sed -i -e 's@{}@{}@g' {}".format(PHP70_SOCKETS_PREFIX,
PHP73_SOCKETS_PREFIX, f)
os.system(c)
os.system(
"rm /etc/logrotate.d/php7.0-fpm"
) # We remove this otherwise the logrotate cron will be unhappy
# Reload/restart the php pools
_run_service_command("restart", "php7.3-fpm")
_run_service_command("enable", "php7.3-fpm")
os.system("systemctl stop php7.0-fpm")
os.system("systemctl disable php7.0-fpm")
# Reload nginx
_run_service_command("reload", "nginx")
def clean():
# Make sure we have a ssowat
os.system("mkdir -p /etc/ssowat/")
app_ssowatconf()
test_apps = ["break_yo_system", "legacy_app", "legacy_app__2", "full_domain_app"]
for test_app in test_apps:
if _is_installed(test_app):
app_remove(test_app)
for filepath in glob.glob("/etc/nginx/conf.d/*.d/*%s*" % test_app):
os.remove(filepath)
for folderpath in glob.glob("/etc/yunohost/apps/*%s*" % test_app):
shutil.rmtree(folderpath, ignore_errors=True)
for folderpath in glob.glob("/var/www/*%s*" % test_app):
shutil.rmtree(folderpath, ignore_errors=True)
os.system(
"bash -c \"mysql -u root --password=$(cat /etc/yunohost/mysql) 2>/dev/null <<< 'DROP DATABASE %s' \""
% test_app
)
os.system(
"bash -c \"mysql -u root --password=$(cat /etc/yunohost/mysql) 2>/dev/null <<< 'DROP USER %s@localhost'\""
% test_app
)
os.system(
"systemctl reset-failed nginx"
) # Reset failed quota for service to avoid running into start-limit rate ?
os.system("systemctl start nginx")
# Clean permissions
for permission_name in user_permission_list(short=True)["permissions"]:
if any(test_app in permission_name for test_app in test_apps):
permission_delete(permission_name, force=True)
def app_is_not_installed(domain, app):
return not _is_installed(app) and not all(
os.path.exists(f) for f in app_expected_files(domain, app)
)
def backup_restore(auth,
name,
hooks=[],
ignore_hooks=False,
apps=[],
ignore_apps=False,
force=False):
"""
Restore from a local backup archive
Keyword argument:
name -- Name of the local backup archive
hooks -- List of restoration hooks names to execute
ignore_hooks -- Do not execute backup hooks
apps -- List of application names to restore
ignore_apps -- Do not restore apps
force -- Force restauration on an already installed system
"""
# Validate what to restore
if ignore_hooks and ignore_apps:
raise MoulinetteError(errno.EINVAL, m18n.n('restore_action_required'))
# Retrieve and open the archive
info = backup_info(name)
archive_file = info['path']
try:
tar = tarfile.open(archive_file, "r:gz")
except:
logger.debug("cannot open backup archive '%s'",
archive_file,
exc_info=1)
raise MoulinetteError(errno.EIO, m18n.n('backup_archive_open_failed'))
# Check temporary directory
tmp_dir = "%s/tmp/%s" % (backup_path, name)
if os.path.isdir(tmp_dir):
logger.debug("temporary directory for restoration '%s' already exists",
tmp_dir)
os.system('rm -rf %s' % tmp_dir)
# Check available disk space
statvfs = os.statvfs(backup_path)
free_space = statvfs.f_frsize * statvfs.f_bavail
if free_space < info['size']:
logger.debug("%dB left but %dB is needed", free_space, info['size'])
raise MoulinetteError(
errno.EIO, m18n.n('not_enough_disk_space', path=backup_path))
def _clean_tmp_dir(retcode=0):
ret = hook_callback('post_backup_restore', args=[tmp_dir, retcode])
if not ret['failed']:
filesystem.rm(tmp_dir, True, True)
else:
logger.warning(m18n.n('restore_cleaning_failed'))
# Extract the tarball
logger.info(m18n.n('backup_extracting_archive'))
tar.extractall(tmp_dir)
tar.close()
# Retrieve backup info
info_file = "%s/info.json" % tmp_dir
try:
with open(info_file, 'r') as f:
info = json.load(f)
except IOError:
logger.debug("unable to load '%s'", info_file, exc_info=1)
raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))
else:
logger.debug("restoring from backup '%s' created on %s", name,
time.ctime(info['created_at']))
# Initialize restauration summary result
result = {
'apps': [],
'hooks': {},
}
# Check if YunoHost is installed
if os.path.isfile('/etc/yunohost/installed'):
logger.warning(m18n.n('yunohost_already_installed'))
if not force:
try:
# Ask confirmation for restoring
i = msignals.prompt(
m18n.n('restore_confirm_yunohost_installed',
answers='y/N'))
except NotImplemented:
pass
else:
if i == 'y' or i == 'Y':
force = True
if not force:
_clean_tmp_dir()
raise MoulinetteError(errno.EEXIST, m18n.n('restore_failed'))
else:
# Retrieve the domain from the backup
try:
with open("%s/conf/ynh/current_host" % tmp_dir, 'r') as f:
domain = f.readline().rstrip()
except IOError:
logger.debug("unable to retrieve current_host from the backup",
exc_info=1)
raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))
logger.debug("executing the post-install...")
tools_postinstall(domain, 'yunohost', True)
# Run system hooks
if not ignore_hooks:
# Filter hooks to execute
hooks_list = set(info['hooks'].keys())
_is_hook_in_backup = lambda h: True
if hooks:
def _is_hook_in_backup(h):
if h in hooks_list:
return True
logger.error(m18n.n('backup_archive_hook_not_exec', hook=h))
return False
else:
hooks = hooks_list
# Check hooks availibility
hooks_filtered = set()
for h in hooks:
if not _is_hook_in_backup(h):
continue
try:
hook_info('restore', h)
except:
tmp_hooks = glob('{:s}/hooks/restore/*-{:s}'.format(
tmp_dir, h))
if not tmp_hooks:
logger.exception(m18n.n('restore_hook_unavailable',
hook=h))
continue
# Add restoration hook from the backup to the system
# FIXME: Refactor hook_add and use it instead
restore_hook_folder = custom_hook_folder + 'restore'
filesystem.mkdir(restore_hook_folder, 755, True)
for f in tmp_hooks:
logger.debug(
"adding restoration hook '%s' to the system "
"from the backup archive '%s'", f, archive_file)
shutil.copy(f, restore_hook_folder)
hooks_filtered.add(h)
if hooks_filtered:
logger.info(m18n.n('restore_running_hooks'))
ret = hook_callback('restore', hooks_filtered, args=[tmp_dir])
result['hooks'] = ret['succeed']
# Add apps restore hook
if not ignore_apps:
# Filter applications to restore
apps_list = set(info['apps'].keys())
apps_filtered = set()
if apps:
for a in apps:
if a not in apps_list:
logger.error(m18n.n('backup_archive_app_not_found', app=a))
else:
apps_filtered.add(a)
else:
apps_filtered = apps_list
for app_instance_name in apps_filtered:
tmp_app_dir = '{:s}/apps/{:s}'.format(tmp_dir, app_instance_name)
tmp_app_bkp_dir = tmp_app_dir + '/backup'
# Parse app instance name and id
# TODO: Use app_id to check if app is installed?
app_id, app_instance_nb = _parse_app_instance_name(
app_instance_name)
# Check if the app is not already installed
if _is_installed(app_instance_name):
logger.error(
m18n.n('restore_already_installed_app',
app=app_instance_name))
continue
# Check if the app has a restore script
app_script = tmp_app_dir + '/settings/scripts/restore'
if not os.path.isfile(app_script):
logger.warning(m18n.n('unrestore_app', app=app_instance_name))
continue
tmp_script = '/tmp/restore_' + app_instance_name
app_setting_path = '/etc/yunohost/apps/' + app_instance_name
logger.info(
m18n.n('restore_running_app_script', app=app_instance_name))
try:
# Copy app settings and set permissions
# TODO: Copy app hooks too
shutil.copytree(tmp_app_dir + '/settings', app_setting_path)
filesystem.chmod(app_setting_path, 0555, 0444, True)
filesystem.chmod(app_setting_path + '/settings.yml', 0400)
# Copy restore script in a tmp file
subprocess.call(['install', '-Dm555', app_script, tmp_script])
# Prepare env. var. to pass to script
env_dict = {}
env_dict["YNH_APP_ID"] = app_id
env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name
env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
env_dict["YNH_APP_BACKUP_DIR"] = tmp_app_bkp_dir
# Execute app restore script
hook_exec(tmp_script,
args=[tmp_app_bkp_dir, app_instance_name],
raise_on_error=True,
chdir=tmp_app_bkp_dir,
env=env_dict)
except:
logger.exception(
m18n.n('restore_app_failed', app=app_instance_name))
# Copy remove script in a tmp file
filesystem.rm(tmp_script, force=True)
app_script = tmp_app_dir + '/settings/scripts/remove'
tmp_script = '/tmp/remove_' + app_instance_name
subprocess.call(['install', '-Dm555', app_script, tmp_script])
# Setup environment for remove script
env_dict_remove = {}
env_dict_remove["YNH_APP_ID"] = app_id
env_dict_remove["YNH_APP_INSTANCE_NAME"] = app_instance_name
env_dict_remove["YNH_APP_INSTANCE_NUMBER"] = str(
app_instance_nb)
# Execute remove script
# TODO: call app_remove instead
if hook_exec(tmp_script,
args=[app_instance_name],
env=env_dict_remove) != 0:
logger.warning(
m18n.n('app_not_properly_removed',
app=app_instance_name))
# Cleaning app directory
shutil.rmtree(app_setting_path, ignore_errors=True)
else:
result['apps'].append(app_instance_name)
finally:
filesystem.rm(tmp_script, force=True)
# Check if something has been restored
if not result['hooks'] and not result['apps']:
_clean_tmp_dir(1)
raise MoulinetteError(errno.EINVAL, m18n.n('restore_nothings_done'))
if result['apps']:
app_ssowatconf(auth)
_clean_tmp_dir()
logger.success(m18n.n('restore_complete'))
return result
def backup_restore(auth, name, hooks=[], ignore_hooks=False,
apps=[], ignore_apps=False, force=False):
"""
Restore from a local backup archive
Keyword argument:
name -- Name of the local backup archive
hooks -- List of restoration hooks names to execute
ignore_hooks -- Do not execute backup hooks
apps -- List of application names to restore
ignore_apps -- Do not restore apps
force -- Force restauration on an already installed system
"""
# Validate what to restore
if ignore_hooks and ignore_apps:
raise MoulinetteError(errno.EINVAL,
m18n.n('restore_action_required'))
# Retrieve and open the archive
info = backup_info(name)
archive_file = info['path']
try:
tar = tarfile.open(archive_file, "r:gz")
except:
logger.debug("cannot open backup archive '%s'",
archive_file, exc_info=1)
raise MoulinetteError(errno.EIO, m18n.n('backup_archive_open_failed'))
# Check temporary directory
tmp_dir = "%s/tmp/%s" % (backup_path, name)
if os.path.isdir(tmp_dir):
logger.debug("temporary directory for restoration '%s' already exists",
tmp_dir)
os.system('rm -rf %s' % tmp_dir)
# Check available disk space
statvfs = os.statvfs(backup_path)
free_space = statvfs.f_frsize * statvfs.f_bavail
if free_space < info['size']:
logger.debug("%dB left but %dB is needed", free_space, info['size'])
raise MoulinetteError(
errno.EIO, m18n.n('not_enough_disk_space', path=backup_path))
def _clean_tmp_dir(retcode=0):
ret = hook_callback('post_backup_restore', args=[tmp_dir, retcode])
if not ret['failed']:
filesystem.rm(tmp_dir, True, True)
else:
logger.warning(m18n.n('restore_cleaning_failed'))
# Extract the tarball
logger.info(m18n.n('backup_extracting_archive'))
tar.extractall(tmp_dir)
tar.close()
# Retrieve backup info
info_file = "%s/info.json" % tmp_dir
try:
with open(info_file, 'r') as f:
info = json.load(f)
except IOError:
logger.debug("unable to load '%s'", info_file, exc_info=1)
raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))
else:
logger.debug("restoring from backup '%s' created on %s", name,
time.ctime(info['created_at']))
# Initialize restauration summary result
result = {
'apps': [],
'hooks': {},
}
# Check if YunoHost is installed
if os.path.isfile('/etc/yunohost/installed'):
logger.warning(m18n.n('yunohost_already_installed'))
if not force:
try:
# Ask confirmation for restoring
i = msignals.prompt(m18n.n('restore_confirm_yunohost_installed',
answers='y/N'))
except NotImplemented:
pass
else:
if i == 'y' or i == 'Y':
force = True
if not force:
_clean_tmp_dir()
raise MoulinetteError(errno.EEXIST, m18n.n('restore_failed'))
else:
# Retrieve the domain from the backup
try:
with open("%s/conf/ynh/current_host" % tmp_dir, 'r') as f:
domain = f.readline().rstrip()
except IOError:
logger.debug("unable to retrieve current_host from the backup",
exc_info=1)
raise MoulinetteError(errno.EIO, m18n.n('backup_invalid_archive'))
logger.debug("executing the post-install...")
tools_postinstall(domain, 'yunohost', True)
# Run system hooks
if not ignore_hooks:
# Filter hooks to execute
hooks_list = set(info['hooks'].keys())
_is_hook_in_backup = lambda h: True
if hooks:
def _is_hook_in_backup(h):
if h in hooks_list:
return True
logger.error(m18n.n('backup_archive_hook_not_exec', hook=h))
return False
else:
hooks = hooks_list
# Check hooks availibility
hooks_filtered = set()
for h in hooks:
if not _is_hook_in_backup(h):
continue
try:
hook_info('restore', h)
except:
tmp_hooks = glob('{:s}/hooks/restore/*-{:s}'.format(tmp_dir, h))
if not tmp_hooks:
logger.exception(m18n.n('restore_hook_unavailable', hook=h))
continue
# Add restoration hook from the backup to the system
# FIXME: Refactor hook_add and use it instead
restore_hook_folder = custom_hook_folder + 'restore'
filesystem.mkdir(restore_hook_folder, 755, True)
for f in tmp_hooks:
logger.debug("adding restoration hook '%s' to the system "
"from the backup archive '%s'", f, archive_file)
shutil.copy(f, restore_hook_folder)
hooks_filtered.add(h)
if hooks_filtered:
logger.info(m18n.n('restore_running_hooks'))
ret = hook_callback('restore', hooks_filtered, args=[tmp_dir])
result['hooks'] = ret['succeed']
# Add apps restore hook
if not ignore_apps:
# Filter applications to restore
apps_list = set(info['apps'].keys())
apps_filtered = set()
if apps:
for a in apps:
if a not in apps_list:
logger.error(m18n.n('backup_archive_app_not_found', app=a))
else:
apps_filtered.add(a)
else:
apps_filtered = apps_list
for app_instance_name in apps_filtered:
tmp_app_dir = '{:s}/apps/{:s}'.format(tmp_dir, app_instance_name)
tmp_app_bkp_dir = tmp_app_dir + '/backup'
# Check if the app is not already installed
if _is_installed(app_instance_name):
logger.error(m18n.n('restore_already_installed_app',
app=app_instance_name))
continue
# Check if the app has a restore script
app_script = tmp_app_dir + '/settings/scripts/restore'
if not os.path.isfile(app_script):
logger.warning(m18n.n('unrestore_app', app=app_instance_name))
continue
tmp_script = '/tmp/restore_' + app_instance_name
app_setting_path = '/etc/yunohost/apps/' + app_instance_name
logger.info(m18n.n('restore_running_app_script', app=app_instance_name))
try:
# Copy app settings and set permissions
shutil.copytree(tmp_app_dir + '/settings', app_setting_path)
filesystem.chmod(app_setting_path, 0555, 0444, True)
filesystem.chmod(app_setting_path + '/settings.yml', 0400)
# Execute app restore script
subprocess.call(['install', '-Dm555', app_script, tmp_script])
# Prepare env. var. to pass to script
env_dict = {}
app_id, app_instance_nb = _parse_app_instance_name(app_instance_name)
env_dict["YNH_APP_ID"] = app_id
env_dict["YNH_APP_INSTANCE_NAME"] = app_instance_name
env_dict["YNH_APP_INSTANCE_NUMBER"] = str(app_instance_nb)
env_dict["YNH_APP_BACKUP_DIR"] = tmp_app_bkp_dir
hook_exec(tmp_script, args=[tmp_app_bkp_dir, app_instance_name],
raise_on_error=True, chdir=tmp_app_bkp_dir, env=env_dict)
except:
logger.exception(m18n.n('restore_app_failed', app=app_instance_name))
# Cleaning app directory
shutil.rmtree(app_setting_path, ignore_errors=True)
else:
result['apps'].append(app_instance_name)
finally:
filesystem.rm(tmp_script, force=True)
# Check if something has been restored
if not result['hooks'] and not result['apps']:
_clean_tmp_dir(1)
raise MoulinetteError(errno.EINVAL, m18n.n('restore_nothings_done'))
if result['apps']:
app_ssowatconf(auth)
_clean_tmp_dir()
logger.success(m18n.n('restore_complete'))
return result
