def diagnosis_show(categories=[],
issues=False,
full=False,
share=False,
human_readable=False):
if not os.path.exists(DIAGNOSIS_CACHE):
logger.warning(m18n.n("diagnosis_never_ran_yet"))
return
# Get all the categories
all_categories = _list_diagnosis_categories()
all_categories_names = [category for category, _ in all_categories]
# Check the requested category makes sense
if categories == []:
categories = all_categories_names
else:
unknown_categories = [
c for c in categories if c not in all_categories_names
]
if unknown_categories:
raise YunohostValidationError(
"diagnosis_unknown_categories",
categories=", ".join(unknown_categories))
# Fetch all reports
all_reports = []
for category in categories:
try:
report = Diagnoser.get_cached_report(category)
except Exception as e:
logger.error(
m18n.n("diagnosis_failed", category=category, error=str(e)))
continue
Diagnoser.i18n(report, force_remove_html_tags=share or human_readable)
add_ignore_flag_to_issues(report)
if not full:
del report["timestamp"]
del report["cached_for"]
report["items"] = [
item for item in report["items"] if not item["ignored"]
]
for item in report["items"]:
del item["meta"]
del item["ignored"]
if "data" in item:
del item["data"]
if issues:
report["items"] = [
item for item in report["items"]
if item["status"] in ["WARNING", "ERROR"]
]
# Ignore this category if no issue was found
if not report["items"]:
continue
all_reports.append(report)
if share:
from yunohost.utils.yunopaste import yunopaste
content = _dump_human_readable_reports(all_reports)
url = yunopaste(content)
logger.info(m18n.n("log_available_on_yunopaste", url=url))
if msettings.get("interface") == "api":
return {"url": url}
else:
return
elif human_readable:
print(_dump_human_readable_reports(all_reports))
else:
return {"reports": all_reports}
def _get_settings():
def get_setting_description(key):
if key.startswith("example"):
# (This is for dummy stuff used during unit tests)
return "Dummy %s setting" % key.split(".")[-1]
return m18n.n("global_settings_setting_%s" % key.replace(".", "_"))
settings = {}
for key, value in DEFAULTS.copy().items():
settings[key] = value
settings[key]["value"] = value["default"]
settings[key]["description"] = get_setting_description(key)
if not os.path.exists(SETTINGS_PATH):
return settings
# we have a very strict policy on only allowing settings that we know in
# the OrderedDict DEFAULTS
# For various reason, while reading the local settings we might encounter
# settings that aren't in DEFAULTS, those can come from settings key that
# we have removed, errors or the user trying to modify
# /etc/yunohost/settings.json
# To avoid to simply overwrite them, we store them in
# /etc/yunohost/settings-unknown.json in case of
unknown_settings = {}
unknown_settings_path = SETTINGS_PATH_OTHER_LOCATION % "unknown"
if os.path.exists(unknown_settings_path):
try:
unknown_settings = json.load(open(unknown_settings_path, "r"))
except Exception as e:
logger.warning("Error while loading unknown settings %s" % e)
try:
with open(SETTINGS_PATH) as settings_fd:
local_settings = json.load(settings_fd)
for key, value in local_settings.items():
if key in settings:
settings[key] = value
settings[key]["description"] = get_setting_description(key)
else:
logger.warning(
m18n.n(
"global_settings_unknown_setting_from_settings_file",
setting_key=key,
))
unknown_settings[key] = value
except Exception as e:
raise YunohostValidationError("global_settings_cant_open_settings",
reason=e)
if unknown_settings:
try:
_save_settings(unknown_settings, location=unknown_settings_path)
_save_settings(settings)
except Exception as e:
logger.warning(
"Failed to save unknown settings (because %s), aborting." % e)
return settings
def _diagnosis_ignore(add_filter=None, remove_filter=None, list=False):
"""
This action is meant for the admin to ignore issues reported by the
diagnosis system if they are known and understood by the admin. For
example, the lack of ipv6 on an instance, or badly configured XMPP dns
records if the admin doesn't care so much about XMPP. The point being that
the diagnosis shouldn't keep complaining about those known and "expected"
issues, and instead focus on new unexpected issues that could arise.
For example, to ignore badly XMPP dnsrecords for domain yolo.test:
yunohost diagnosis ignore --add-filter dnsrecords domain=yolo.test category=xmpp
^ ^ ^
the general additional other
diagnosis criterias criteria
category to to target to target
act on specific specific
reports reports
Or to ignore all dnsrecords issues:
yunohost diagnosis ignore --add-filter dnsrecords
The filters are stored in the diagnosis configuration in a data structure like:
ignore_filters: {
"ip": [
{"version": 6} # Ignore all issues related to ipv6
],
"dnsrecords": [
{"domain": "yolo.test", "category": "xmpp"}, # Ignore all issues related to DNS xmpp records for yolo.test
{} # Ignore all issues about dnsrecords
]
}
"""
# Ignore filters are stored in
configuration = _diagnosis_read_configuration()
if list:
return {"ignore_filters": configuration.get("ignore_filters", {})}
def validate_filter_criterias(filter_):
# Get all the categories
all_categories = _list_diagnosis_categories()
all_categories_names = [category for category, _ in all_categories]
# Sanity checks for the provided arguments
if len(filter_) == 0:
raise YunohostValidationError(
"You should provide at least one criteria being the diagnosis category to ignore"
)
category = filter_[0]
if category not in all_categories_names:
raise YunohostValidationError("%s is not a diagnosis category" %
category)
if any("=" not in criteria for criteria in filter_[1:]):
raise YunohostValidationError(
"Criterias should be of the form key=value (e.g. domain=yolo.test)"
)
# Convert the provided criteria into a nice dict
criterias = {c.split("=")[0]: c.split("=")[1] for c in filter_[1:]}
return category, criterias
if add_filter:
category, criterias = validate_filter_criterias(add_filter)
# Fetch current issues for the requested category
current_issues_for_this_category = diagnosis_show(
categories=[category], issues=True, full=True)
current_issues_for_this_category = current_issues_for_this_category[
"reports"][0].get("items", {})
# Accept the given filter only if the criteria effectively match an existing issue
if not any(
issue_matches_criterias(i, criterias)
for i in current_issues_for_this_category):
raise YunohostError(
"No issues was found matching the given criteria.")
# Make sure the subdicts/lists exists
if "ignore_filters" not in configuration:
configuration["ignore_filters"] = {}
if category not in configuration["ignore_filters"]:
configuration["ignore_filters"][category] = []
if criterias in configuration["ignore_filters"][category]:
logger.warning("This filter already exists.")
return
configuration["ignore_filters"][category].append(criterias)
_diagnosis_write_configuration(configuration)
logger.success("Filter added")
return
if remove_filter:
category, criterias = validate_filter_criterias(remove_filter)
# Make sure the subdicts/lists exists
if "ignore_filters" not in configuration:
configuration["ignore_filters"] = {}
if category not in configuration["ignore_filters"]:
configuration["ignore_filters"][category] = []
if criterias not in configuration["ignore_filters"][category]:
raise YunohostValidationError("This filter does not exists.")
configuration["ignore_filters"][category].remove(criterias)
_diagnosis_write_configuration(configuration)
logger.success("Filter removed")
return
def log_show(path,
number=None,
share=False,
filter_irrelevant=False,
with_suboperations=False):
"""
Display a log file enriched with metadata if any.
If the file_name is not an absolute path, it will try to search the file in
the unit operations log path (see OPERATIONS_PATH).
Argument:
file_name
number
share
"""
if share:
filter_irrelevant = True
if filter_irrelevant:
filters = [
r"set [+-]x$",
r"set [+-]o xtrace$",
r"local \w+$",
r"local legacy_args=.*$",
r".*Helper used in legacy mode.*",
r"args_array=.*$",
r"local -A args_array$",
r"ynh_handle_getopts_args",
r"ynh_script_progression",
]
else:
filters = []
def _filter_lines(lines, filters=[]):
filters = [re.compile(f) for f in filters]
return [
line for line in lines
if not any(f.search(line.strip()) for f in filters)
]
# Normalize log/metadata paths and filenames
abs_path = path
log_path = None
if not path.startswith("/"):
abs_path = os.path.join(OPERATIONS_PATH, path)
if os.path.exists(abs_path) and not path.endswith(METADATA_FILE_EXT):
log_path = abs_path
if abs_path.endswith(METADATA_FILE_EXT) or abs_path.endswith(LOG_FILE_EXT):
base_path = "".join(os.path.splitext(abs_path)[:-1])
else:
base_path = abs_path
base_filename = os.path.basename(base_path)
md_path = base_path + METADATA_FILE_EXT
if log_path is None:
log_path = base_path + LOG_FILE_EXT
if not os.path.exists(md_path) and not os.path.exists(log_path):
raise YunohostValidationError("log_does_exists", log=path)
infos = {}
# If it's a unit operation, display the name and the description
if base_path.startswith(CATEGORIES_PATH):
infos["description"] = _get_description_from_name(base_filename)
infos["name"] = base_filename
if share:
from yunohost.utils.yunopaste import yunopaste
content = ""
if os.path.exists(md_path):
content += read_file(md_path)
content += "\n============\n\n"
if os.path.exists(log_path):
actual_log = read_file(log_path)
content += "\n".join(_filter_lines(actual_log.split("\n"),
filters))
url = yunopaste(content)
logger.info(m18n.n("log_available_on_yunopaste", url=url))
if msettings.get("interface") == "api":
return {"url": url}
else:
return
# Display metadata if exist
if os.path.exists(md_path):
try:
metadata = read_yaml(md_path)
except MoulinetteError as e:
error = m18n.n("log_corrupted_md_file", md_file=md_path, error=e)
if os.path.exists(log_path):
logger.warning(error)
else:
raise YunohostError(error)
else:
infos["metadata_path"] = md_path
infos["metadata"] = metadata
if "log_path" in metadata:
log_path = metadata["log_path"]
if with_suboperations:
def suboperations():
try:
log_start = _get_datetime_from_name(base_filename)
except ValueError:
return
for filename in os.listdir(OPERATIONS_PATH):
if not filename.endswith(METADATA_FILE_EXT):
continue
# We first retrict search to a ~48h time window to limit the number
# of .yml we look into
try:
date = _get_datetime_from_name(base_filename)
except ValueError:
continue
if (date < log_start) or (
date > log_start + timedelta(hours=48)):
continue
try:
submetadata = read_yaml(
os.path.join(OPERATIONS_PATH, filename))
except Exception:
continue
if submetadata and submetadata.get(
"parent") == base_filename:
yield {
"name":
filename[:-len(METADATA_FILE_EXT)],
"description":
_get_description_from_name(
filename[:-len(METADATA_FILE_EXT)]),
"success":
submetadata.get("success", "?"),
}
metadata["suboperations"] = list(suboperations())
# Display logs if exist
if os.path.exists(log_path):
from yunohost.service import _tail
if number and filters:
logs = _tail(log_path, int(number * 4))
elif number:
logs = _tail(log_path, int(number))
else:
logs = read_file(log_path)
logs = _filter_lines(logs, filters)
if number:
logs = logs[-number:]
infos["log_path"] = log_path
infos["logs"] = logs
return infos
def permission_create(
operation_logger,
permission,
allowed=None,
url=None,
additional_urls=None,
auth_header=True,
label=None,
show_tile=False,
protected=False,
sync_perm=True,
):
"""
Create a new permission for a specific application
Keyword argument:
permission -- Name of the permission (e.g. mail or nextcloud or wordpress.editors)
allowed -- (optional) List of group/user to allow for the permission
url -- (optional) URL for which access will be allowed/forbidden
additional_urls -- (optional) List of additional URL for which access will be allowed/forbidden
auth_header -- (optional) Define for the URL of this permission, if SSOwat pass the authentication header to the application
label -- (optional) Define a name for the permission. This label will be shown on the SSO and in the admin. Default is "permission name"
show_tile -- (optional) Define if a tile will be shown in the SSO
protected -- (optional) Define if the permission can be added/removed to the visitor group
If provided, 'url' is assumed to be relative to the app domain/path if they
start with '/'. For example:
/ -> domain.tld/app
/admin -> domain.tld/app/admin
domain.tld/app/api -> domain.tld/app/api
'url' can be later treated as a regex if it starts with "re:".
For example:
re:/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$
re:domain.tld/app/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$
"""
from yunohost.utils.ldap import _get_ldap_interface
from yunohost.user import user_group_list
ldap = _get_ldap_interface()
# By default, manipulate main permission
if "." not in permission:
permission = permission + ".main"
# Validate uniqueness of permission in LDAP
if ldap.get_conflict({"cn": permission},
base_dn="ou=permission,dc=yunohost,dc=org"):
raise YunohostValidationError("permission_already_exist",
permission=permission)
# Get random GID
all_gid = {x.gr_gid for x in grp.getgrall()}
uid_guid_found = False
while not uid_guid_found:
gid = str(random.randint(200, 99999))
uid_guid_found = gid not in all_gid
app, subperm = permission.split(".")
attr_dict = {
"objectClass": ["top", "permissionYnh", "posixGroup"],
"cn":
str(permission),
"gidNumber":
gid,
"authHeader": ["TRUE"],
"label": [
str(label) if label else
(subperm if subperm != "main" else app.title())
],
"showTile": [
"FALSE"
], # Dummy value, it will be fixed when we call '_update_ldap_group_permission'
"isProtected": [
"FALSE"
], # Dummy value, it will be fixed when we call '_update_ldap_group_permission'
}
if allowed is not None:
if not isinstance(allowed, list):
allowed = [allowed]
# Validate that the groups to add actually exist
all_existing_groups = user_group_list()["groups"].keys()
for group in allowed or []:
if group not in all_existing_groups:
raise YunohostValidationError("group_unknown", group=group)
operation_logger.related_to.append(("app", permission.split(".")[0]))
operation_logger.start()
try:
ldap.add("cn=%s,ou=permission" % permission, attr_dict)
except Exception as e:
raise YunohostError("permission_creation_failed",
permission=permission,
error=e)
permission_url(
permission,
url=url,
add_url=additional_urls,
auth_header=auth_header,
sync_perm=False,
)
new_permission = _update_ldap_group_permission(
permission=permission,
allowed=allowed,
label=label,
show_tile=show_tile,
protected=protected,
sync_perm=sync_perm,
)
logger.debug(m18n.n("permission_created", permission=permission))
return new_permission
def hook_list(action, list_by="name", show_info=False):
"""
List available hooks for an action
Keyword argument:
action -- Action name
list_by -- Property to list hook by
show_info -- Show hook information
"""
result = {}
# Process the property to list hook by
if list_by == "priority":
if show_info:
def _append_hook(d, priority, name, path):
# Use the priority as key and a dict of hooks names
# with their info as value
value = {"path": path}
try:
d[priority][name] = value
except KeyError:
d[priority] = {name: value}
else:
def _append_hook(d, priority, name, path):
# Use the priority as key and the name as value
try:
d[priority].add(name)
except KeyError:
d[priority] = set([name])
elif list_by == "name" or list_by == "folder":
if show_info:
def _append_hook(d, priority, name, path):
# Use the name as key and a list of hooks info - the
# executed ones with this name - as value
name_list = d.get(name, list())
for h in name_list:
# Only one priority for the hook is accepted
if h["priority"] == priority:
# Custom hooks overwrite system ones and they
# are appended at the end - so overwite it
if h["path"] != path:
h["path"] = path
return
name_list.append({"priority": priority, "path": path})
d[name] = name_list
else:
if list_by == "name":
result = set()
def _append_hook(d, priority, name, path):
# Add only the name
d.add(name)
else:
raise YunohostValidationError("hook_list_by_invalid")
def _append_folder(d, folder):
# Iterate over and add hook from a folder
for f in os.listdir(folder + action):
if (f[0] == "." or f[-1] == "~" or f.endswith(".pyc")
or (f.startswith("__") and f.endswith("__"))):
continue
path = "%s%s/%s" % (folder, action, f)
priority, name = _extract_filename_parts(f)
_append_hook(d, priority, name, path)
try:
# Append system hooks first
if list_by == "folder":
result["system"] = dict() if show_info else set()
_append_folder(result["system"], HOOK_FOLDER)
else:
_append_folder(result, HOOK_FOLDER)
except OSError:
pass
try:
# Append custom hooks
if list_by == "folder":
result["custom"] = dict() if show_info else set()
_append_folder(result["custom"], CUSTOM_HOOK_FOLDER)
else:
_append_folder(result, CUSTOM_HOOK_FOLDER)
except OSError:
pass
return {"hooks": result}
def service_log(name, number=50):
"""
Log every log files of a service
Keyword argument:
name -- Service name to log
number -- Number of lines to display
"""
services = _get_services()
number = int(number)
if name not in services.keys():
raise YunohostValidationError("service_unknown", service=name)
log_list = services[name].get("log", [])
if not isinstance(log_list, list):
log_list = [log_list]
# Legacy stuff related to --log_type where we'll typically have the service
# name in the log list but it's not an actual logfile. Nowadays journalctl
# is automatically fetch as well as regular log files.
if name in log_list:
log_list.remove(name)
result = {}
# First we always add the logs from journalctl / systemd
result["journalctl"] = _get_journalctl_logs(name, number).splitlines()
for log_path in log_list:
if not os.path.exists(log_path):
continue
# Make sure to resolve symlinks
log_path = os.path.realpath(log_path)
# log is a file, read it
if os.path.isfile(log_path):
result[log_path] = _tail(log_path, number)
continue
elif not os.path.isdir(log_path):
result[log_path] = []
continue
for log_file in os.listdir(log_path):
log_file_path = os.path.join(log_path, log_file)
# not a file : skip
if not os.path.isfile(log_file_path):
continue
if not log_file.endswith(".log"):
continue
result[log_file_path] = (
_tail(log_file_path, number) if os.path.exists(log_file_path) else []
)
return result
def tools_postinstall(
operation_logger,
domain,
password,
ignore_dyndns=False,
force_password=False,
force_diskspace=False,
):
"""
YunoHost post-install
Keyword argument:
domain -- YunoHost main domain
ignore_dyndns -- Do not subscribe domain to a DynDNS service (only
needed for nohost.me, noho.st domains)
password -- YunoHost admin password
"""
from yunohost.utils.password import assert_password_is_strong_enough
from yunohost.domain import domain_main_domain
import psutil
dyndns_provider = "dyndns.yunohost.org"
# Do some checks at first
if os.path.isfile("/etc/yunohost/installed"):
raise YunohostValidationError("yunohost_already_installed")
if os.path.isdir(
"/etc/yunohost/apps") and os.listdir("/etc/yunohost/apps") != []:
raise YunohostValidationError(
"It looks like you're trying to re-postinstall a system that was already working previously ... If you recently had some bug or issues with your installation, please first discuss with the team on how to fix the situation instead of savagely re-running the postinstall ...",
raw_msg=True,
)
# Check there's at least 10 GB on the rootfs...
disk_partitions = sorted(psutil.disk_partitions(),
key=lambda k: k.mountpoint)
main_disk_partitions = [
d for d in disk_partitions if d.mountpoint in ["/", "/var"]
]
main_space = sum(
[psutil.disk_usage(d.mountpoint).total for d in main_disk_partitions])
GB = 1024**3
if not force_diskspace and main_space < 10 * GB:
raise YunohostValidationError("postinstall_low_rootfsspace")
# Check password
if not force_password:
assert_password_is_strong_enough("admin", password)
if not ignore_dyndns:
# Check if yunohost dyndns can handle the given domain
# (i.e. is it a .nohost.me ? a .noho.st ?)
try:
is_nohostme_or_nohost = _dyndns_provides(dyndns_provider, domain)
# If an exception is thrown, most likely we don't have internet
# connectivity or something. Assume that this domain isn't manageable
# and inform the user that we could not contact the dyndns host server.
except Exception:
logger.warning(
m18n.n("dyndns_provider_unreachable",
provider=dyndns_provider))
is_nohostme_or_nohost = False
# If this is a nohost.me/noho.st, actually check for availability
if is_nohostme_or_nohost:
# (Except if the user explicitly said he/she doesn't care about dyndns)
if ignore_dyndns:
dyndns = False
# Check if the domain is available...
elif _dyndns_available(dyndns_provider, domain):
dyndns = True
# If not, abort the postinstall
else:
raise YunohostValidationError("dyndns_unavailable",
domain=domain)
else:
dyndns = False
else:
dyndns = False
if os.system("iptables -V >/dev/null 2>/dev/null") != 0:
raise YunohostValidationError(
"iptables/nftables does not seems to be working on your setup. You may be in a container or your kernel does have the proper modules loaded. Sometimes, rebooting the machine may solve the issue.",
raw_msg=True,
)
operation_logger.start()
logger.info(m18n.n("yunohost_installing"))
# New domain config
domain_add(domain, dyndns)
domain_main_domain(domain)
# Update LDAP admin and create home dir
tools_adminpw(password, check_strength=not force_password)
# Enable UPnP silently and reload firewall
firewall_upnp("enable", no_refresh=True)
# Initialize the apps catalog system
_initialize_apps_catalog_system()
# Try to update the apps catalog ...
# we don't fail miserably if this fails,
# because that could be for example an offline installation...
try:
_update_apps_catalog()
except Exception as e:
logger.warning(str(e))
# Init migrations (skip them, no need to run them on a fresh system)
_skip_all_migrations()
os.system("touch /etc/yunohost/installed")
# Enable and start YunoHost firewall at boot time
service_enable("yunohost-firewall")
service_start("yunohost-firewall")
regen_conf(names=["ssh"], force=True)
# Restore original ssh conf, as chosen by the
# admin during the initial install
#
# c.f. the install script and in particular
# https://github.com/YunoHost/install_script/pull/50
# The user can now choose during the install to keep
# the initial, existing sshd configuration
# instead of YunoHost's recommended conf
#
original_sshd_conf = "/etc/ssh/sshd_config.before_yunohost"
if os.path.exists(original_sshd_conf):
os.rename(original_sshd_conf, "/etc/ssh/sshd_config")
regen_conf(force=True)
logger.success(m18n.n("yunohost_configured"))
logger.warning(m18n.n("yunohost_postinstall_end_tip"))
def tools_upgrade(operation_logger,
target=None,
apps=False,
system=False,
allow_yunohost_upgrade=True):
"""
Update apps & package cache, then display changelog
Keyword arguments:
apps -- List of apps to upgrade (or [] to update all apps)
system -- True to upgrade system
"""
from yunohost.utils import packages
if packages.dpkg_is_broken():
raise YunohostValidationError("dpkg_is_broken")
# Check for obvious conflict with other dpkg/apt commands already running in parallel
if not packages.dpkg_lock_available():
raise YunohostValidationError("dpkg_lock_not_available")
# Legacy options management (--system, --apps)
if target is None:
logger.warning(
"Using 'yunohost tools upgrade' with --apps / --system is deprecated, just write 'yunohost tools upgrade apps' or 'system' (no -- prefix anymore)"
)
if (system, apps) == (True, True):
raise YunohostValidationError("tools_upgrade_cant_both")
if (system, apps) == (False, False):
raise YunohostValidationError("tools_upgrade_at_least_one")
target = "apps" if apps else "system"
if target not in ["apps", "system"]:
raise Exception(
"Uhoh ?! tools_upgrade should have 'apps' or 'system' value for argument target"
)
#
# Apps
# This is basically just an alias to yunohost app upgrade ...
#
if target == "apps":
# Make sure there's actually something to upgrade
upgradable_apps = [app["id"] for app in _list_upgradable_apps()]
if not upgradable_apps:
logger.info(m18n.n("apps_already_up_to_date"))
return
# Actually start the upgrades
try:
app_upgrade(app=upgradable_apps)
except Exception as e:
logger.warning("unable to upgrade apps: %s" % str(e))
logger.error(m18n.n("app_upgrade_some_app_failed"))
return
#
# System
#
if target == "system":
# Check that there's indeed some packages to upgrade
upgradables = list(_list_upgradable_apt_packages())
if not upgradables:
logger.info(m18n.n("already_up_to_date"))
logger.info(m18n.n("upgrading_packages"))
operation_logger.start()
# Critical packages are packages that we can't just upgrade
# randomly from yunohost itself... upgrading them is likely to
critical_packages = [
"moulinette", "yunohost", "yunohost-admin", "ssowat"
]
critical_packages_upgradable = [
p["name"] for p in upgradables if p["name"] in critical_packages
]
noncritical_packages_upgradable = [
p["name"] for p in upgradables
if p["name"] not in critical_packages
]
# Prepare dist-upgrade command
dist_upgrade = "DEBIAN_FRONTEND=noninteractive"
dist_upgrade += " APT_LISTCHANGES_FRONTEND=none"
dist_upgrade += " apt-get"
dist_upgrade += (
" --fix-broken --show-upgraded --assume-yes --quiet -o=Dpkg::Use-Pty=0"
)
for conf_flag in ["old", "miss", "def"]:
dist_upgrade += ' -o Dpkg::Options::="--force-conf{}"'.format(
conf_flag)
dist_upgrade += " dist-upgrade"
#
# "Regular" packages upgrade
#
if noncritical_packages_upgradable:
logger.info(m18n.n("tools_upgrade_regular_packages"))
# Mark all critical packages as held
for package in critical_packages:
check_output("apt-mark hold %s" % package)
# Doublecheck with apt-mark showhold that packages are indeed held ...
held_packages = check_output("apt-mark showhold").split("\n")
if any(p not in held_packages for p in critical_packages):
logger.warning(
m18n.n("tools_upgrade_cant_hold_critical_packages"))
operation_logger.error(m18n.n("packages_upgrade_failed"))
raise YunohostError(m18n.n("packages_upgrade_failed"))
logger.debug("Running apt command :\n{}".format(dist_upgrade))
def is_relevant(line):
irrelevants = [
"service sudo-ldap already provided",
"Reading database ...",
]
return all(i not in line.rstrip() for i in irrelevants)
callbacks = (
lambda l: logger.info("+ " + l.rstrip() + "\r")
if is_relevant(l) else logger.debug(l.rstrip() + "\r"),
lambda l: logger.warning(l.rstrip())
if is_relevant(l) else logger.debug(l.rstrip()),
)
returncode = call_async_output(dist_upgrade, callbacks, shell=True)
if returncode != 0:
upgradables = list(_list_upgradable_apt_packages())
noncritical_packages_upgradable = [
p["name"] for p in upgradables
if p["name"] not in critical_packages
]
logger.warning(
m18n.n(
"tools_upgrade_regular_packages_failed",
packages_list=", ".join(
noncritical_packages_upgradable),
))
operation_logger.error(m18n.n("packages_upgrade_failed"))
raise YunohostError(m18n.n("packages_upgrade_failed"))
#
# Critical packages upgrade
#
if critical_packages_upgradable and allow_yunohost_upgrade:
logger.info(m18n.n("tools_upgrade_special_packages"))
# Mark all critical packages as unheld
for package in critical_packages:
check_output("apt-mark unhold %s" % package)
# Doublecheck with apt-mark showhold that packages are indeed unheld ...
held_packages = check_output("apt-mark showhold").split("\n")
if any(p in held_packages for p in critical_packages):
logger.warning(
m18n.n("tools_upgrade_cant_unhold_critical_packages"))
operation_logger.error(m18n.n("packages_upgrade_failed"))
raise YunohostError(m18n.n("packages_upgrade_failed"))
#
# Here we use a dirty hack to run a command after the current
# "yunohost tools upgrade", because the upgrade of yunohost
# will also trigger other yunohost commands (e.g. "yunohost tools migrations run")
# (also the upgrade of the package, if executed from the webadmin, is
# likely to kill/restart the api which is in turn likely to kill this
# command before it ends...)
#
logfile = operation_logger.log_path
dist_upgrade = dist_upgrade + " 2>&1 | tee -a {}".format(logfile)
MOULINETTE_LOCK = "/var/run/moulinette_yunohost.lock"
wait_until_end_of_yunohost_command = (
"(while [ -f {} ]; do sleep 2; done)".format(MOULINETTE_LOCK))
mark_success = (
"(echo 'Done!' | tee -a {} && echo 'success: true' >> {})".
format(logfile, operation_logger.md_path))
mark_failure = (
"(echo 'Failed :(' | tee -a {} && echo 'success: false' >> {})"
.format(logfile, operation_logger.md_path))
update_log_metadata = "sed -i \"s/ended_at: .*$/ended_at: $(date -u +'%Y-%m-%d %H:%M:%S.%N')/\" {}"
update_log_metadata = update_log_metadata.format(
operation_logger.md_path)
# Dirty hack such that the operation_logger does not add ended_at
# and success keys in the log metadata. (c.f. the code of the
# is_unit_operation + operation_logger.close()) We take care of
# this ourselves (c.f. the mark_success and updated_log_metadata in
# the huge command launched by os.system)
operation_logger.ended_at = "notyet"
upgrade_completed = "\n" + m18n.n(
"tools_upgrade_special_packages_completed")
command = "({wait} && {dist_upgrade}) && {mark_success} || {mark_failure}; {update_metadata}; echo '{done}'".format(
wait=wait_until_end_of_yunohost_command,
dist_upgrade=dist_upgrade,
mark_success=mark_success,
mark_failure=mark_failure,
update_metadata=update_log_metadata,
done=upgrade_completed,
)
logger.warning(
m18n.n("tools_upgrade_special_packages_explanation"))
logger.debug("Running command :\n{}".format(command))
open("/tmp/yunohost-selfupgrade",
"w").write("rm /tmp/yunohost-selfupgrade; " + command)
# Using systemd-run --scope is like nohup/disown and &, but more robust somehow
# (despite using nohup/disown and &, the self-upgrade process was still getting killed...)
# ref: https://unix.stackexchange.com/questions/420594/why-process-killed-with-nohup
# (though I still don't understand it 100%...)
os.system("systemd-run --scope bash /tmp/yunohost-selfupgrade &")
return
else:
logger.success(m18n.n("system_upgraded"))
operation_logger.success()
def user_group_update(operation_logger,
groupname,
add=None,
remove=None,
force=False,
sync_perm=True):
"""
Update user informations
Keyword argument:
groupname -- Groupname to update
add -- User(s) to add in group
remove -- User(s) to remove in group
"""
from yunohost.permission import permission_sync_to_user
from yunohost.utils.ldap import _get_ldap_interface
existing_users = list(user_list()["users"].keys())
# Refuse to edit a primary group of a user (e.g. group 'sam' related to user 'sam')
# Those kind of group should only ever contain the user (e.g. sam) and only this one.
# We also can't edit "all_users" without the force option because that's a special group...
if not force:
if groupname == "all_users":
raise YunohostValidationError("group_cannot_edit_all_users")
elif groupname == "visitors":
raise YunohostValidationError("group_cannot_edit_visitors")
elif groupname in existing_users:
raise YunohostValidationError("group_cannot_edit_primary_group",
group=groupname)
# We extract the uid for each member of the group to keep a simple flat list of members
current_group = user_group_info(groupname)["members"]
new_group = copy.copy(current_group)
if add:
users_to_add = [add] if not isinstance(add, list) else add
for user in users_to_add:
if user not in existing_users:
raise YunohostValidationError("user_unknown", user=user)
if user in current_group:
logger.warning(
m18n.n("group_user_already_in_group",
user=user,
group=groupname))
else:
operation_logger.related_to.append(("user", user))
new_group += users_to_add
if remove:
users_to_remove = [remove] if not isinstance(remove, list) else remove
for user in users_to_remove:
if user not in current_group:
logger.warning(
m18n.n("group_user_not_in_group",
user=user,
group=groupname))
else:
operation_logger.related_to.append(("user", user))
# Remove users_to_remove from new_group
# Kinda like a new_group -= users_to_remove
new_group = [u for u in new_group if u not in users_to_remove]
new_group_dns = [
"uid=" + user + ",ou=users,dc=yunohost,dc=org" for user in new_group
]
if set(new_group) != set(current_group):
operation_logger.start()
ldap = _get_ldap_interface()
try:
ldap.update(
"cn=%s,ou=groups" % groupname,
{
"member": set(new_group_dns),
"memberUid": set(new_group)
},
)
except Exception as e:
raise YunohostError("group_update_failed",
group=groupname,
error=e)
if groupname != "all_users":
logger.success(m18n.n("group_updated", group=groupname))
else:
logger.debug(m18n.n("group_updated", group=groupname))
if sync_perm:
permission_sync_to_user()
return user_group_info(groupname)
def user_create(
operation_logger,
username,
firstname,
lastname,
domain,
password,
mailbox_quota="0",
mail=None,
):
from yunohost.domain import domain_list, _get_maindomain
from yunohost.hook import hook_callback
from yunohost.utils.password import assert_password_is_strong_enough
from yunohost.utils.ldap import _get_ldap_interface
# Ensure sufficiently complex password
assert_password_is_strong_enough("user", password)
if mail is not None:
logger.warning(
"Packagers ! Using --mail in 'yunohost user create' is deprecated ... please use --domain instead."
)
domain = mail.split("@")[-1]
# Validate domain used for email address/xmpp account
if domain is None:
if msettings.get("interface") == "api":
raise YunohostValidationError(
"Invalid usage, you should specify a domain argument")
else:
# On affiche les differents domaines possibles
msignals.display(m18n.n("domains_available"))
for domain in domain_list()["domains"]:
msignals.display("- {}".format(domain))
maindomain = _get_maindomain()
domain = msignals.prompt(
m18n.n("ask_user_domain") + " (default: %s)" % maindomain)
if not domain:
domain = maindomain
# Check that the domain exists
if domain not in domain_list()["domains"]:
raise YunohostValidationError("domain_name_unknown", domain=domain)
mail = username + "@" + domain
ldap = _get_ldap_interface()
if username in user_list()["users"]:
raise YunohostValidationError("user_already_exists", user=username)
# Validate uniqueness of username and mail in LDAP
try:
ldap.validate_uniqueness({
"uid": username,
"mail": mail,
"cn": username
})
except Exception as e:
raise YunohostValidationError("user_creation_failed",
user=username,
error=e)
# Validate uniqueness of username in system users
all_existing_usernames = {x.pw_name for x in pwd.getpwall()}
if username in all_existing_usernames:
raise YunohostValidationError("system_username_exists")
main_domain = _get_maindomain()
aliases = [
"root@" + main_domain,
"admin@" + main_domain,
"webmaster@" + main_domain,
"postmaster@" + main_domain,
"abuse@" + main_domain,
]
if mail in aliases:
raise YunohostValidationError("mail_unavailable")
operation_logger.start()
# Get random UID/GID
all_uid = {str(x.pw_uid) for x in pwd.getpwall()}
all_gid = {str(x.gr_gid) for x in grp.getgrall()}
uid_guid_found = False
while not uid_guid_found:
# LXC uid number is limited to 65536 by default
uid = str(random.randint(1001, 65000))
uid_guid_found = uid not in all_uid and uid not in all_gid
# Adapt values for LDAP
fullname = "%s %s" % (firstname, lastname)
attr_dict = {
"objectClass": [
"mailAccount",
"inetOrgPerson",
"posixAccount",
"userPermissionYnh",
],
"givenName": [firstname],
"sn": [lastname],
"displayName": [fullname],
"cn": [fullname],
"uid": [username],
"mail":
mail, # NOTE: this one seems to be already a list
"maildrop": [username],
"mailuserquota": [mailbox_quota],
"userPassword": [_hash_user_password(password)],
"gidNumber": [uid],
"uidNumber": [uid],
"homeDirectory": ["/home/" + username],
"loginShell": ["/bin/bash"],
}
# If it is the first user, add some aliases
if not ldap.search(base="ou=users,dc=yunohost,dc=org", filter="uid=*"):
attr_dict["mail"] = [attr_dict["mail"]] + aliases
try:
ldap.add("uid=%s,ou=users" % username, attr_dict)
except Exception as e:
raise YunohostError("user_creation_failed", user=username, error=e)
# Invalidate passwd and group to take user and group creation into account
subprocess.call(["nscd", "-i", "passwd"])
subprocess.call(["nscd", "-i", "group"])
try:
# Attempt to create user home folder
subprocess.check_call(["mkhomedir_helper", username])
except subprocess.CalledProcessError:
if not os.path.isdir("/home/{0}".format(username)):
logger.warning(m18n.n("user_home_creation_failed"), exc_info=1)
try:
subprocess.check_call(
["setfacl", "-m", "g:all_users:---",
"/home/%s" % username])
except subprocess.CalledProcessError:
logger.warning("Failed to protect /home/%s" % username, exc_info=1)
# Create group for user and add to group 'all_users'
user_group_create(groupname=username,
gid=uid,
primary_group=True,
sync_perm=False)
user_group_update(groupname="all_users",
add=username,
force=True,
sync_perm=True)
# Trigger post_user_create hooks
env_dict = {
"YNH_USER_USERNAME": username,
"YNH_USER_MAIL": mail,
"YNH_USER_PASSWORD": password,
"YNH_USER_FIRSTNAME": firstname,
"YNH_USER_LASTNAME": lastname,
}
hook_callback("post_user_create", args=[username, mail], env=env_dict)
# TODO: Send a welcome mail to user
logger.success(m18n.n("user_created"))
return {"fullname": fullname, "username": username, "mail": mail}
def user_group_create(operation_logger,
groupname,
gid=None,
primary_group=False,
sync_perm=True):
"""
Create group
Keyword argument:
groupname -- Must be unique
"""
from yunohost.permission import permission_sync_to_user
from yunohost.utils.ldap import _get_ldap_interface
ldap = _get_ldap_interface()
# Validate uniqueness of groupname in LDAP
conflict = ldap.get_conflict({"cn": groupname},
base_dn="ou=groups,dc=yunohost,dc=org")
if conflict:
raise YunohostValidationError("group_already_exist", group=groupname)
# Validate uniqueness of groupname in system group
all_existing_groupnames = {x.gr_name for x in grp.getgrall()}
if groupname in all_existing_groupnames:
if primary_group:
logger.warning(
m18n.n("group_already_exist_on_system_but_removing_it",
group=groupname))
subprocess.check_call("sed --in-place '/^%s:/d' /etc/group" %
groupname,
shell=True)
else:
raise YunohostValidationError("group_already_exist_on_system",
group=groupname)
if not gid:
# Get random GID
all_gid = {x.gr_gid for x in grp.getgrall()}
uid_guid_found = False
while not uid_guid_found:
gid = str(random.randint(200, 99999))
uid_guid_found = gid not in all_gid
attr_dict = {
"objectClass": ["top", "groupOfNamesYnh", "posixGroup"],
"cn": groupname,
"gidNumber": gid,
}
# Here we handle the creation of a primary group
# We want to initialize this group to contain the corresponding user
# (then we won't be able to add/remove any user in this group)
if primary_group:
attr_dict["member"] = [
"uid=" + groupname + ",ou=users,dc=yunohost,dc=org"
]
operation_logger.start()
try:
ldap.add("cn=%s,ou=groups" % groupname, attr_dict)
except Exception as e:
raise YunohostError("group_creation_failed", group=groupname, error=e)
if sync_perm:
permission_sync_to_user()
if not primary_group:
logger.success(m18n.n("group_created", group=groupname))
else:
logger.debug(m18n.n("group_created", group=groupname))
return {"name": groupname}
def user_info(username):
"""
Get user informations
Keyword argument:
username -- Username or mail to get informations
"""
from yunohost.utils.ldap import _get_ldap_interface
ldap = _get_ldap_interface()
user_attrs = [
"cn", "mail", "uid", "maildrop", "givenName", "sn", "mailuserquota"
]
if len(username.split("@")) == 2:
filter = "mail=" + username
else:
filter = "uid=" + username
result = ldap.search("ou=users,dc=yunohost,dc=org", filter, user_attrs)
if result:
user = result[0]
else:
raise YunohostValidationError("user_unknown", user=username)
result_dict = {
"username": user["uid"][0],
"fullname": user["cn"][0],
"firstname": user["givenName"][0],
"lastname": user["sn"][0],
"mail": user["mail"][0],
}
if len(user["mail"]) > 1:
result_dict["mail-aliases"] = user["mail"][1:]
if len(user["maildrop"]) > 1:
result_dict["mail-forward"] = user["maildrop"][1:]
if "mailuserquota" in user:
userquota = user["mailuserquota"][0]
if isinstance(userquota, int):
userquota = str(userquota)
# Test if userquota is '0' or '0M' ( quota pattern is ^(\d+[bkMGT])|0$ )
is_limited = not re.match("0[bkMGT]?", userquota)
storage_use = "?"
if service_status("dovecot")["status"] != "running":
logger.warning(m18n.n("mailbox_used_space_dovecot_down"))
elif username not in user_permission_info(
"mail.main")["corresponding_users"]:
logger.warning(m18n.n("mailbox_disabled", user=username))
else:
try:
cmd = "doveadm -f flow quota get -u %s" % user["uid"][0]
cmd_result = check_output(cmd)
except Exception as e:
cmd_result = ""
logger.warning("Failed to fetch quota info ... : %s " % str(e))
# Exemple of return value for cmd:
# """Quota name=User quota Type=STORAGE Value=0 Limit=- %=0
# Quota name=User quota Type=MESSAGE Value=0 Limit=- %=0"""
has_value = re.search(r"Value=(\d+)", cmd_result)
if has_value:
storage_use = int(has_value.group(1))
storage_use = _convertSize(storage_use)
if is_limited:
has_percent = re.search(r"%=(\d+)", cmd_result)
if has_percent:
percentage = int(has_percent.group(1))
storage_use += " (%s%%)" % percentage
result_dict["mailbox-quota"] = {
"limit": userquota if is_limited else m18n.n("unlimit"),
"use": storage_use,
}
return result_dict
def user_update(
operation_logger,
username,
firstname=None,
lastname=None,
mail=None,
change_password=None,
add_mailforward=None,
remove_mailforward=None,
add_mailalias=None,
remove_mailalias=None,
mailbox_quota=None,
):
"""
Update user informations
Keyword argument:
lastname
mail
firstname
add_mailalias -- Mail aliases to add
remove_mailforward -- Mailforward addresses to remove
username -- Username of user to update
add_mailforward -- Mailforward addresses to add
change_password -- New password to set
remove_mailalias -- Mail aliases to remove
"""
from yunohost.domain import domain_list, _get_maindomain
from yunohost.app import app_ssowatconf
from yunohost.utils.password import assert_password_is_strong_enough
from yunohost.utils.ldap import _get_ldap_interface
from yunohost.hook import hook_callback
domains = domain_list()["domains"]
# Populate user informations
ldap = _get_ldap_interface()
attrs_to_fetch = ["givenName", "sn", "mail", "maildrop"]
result = ldap.search(
base="ou=users,dc=yunohost,dc=org",
filter="uid=" + username,
attrs=attrs_to_fetch,
)
if not result:
raise YunohostValidationError("user_unknown", user=username)
user = result[0]
env_dict = {"YNH_USER_USERNAME": username}
# Get modifications from arguments
new_attr_dict = {}
if firstname:
new_attr_dict["givenName"] = [firstname] # TODO: Validate
new_attr_dict["cn"] = new_attr_dict["displayName"] = [
firstname + " " + user["sn"][0]
]
env_dict["YNH_USER_FIRSTNAME"] = firstname
if lastname:
new_attr_dict["sn"] = [lastname] # TODO: Validate
new_attr_dict["cn"] = new_attr_dict["displayName"] = [
user["givenName"][0] + " " + lastname
]
env_dict["YNH_USER_LASTNAME"] = lastname
if lastname and firstname:
new_attr_dict["cn"] = new_attr_dict["displayName"] = [
firstname + " " + lastname
]
# change_password is None if user_update is not called to change the password
if change_password is not None:
# when in the cli interface if the option to change the password is called
# without a specified value, change_password will be set to the const 0.
# In this case we prompt for the new password.
if msettings.get("interface") == "cli" and not change_password:
change_password = msignals.prompt(m18n.n("ask_password"), True,
True)
# Ensure sufficiently complex password
assert_password_is_strong_enough("user", change_password)
new_attr_dict["userPassword"] = [_hash_user_password(change_password)]
env_dict["YNH_USER_PASSWORD"] = change_password
if mail:
main_domain = _get_maindomain()
aliases = [
"root@" + main_domain,
"admin@" + main_domain,
"webmaster@" + main_domain,
"postmaster@" + main_domain,
]
try:
ldap.validate_uniqueness({"mail": mail})
except Exception as e:
raise YunohostValidationError("user_update_failed",
user=username,
error=e)
if mail[mail.find("@") + 1:] not in domains:
raise YunohostValidationError("mail_domain_unknown",
domain=mail[mail.find("@") + 1:])
if mail in aliases:
raise YunohostValidationError("mail_unavailable")
del user["mail"][0]
new_attr_dict["mail"] = [mail] + user["mail"]
if add_mailalias:
if not isinstance(add_mailalias, list):
add_mailalias = [add_mailalias]
for mail in add_mailalias:
try:
ldap.validate_uniqueness({"mail": mail})
except Exception as e:
raise YunohostValidationError("user_update_failed",
user=username,
error=e)
if mail[mail.find("@") + 1:] not in domains:
raise YunohostValidationError("mail_domain_unknown",
domain=mail[mail.find("@") + 1:])
user["mail"].append(mail)
new_attr_dict["mail"] = user["mail"]
if remove_mailalias:
if not isinstance(remove_mailalias, list):
remove_mailalias = [remove_mailalias]
for mail in remove_mailalias:
if len(user["mail"]) > 1 and mail in user["mail"][1:]:
user["mail"].remove(mail)
else:
raise YunohostValidationError("mail_alias_remove_failed",
mail=mail)
new_attr_dict["mail"] = user["mail"]
if "mail" in new_attr_dict:
env_dict["YNH_USER_MAILS"] = ",".join(new_attr_dict["mail"])
if add_mailforward:
if not isinstance(add_mailforward, list):
add_mailforward = [add_mailforward]
for mail in add_mailforward:
if mail in user["maildrop"][1:]:
continue
user["maildrop"].append(mail)
new_attr_dict["maildrop"] = user["maildrop"]
if remove_mailforward:
if not isinstance(remove_mailforward, list):
remove_mailforward = [remove_mailforward]
for mail in remove_mailforward:
if len(user["maildrop"]) > 1 and mail in user["maildrop"][1:]:
user["maildrop"].remove(mail)
else:
raise YunohostValidationError("mail_forward_remove_failed",
mail=mail)
new_attr_dict["maildrop"] = user["maildrop"]
if "maildrop" in new_attr_dict:
env_dict["YNH_USER_MAILFORWARDS"] = ",".join(new_attr_dict["maildrop"])
if mailbox_quota is not None:
new_attr_dict["mailuserquota"] = [mailbox_quota]
env_dict["YNH_USER_MAILQUOTA"] = mailbox_quota
operation_logger.start()
try:
ldap.update("uid=%s,ou=users" % username, new_attr_dict)
except Exception as e:
raise YunohostError("user_update_failed", user=username, error=e)
# Trigger post_user_update hooks
hook_callback("post_user_update", env=env_dict)
logger.success(m18n.n("user_updated"))
app_ssowatconf()
return user_info(username)
def domain_remove(operation_logger, domain, remove_apps=False, force=False):
"""
Delete domains
Keyword argument:
domain -- Domain to delete
remove_apps -- Remove applications installed on the domain
force -- Force the domain removal and don't not ask confirmation to
remove apps if remove_apps is specified
"""
from yunohost.hook import hook_callback
from yunohost.app import app_ssowatconf, app_info, app_remove
from yunohost.utils.ldap import _get_ldap_interface
# the 'force' here is related to the exception happening in domain_add ...
# we don't want to check the domain exists because the ldap add may have
# failed
if not force and domain not in domain_list()["domains"]:
raise YunohostValidationError("domain_name_unknown", domain=domain)
# Check domain is not the main domain
if domain == _get_maindomain():
other_domains = domain_list()["domains"]
other_domains.remove(domain)
if other_domains:
raise YunohostValidationError(
"domain_cannot_remove_main",
domain=domain,
other_domains="\n * " + ("\n * ".join(other_domains)),
)
else:
raise YunohostValidationError(
"domain_cannot_remove_main_add_new_one", domain=domain)
# Check if apps are installed on the domain
apps_on_that_domain = []
for app in _installed_apps():
settings = _get_app_settings(app)
label = app_info(app)["name"]
if settings.get("domain") == domain:
apps_on_that_domain.append((
app,
' - %s "%s" on https://%s%s' %
(app, label, domain, settings["path"])
if "path" in settings else app,
))
if apps_on_that_domain:
if remove_apps:
if msettings.get("interface") == "cli" and not force:
answer = msignals.prompt(
m18n.n(
"domain_remove_confirm_apps_removal",
apps="\n".join([x[1] for x in apps_on_that_domain]),
answers="y/N",
),
color="yellow",
)
if answer.upper() != "Y":
raise YunohostError("aborting")
for app, _ in apps_on_that_domain:
app_remove(app)
else:
raise YunohostValidationError(
"domain_uninstall_app_first",
apps="\n".join([x[1] for x in apps_on_that_domain]),
)
operation_logger.start()
ldap = _get_ldap_interface()
try:
ldap.remove("virtualdomain=" + domain + ",ou=domains")
except Exception as e:
raise YunohostError("domain_deletion_failed", domain=domain, error=e)
os.system("rm -rf /etc/yunohost/certs/%s" % domain)
# Delete dyndns keys for this domain (if any)
os.system("rm -rf /etc/yunohost/dyndns/K%s.+*" % domain)
# Sometime we have weird issues with the regenconf where some files
# appears as manually modified even though they weren't touched ...
# There are a few ideas why this happens (like backup/restore nginx
# conf ... which we shouldnt do ...). This in turns creates funky
# situation where the regenconf may refuse to re-create the conf
# (when re-creating a domain..)
#
# So here we force-clear the has out of the regenconf if it exists.
# This is a pretty ad hoc solution and only applied to nginx
# because it's one of the major service, but in the long term we
# should identify the root of this bug...
_force_clear_hashes(["/etc/nginx/conf.d/%s.conf" % domain])
# And in addition we even force-delete the file Otherwise, if the file was
# manually modified, it may not get removed by the regenconf which leads to
# catastrophic consequences of nginx breaking because it can't load the
# cert file which disappeared etc..
if os.path.exists("/etc/nginx/conf.d/%s.conf" % domain):
_process_regen_conf("/etc/nginx/conf.d/%s.conf" % domain,
new_conf=None,
save=True)
regen_conf(names=["nginx", "metronome", "dnsmasq", "postfix"])
app_ssowatconf()
hook_callback("post_domain_remove", args=[domain])
logger.success(m18n.n("domain_deleted"))
def tools_migrations_run(targets=[],
skip=False,
auto=False,
force_rerun=False,
accept_disclaimer=False):
"""
Perform migrations
targets A list migrations to run (all pendings by default)
--skip Skip specified migrations (to be used only if you know what you are doing) (must explicit which migrations)
--auto Automatic mode, won't run manual migrations (to be used only if you know what you are doing)
--force-rerun Re-run already-ran migrations (to be used only if you know what you are doing)(must explicit which migrations)
--accept-disclaimer Accept disclaimers of migrations (please read them before using this option) (only valid for one migration)
"""
all_migrations = _get_migrations_list()
# Small utility that allows up to get a migration given a name, id or number later
def get_matching_migration(target):
for m in all_migrations:
if m.id == target or m.name == target or m.id.split(
"_")[0] == target:
return m
raise YunohostValidationError("migrations_no_such_migration",
id=target)
# auto, skip and force are exclusive options
if auto + skip + force_rerun > 1:
raise YunohostValidationError("migrations_exclusive_options")
# If no target specified
if not targets:
# skip, revert or force require explicit targets
if skip or force_rerun:
raise YunohostValidationError(
"migrations_must_provide_explicit_targets")
# Otherwise, targets are all pending migrations
targets = [m for m in all_migrations if m.state == "pending"]
# If explicit targets are provided, we shall validate them
else:
targets = [get_matching_migration(t) for t in targets]
done = [t.id for t in targets if t.state != "pending"]
pending = [t.id for t in targets if t.state == "pending"]
if skip and done:
raise YunohostValidationError("migrations_not_pending_cant_skip",
ids=", ".join(done))
if force_rerun and pending:
raise YunohostValidationError("migrations_pending_cant_rerun",
ids=", ".join(pending))
if not (skip or force_rerun) and done:
raise YunohostValidationError("migrations_already_ran",
ids=", ".join(done))
# So, is there actually something to do ?
if not targets:
logger.info(m18n.n("migrations_no_migrations_to_run"))
return
# Actually run selected migrations
for migration in targets:
# If we are migrating in "automatic mode" (i.e. from debian configure
# during an upgrade of the package) but we are asked for running
# migrations to be ran manually by the user, stop there and ask the
# user to run the migration manually.
if auto and migration.mode == "manual":
logger.warn(
m18n.n("migrations_to_be_ran_manually", id=migration.id))
# We go to the next migration
continue
# Check for migration dependencies
if not skip:
dependencies = [
get_matching_migration(dep) for dep in migration.dependencies
]
pending_dependencies = [
dep.id for dep in dependencies if dep.state == "pending"
]
if pending_dependencies:
logger.error(
m18n.n(
"migrations_dependencies_not_satisfied",
id=migration.id,
dependencies_id=", ".join(pending_dependencies),
))
continue
# If some migrations have disclaimers (and we're not trying to skip them)
if migration.disclaimer and not skip:
# require the --accept-disclaimer option.
# Otherwise, go to the next migration
if not accept_disclaimer:
logger.warn(
m18n.n(
"migrations_need_to_accept_disclaimer",
id=migration.id,
disclaimer=migration.disclaimer,
))
continue
# --accept-disclaimer will only work for the first migration
else:
accept_disclaimer = False
# Start register change on system
operation_logger = OperationLogger("tools_migrations_migrate_forward")
operation_logger.start()
if skip:
logger.warn(m18n.n("migrations_skip_migration", id=migration.id))
migration.state = "skipped"
_write_migration_state(migration.id, "skipped")
operation_logger.success()
else:
try:
migration.operation_logger = operation_logger
logger.info(
m18n.n("migrations_running_forward", id=migration.id))
migration.run()
except Exception as e:
# migration failed, let's stop here but still update state because
# we managed to run the previous ones
msg = m18n.n("migrations_migration_has_failed",
exception=e,
id=migration.id)
logger.error(msg, exc_info=1)
operation_logger.error(msg)
else:
logger.success(
m18n.n("migrations_success_forward", id=migration.id))
migration.state = "done"
_write_migration_state(migration.id, "done")
operation_logger.success()
def domain_add(operation_logger, domain, dyndns=False):
"""
Create a custom domain
Keyword argument:
domain -- Domain name to add
dyndns -- Subscribe to DynDNS
"""
from yunohost.hook import hook_callback
from yunohost.app import app_ssowatconf
from yunohost.utils.ldap import _get_ldap_interface
from yunohost.certificate import _certificate_install_selfsigned
if domain.startswith("xmpp-upload."):
raise YunohostValidationError("domain_cannot_add_xmpp_upload")
ldap = _get_ldap_interface()
try:
ldap.validate_uniqueness({"virtualdomain": domain})
except MoulinetteError:
raise YunohostValidationError("domain_exists")
# Lower domain to avoid some edge cases issues
# See: https://forum.yunohost.org/t/invalid-domain-causes-diagnosis-web-to-fail-fr-on-demand/11765
domain = domain.lower()
# DynDNS domain
if dyndns:
from yunohost.dyndns import _dyndns_provides, _guess_current_dyndns_domain
# Do not allow to subscribe to multiple dyndns domains...
if _guess_current_dyndns_domain("dyndns.yunohost.org") != (None, None):
raise YunohostValidationError("domain_dyndns_already_subscribed")
# Check that this domain can effectively be provided by
# dyndns.yunohost.org. (i.e. is it a nohost.me / noho.st)
if not _dyndns_provides("dyndns.yunohost.org", domain):
raise YunohostValidationError("domain_dyndns_root_unknown")
operation_logger.start()
if dyndns:
from yunohost.dyndns import dyndns_subscribe
# Actually subscribe
dyndns_subscribe(domain=domain)
_certificate_install_selfsigned([domain], False)
try:
attr_dict = {
"objectClass": ["mailDomain", "top"],
"virtualdomain": domain,
}
try:
ldap.add("virtualdomain=%s,ou=domains" % domain, attr_dict)
except Exception as e:
raise YunohostError("domain_creation_failed",
domain=domain,
error=e)
# Don't regen these conf if we're still in postinstall
if os.path.exists("/etc/yunohost/installed"):
# Sometime we have weird issues with the regenconf where some files
# appears as manually modified even though they weren't touched ...
# There are a few ideas why this happens (like backup/restore nginx
# conf ... which we shouldnt do ...). This in turns creates funky
# situation where the regenconf may refuse to re-create the conf
# (when re-creating a domain..)
# So here we force-clear the has out of the regenconf if it exists.
# This is a pretty ad hoc solution and only applied to nginx
# because it's one of the major service, but in the long term we
# should identify the root of this bug...
_force_clear_hashes(["/etc/nginx/conf.d/%s.conf" % domain])
regen_conf(
names=["nginx", "metronome", "dnsmasq", "postfix", "rspamd"])
app_ssowatconf()
except Exception as e:
# Force domain removal silently
try:
domain_remove(domain, force=True)
except Exception:
pass
raise e
hook_callback("post_domain_add", args=[domain])
logger.success(m18n.n("domain_created"))
def firewall_upnp(action="status", no_refresh=False):
"""
Manage port forwarding using UPnP
Note: 'reload' action is deprecated and will be removed in the near
future. You should use 'status' instead - which retrieve UPnP status
and automatically refresh port forwarding if 'no_refresh' is False.
Keyword argument:
action -- Action to perform
no_refresh -- Do not refresh port forwarding
"""
firewall = firewall_list(raw=True)
enabled = firewall["uPnP"]["enabled"]
# Compatibility with previous version
if action == "reload":
logger.debug("'reload' action is deprecated and will be removed")
try:
# Remove old cron job
os.remove("/etc/cron.d/yunohost-firewall")
except Exception:
pass
action = "status"
no_refresh = False
if action == "status" and no_refresh:
# Only return current state
return {"enabled": enabled}
elif action == "enable" or (enabled and action == "status"):
# Add cron job
with open(UPNP_CRON_JOB, "w+") as f:
f.write(
"*/50 * * * * root "
"/usr/bin/yunohost firewall upnp status >>/dev/null\n"
)
# Open port 1900 to receive discovery message
if 1900 not in firewall["ipv4"]["UDP"]:
firewall_allow("UDP", 1900, no_upnp=True, no_reload=True)
if not enabled:
firewall_reload(skip_upnp=True)
enabled = True
elif action == "disable" or (not enabled and action == "status"):
try:
# Remove cron job
os.remove(UPNP_CRON_JOB)
except Exception:
pass
enabled = False
if action == "status":
no_refresh = True
else:
raise YunohostValidationError("action_invalid", action=action)
# Refresh port mapping using UPnP
if not no_refresh:
upnpc = miniupnpc.UPnP(localport=1)
upnpc.discoverdelay = 3000
# Discover UPnP device(s)
logger.debug("discovering UPnP devices...")
nb_dev = upnpc.discover()
logger.debug("found %d UPnP device(s)", int(nb_dev))
if nb_dev < 1:
logger.error(m18n.n("upnp_dev_not_found"))
enabled = False
else:
try:
# Select UPnP device
upnpc.selectigd()
except Exception:
logger.debug("unable to select UPnP device", exc_info=1)
enabled = False
else:
# Iterate over ports
for protocol in ["TCP", "UDP"]:
if protocol + "_TO_CLOSE" in firewall["uPnP"]:
for port in firewall["uPnP"][protocol + "_TO_CLOSE"]:
# Clean the mapping of this port
if upnpc.getspecificportmapping(port, protocol):
try:
upnpc.deleteportmapping(port, protocol)
except Exception:
pass
firewall["uPnP"][protocol + "_TO_CLOSE"] = []
for port in firewall["uPnP"][protocol]:
# Clean the mapping of this port
if upnpc.getspecificportmapping(port, protocol):
try:
upnpc.deleteportmapping(port, protocol)
except Exception:
pass
if not enabled:
continue
try:
# Add new port mapping
upnpc.addportmapping(
port,
protocol,
upnpc.lanaddr,
port,
"yunohost firewall: port %d" % port,
"",
)
except Exception:
logger.debug(
"unable to add port %d using UPnP", port, exc_info=1
)
enabled = False
_update_firewall_file(firewall)
if enabled != firewall["uPnP"]["enabled"]:
firewall = firewall_list(raw=True)
firewall["uPnP"]["enabled"] = enabled
_update_firewall_file(firewall)
if not no_refresh:
# Display success message if needed
if action == "enable" and enabled:
logger.success(m18n.n("upnp_enabled"))
elif action == "disable" and not enabled:
logger.success(m18n.n("upnp_disabled"))
# Make sure to disable UPnP
elif action != "disable" and not enabled:
firewall_upnp("disable", no_refresh=True)
if not enabled and (action == "enable" or 1900 in firewall["ipv4"]["UDP"]):
# Close unused port 1900
firewall_disallow("UDP", 1900, no_reload=True)
if not no_refresh:
firewall_reload(skip_upnp=True)
if action == "enable" and not enabled:
raise YunohostError("upnp_port_open_failed")
return {"enabled": enabled}
def hook_callback(
action,
hooks=[],
args=None,
chdir=None,
env=None,
pre_callback=None,
post_callback=None,
):
"""
Execute all scripts binded to an action
Keyword argument:
action -- Action name
hooks -- List of hooks names to execute
args -- Ordered list of arguments to pass to the scripts
chdir -- The directory from where the scripts will be executed
env -- Dictionnary of environment variables to export
pre_callback -- An object to call before each script execution with
(name, priority, path, args) as arguments and which must return
the arguments to pass to the script
post_callback -- An object to call after each script execution with
(name, priority, path, succeed) as arguments
"""
result = {}
hooks_dict = {}
# Retrieve hooks
if not hooks:
hooks_dict = hook_list(action, list_by="priority",
show_info=True)["hooks"]
else:
hooks_names = hook_list(action, list_by="name",
show_info=True)["hooks"]
# Add similar hooks to the list
# For example: Having a 16-postfix hook in the list will execute a
# xx-postfix_dkim as well
all_hooks = []
for n in hooks:
for key in hooks_names.keys():
if key == n or key.startswith(
"%s_" % n) and key not in all_hooks:
all_hooks.append(key)
# Iterate over given hooks names list
for n in all_hooks:
try:
hl = hooks_names[n]
except KeyError:
raise YunohostValidationError("hook_name_unknown", n)
# Iterate over hooks with this name
for h in hl:
# Update hooks dict
d = hooks_dict.get(h["priority"], dict())
d.update({n: {"path": h["path"]}})
hooks_dict[h["priority"]] = d
if not hooks_dict:
return result
# Validate callbacks
if not callable(pre_callback):
def pre_callback(name, priority, path, args):
return args
if not callable(post_callback):
def post_callback(name, priority, path, succeed):
return None
# Iterate over hooks and execute them
for priority in sorted(hooks_dict):
for name, info in iter(hooks_dict[priority].items()):
state = "succeed"
path = info["path"]
try:
hook_args = pre_callback(name=name,
priority=priority,
path=path,
args=args)
hook_return = hook_exec(path,
args=hook_args,
chdir=chdir,
env=env,
raise_on_error=True)[1]
except YunohostError as e:
state = "failed"
hook_return = {}
logger.error(e.strerror, exc_info=1)
post_callback(name=name,
priority=priority,
path=path,
succeed=False)
else:
post_callback(name=name,
priority=priority,
path=path,
succeed=True)
if name not in result:
result[name] = {}
result[name][path] = {"state": state, "stdreturn": hook_return}
return result
def _certificate_install_selfsigned(domain_list, force=False):
for domain in domain_list:
operation_logger = OperationLogger("selfsigned_cert_install",
[("domain", domain)],
args={"force": force})
# Paths of files and folder we'll need
date_tag = datetime.utcnow().strftime("%Y%m%d.%H%M%S")
new_cert_folder = "%s/%s-history/%s-selfsigned" % (
CERT_FOLDER,
domain,
date_tag,
)
conf_template = os.path.join(SSL_DIR, "openssl.cnf")
csr_file = os.path.join(SSL_DIR, "certs", "yunohost_csr.pem")
conf_file = os.path.join(new_cert_folder, "openssl.cnf")
key_file = os.path.join(new_cert_folder, "key.pem")
crt_file = os.path.join(new_cert_folder, "crt.pem")
ca_file = os.path.join(new_cert_folder, "ca.pem")
# Check we ain't trying to overwrite a good cert !
current_cert_file = os.path.join(CERT_FOLDER, domain, "crt.pem")
if not force and os.path.isfile(current_cert_file):
status = _get_status(domain)
if status["summary"]["code"] in ("good", "great"):
raise YunohostValidationError(
"certmanager_attempt_to_replace_valid_cert", domain=domain)
operation_logger.start()
# Create output folder for new certificate stuff
os.makedirs(new_cert_folder)
# Create our conf file, based on template, replacing the occurences of
# "yunohost.org" with the given domain
with open(conf_file, "w") as f, open(conf_template, "r") as template:
for line in template:
f.write(line.replace("yunohost.org", domain))
# Use OpenSSL command line to create a certificate signing request,
# and self-sign the cert
commands = [
"openssl req -new -config %s -days 3650 -out %s -keyout %s -nodes -batch"
% (conf_file, csr_file, key_file),
"openssl ca -config %s -days 3650 -in %s -out %s -batch" %
(conf_file, csr_file, crt_file),
]
for command in commands:
p = subprocess.Popen(command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, _ = p.communicate()
out = out.decode("utf-8")
if p.returncode != 0:
logger.warning(out)
raise YunohostError("domain_cert_gen_failed")
else:
logger.debug(out)
# Link the CA cert (not sure it's actually needed in practice though,
# since we append it at the end of crt.pem. For instance for Let's
# Encrypt certs, we only need the crt.pem and key.pem)
os.symlink(SELF_CA_FILE, ca_file)
# Append ca.pem at the end of crt.pem
with open(ca_file, "r") as ca_pem, open(crt_file, "a") as crt_pem:
crt_pem.write("\n")
crt_pem.write(ca_pem.read())
# Set appropriate permissions
_set_permissions(new_cert_folder, "root", "root", 0o755)
_set_permissions(key_file, "root", "ssl-cert", 0o640)
_set_permissions(crt_file, "root", "ssl-cert", 0o640)
_set_permissions(conf_file, "root", "root", 0o600)
# Actually enable the certificate we created
_enable_certificate(domain, new_cert_folder)
# Check new status indicate a recently created self-signed certificate
status = _get_status(domain)
if (status and status["CA_type"]["code"] == "self-signed"
and status["validity"] > 3648):
logger.success(
m18n.n("certmanager_cert_install_success_selfsigned",
domain=domain))
operation_logger.success()
else:
msg = (
"Installation of self-signed certificate installation for %s failed !"
% (domain))
logger.error(msg)
operation_logger.error(msg)
def _run_service_command(action, service):
"""
Run services management command (start, stop, enable, disable, restart, reload)
Keyword argument:
action -- Action to perform
service -- Service name
"""
services = _get_services()
if service not in services.keys():
raise YunohostValidationError("service_unknown", service=service)
possible_actions = [
"start",
"stop",
"restart",
"reload",
"reload-or-restart",
"enable",
"disable",
]
if action not in possible_actions:
raise ValueError(
"Unknown action '%s', available actions are: %s"
% (action, ", ".join(possible_actions))
)
cmd = "systemctl %s %s" % (action, service)
need_lock = services[service].get("need_lock", False) and action in [
"start",
"stop",
"restart",
"reload",
"reload-or-restart",
]
if action in ["enable", "disable"]:
cmd += " --quiet"
try:
# Launch the command
logger.debug("Running '%s'" % cmd)
p = subprocess.Popen(cmd.split(), stderr=subprocess.STDOUT)
# If this command needs a lock (because the service uses yunohost
# commands inside), find the PID and add a lock for it
if need_lock:
PID = _give_lock(action, service, p)
# Wait for the command to complete
p.communicate()
if p.returncode != 0:
logger.warning(m18n.n("service_cmd_exec_failed", command=cmd))
return False
except Exception as e:
logger.warning(m18n.n("unexpected_error", error=str(e)))
return False
finally:
# Remove the lock if one was given
if need_lock and PID != 0:
_remove_lock(PID)
return True
def _certificate_install_letsencrypt(domain_list,
force=False,
no_checks=False,
staging=False):
import yunohost.domain
if not os.path.exists(ACCOUNT_KEY_FILE):
_generate_account_key()
# If no domains given, consider all yunohost domains with self-signed
# certificates
if domain_list == []:
for domain in yunohost.domain.domain_list()["domains"]:
status = _get_status(domain)
if status["CA_type"]["code"] != "self-signed":
continue
domain_list.append(domain)
# Else, validate that yunohost knows the domains given
else:
for domain in domain_list:
yunohost_domains_list = yunohost.domain.domain_list()["domains"]
if domain not in yunohost_domains_list:
raise YunohostValidationError("domain_name_unknown",
domain=domain)
# Is it self-signed?
status = _get_status(domain)
if not force and status["CA_type"]["code"] != "self-signed":
raise YunohostValidationError(
"certmanager_domain_cert_not_selfsigned", domain=domain)
if staging:
logger.warning(
"Please note that you used the --staging option, and that no new certificate will actually be enabled !"
)
# Actual install steps
for domain in domain_list:
if not no_checks:
try:
_check_domain_is_ready_for_ACME(domain)
except Exception as e:
logger.error(e)
continue
logger.info("Now attempting install of certificate for domain %s!",
domain)
operation_logger = OperationLogger(
"letsencrypt_cert_install",
[("domain", domain)],
args={
"force": force,
"no_checks": no_checks,
"staging": staging
},
)
operation_logger.start()
try:
_fetch_and_enable_new_certificate(domain,
staging,
no_checks=no_checks)
except Exception as e:
msg = "Certificate installation for %s failed !\nException: %s" % (
domain,
e,
)
logger.error(msg)
operation_logger.error(msg)
if no_checks:
logger.error(
"Please consider checking the 'DNS records' (basic) and 'Web' categories of the diagnosis to check for possible issues that may prevent installing a Let's Encrypt certificate on domain %s."
% domain)
else:
logger.success(
m18n.n("certmanager_cert_install_success", domain=domain))
operation_logger.success()
def user_permission_update(
operation_logger,
permission,
add=None,
remove=None,
label=None,
show_tile=None,
protected=None,
force=False,
sync_perm=True,
):
"""
Allow or Disallow a user or group to a permission for a specific application
Keyword argument:
permission -- Name of the permission (e.g. mail or or wordpress or wordpress.editors)
add -- (optional) List of groups or usernames to add to this permission
remove -- (optional) List of groups or usernames to remove from to this permission
label -- (optional) Define a name for the permission. This label will be shown on the SSO and in the admin
show_tile -- (optional) Define if a tile will be shown in the SSO
protected -- (optional) Define if the permission can be added/removed to the visitor group
force -- (optional) Give the possibility to add/remove access from the visitor group to a protected permission
"""
from yunohost.user import user_group_list
# By default, manipulate main permission
if "." not in permission:
permission = permission + ".main"
existing_permission = user_permission_info(permission)
# Refuse to add "visitors" to mail, xmpp ... they require an account to make sense.
if add and "visitors" in add and permission.split(".")[0] in SYSTEM_PERMS:
raise YunohostValidationError("permission_require_account",
permission=permission)
# Refuse to add "visitors" to protected permission
if ((add and "visitors" in add and existing_permission["protected"]) or
(remove and "visitors" in remove
and existing_permission["protected"])) and not force:
raise YunohostValidationError("permission_protected",
permission=permission)
# Refuse to add "all_users" to ssh/sftp permissions
if (permission.split(".")[0] in ["ssh", "sftp"]
and (add and "all_users" in add) and not force):
raise YunohostValidationError("permission_cant_add_to_all_users",
permission=permission)
# Fetch currently allowed groups for this permission
current_allowed_groups = existing_permission["allowed"]
operation_logger.related_to.append(("app", permission.split(".")[0]))
# Compute new allowed group list (and make sure what we're doing make sense)
new_allowed_groups = copy.copy(current_allowed_groups)
all_existing_groups = user_group_list()["groups"].keys()
if add:
groups_to_add = [add] if not isinstance(add, list) else add
for group in groups_to_add:
if group not in all_existing_groups:
raise YunohostValidationError("group_unknown", group=group)
if group in current_allowed_groups:
logger.warning(
m18n.n("permission_already_allowed",
permission=permission,
group=group))
else:
operation_logger.related_to.append(("group", group))
new_allowed_groups += [group]
if remove:
groups_to_remove = [remove] if not isinstance(remove, list) else remove
for group in groups_to_remove:
if group not in current_allowed_groups:
logger.warning(
m18n.n(
"permission_already_disallowed",
permission=permission,
group=group,
))
else:
operation_logger.related_to.append(("group", group))
new_allowed_groups = [
g for g in new_allowed_groups if g not in groups_to_remove
]
# If we end up with something like allowed groups is ["all_users", "volunteers"]
# we shall warn the users that they should probably choose between one or
# the other, because the current situation is probably not what they expect
# / is temporary ? Note that it's fine to have ["all_users", "visitors"]
# though, but it's not fine to have ["all_users", "visitors", "volunteers"]
if "all_users" in new_allowed_groups and len(new_allowed_groups) >= 2:
if "visitors" not in new_allowed_groups or len(
new_allowed_groups) >= 3:
logger.warning(
m18n.n("permission_currently_allowed_for_all_users"))
# Note that we can get this argument as string if we it come from the CLI
if isinstance(show_tile, str):
if show_tile.lower() == "true":
show_tile = True
else:
show_tile = False
if (existing_permission["url"]
and existing_permission["url"].startswith("re:") and show_tile):
logger.warning(
m18n.n(
"regex_incompatible_with_tile",
regex=existing_permission["url"],
permission=permission,
))
# Commit the new allowed group list
operation_logger.start()
new_permission = _update_ldap_group_permission(
permission=permission,
allowed=new_allowed_groups,
label=label,
show_tile=show_tile,
protected=protected,
sync_perm=sync_perm,
)
logger.debug(m18n.n("permission_updated", permission=permission))
return new_permission
def certificate_renew(domain_list,
force=False,
no_checks=False,
email=False,
staging=False):
"""
Renew Let's Encrypt certificate for given domains (all by default)
Keyword argument:
domain_list -- Domains for which to renew the certificates
force -- Ignore the validity threshold (15 days)
no-check -- Disable some checks about the reachability of web server
before attempting the renewing
email -- Emails root if some renewing failed
"""
import yunohost.domain
# If no domains given, consider all yunohost domains with Let's Encrypt
# certificates
if domain_list == []:
for domain in yunohost.domain.domain_list()["domains"]:
# Does it have a Let's Encrypt cert?
status = _get_status(domain)
if status["CA_type"]["code"] != "lets-encrypt":
continue
# Does it expire soon?
if status["validity"] > VALIDITY_LIMIT and not force:
continue
# Check ACME challenge configured for given domain
if not _check_acme_challenge_configuration(domain):
logger.warning(
m18n.n("certmanager_acme_not_configured_for_domain",
domain=domain))
continue
domain_list.append(domain)
if len(domain_list) == 0 and not email:
logger.info("No certificate needs to be renewed.")
# Else, validate the domain list given
else:
for domain in domain_list:
# Is it in Yunohost dmomain list?
if domain not in yunohost.domain.domain_list()["domains"]:
raise YunohostValidationError("domain_name_unknown",
domain=domain)
status = _get_status(domain)
# Does it expire soon?
if status["validity"] > VALIDITY_LIMIT and not force:
raise YunohostValidationError(
"certmanager_attempt_to_renew_valid_cert", domain=domain)
# Does it have a Let's Encrypt cert?
if status["CA_type"]["code"] != "lets-encrypt":
raise YunohostValidationError(
"certmanager_attempt_to_renew_nonLE_cert", domain=domain)
# Check ACME challenge configured for given domain
if not _check_acme_challenge_configuration(domain):
raise YunohostValidationError(
"certmanager_acme_not_configured_for_domain",
domain=domain)
if staging:
logger.warning(
"Please note that you used the --staging option, and that no new certificate will actually be enabled !"
)
# Actual renew steps
for domain in domain_list:
if not no_checks:
try:
_check_domain_is_ready_for_ACME(domain)
except Exception as e:
logger.error(e)
if email:
logger.error("Sending email with details to root ...")
_email_renewing_failed(domain, e)
continue
logger.info("Now attempting renewing of certificate for domain %s !",
domain)
operation_logger = OperationLogger(
"letsencrypt_cert_renew",
[("domain", domain)],
args={
"force": force,
"no_checks": no_checks,
"staging": staging,
"email": email,
},
)
operation_logger.start()
try:
_fetch_and_enable_new_certificate(domain,
staging,
no_checks=no_checks)
except Exception as e:
import traceback
from io import StringIO
stack = StringIO()
traceback.print_exc(file=stack)
msg = "Certificate renewing for %s failed !" % (domain)
if no_checks:
msg += (
"\nPlease consider checking the 'DNS records' (basic) and 'Web' categories of the diagnosis to check for possible issues that may prevent installing a Let's Encrypt certificate on domain %s."
% domain)
logger.error(msg)
operation_logger.error(msg)
logger.error(stack.getvalue())
logger.error(str(e))
if email:
logger.error("Sending email with details to root ...")
_email_renewing_failed(domain, msg + "\n" + str(e),
stack.getvalue())
else:
logger.success(
m18n.n("certmanager_cert_renew_success", domain=domain))
operation_logger.success()
def _validate_and_sanitize_permission_url(url, app_base_path, app):
"""
Check and normalize the urls passed for all permissions
Also check that the Regex is valid
As documented in the 'ynh_permission_create' helper:
If provided, 'url' is assumed to be relative to the app domain/path if they
start with '/'. For example:
/ -> domain.tld/app
/admin -> domain.tld/app/admin
domain.tld/app/api -> domain.tld/app/api
domain.tld -> domain.tld
'url' can be later treated as a regex if it starts with "re:".
For example:
re:/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$
re:domain.tld/app/api/[A-Z]*$ -> domain.tld/app/api/[A-Z]*$
We can also have less-trivial regexes like:
re:^/api/.*|/scripts/api.js$
"""
from yunohost.domain import domain_list
from yunohost.app import _assert_no_conflicting_apps
domains = domain_list()["domains"]
#
# Regexes
#
def validate_regex(regex):
if "%" in regex:
logger.warning(
"/!\\ Packagers! You are probably using a lua regex. You should use a PCRE regex instead."
)
return
try:
re.compile(regex)
except Exception:
raise YunohostValidationError("invalid_regex", regex=regex)
if url.startswith("re:"):
# regex without domain
# we check for the first char after 're:'
if url[3] in ["/", "^", "\\"]:
validate_regex(url[3:])
return url
# regex with domain
if "/" not in url:
raise YunohostValidationError("regex_with_only_domain")
domain, path = url[3:].split("/", 1)
path = "/" + path
if domain.replace("%", "").replace("\\", "") not in domains:
raise YunohostValidationError("domain_name_unknown", domain=domain)
validate_regex(path)
return "re:" + domain + path
#
# "Regular" URIs
#
def split_domain_path(url):
url = url.strip("/")
(domain, path) = url.split("/", 1) if "/" in url else (url, "/")
if path != "/":
path = "/" + path
return (domain, path)
# uris without domain
if url.startswith("/"):
# if url is for example /admin/
# we want sanitized_url to be: /admin
# and (domain, path) to be : (domain.tld, /app/admin)
sanitized_url = "/" + url.strip("/")
domain, path = split_domain_path(app_base_path)
path = "/" + path.strip("/") + sanitized_url
# uris with domain
else:
# if url is for example domain.tld/wat/
# we want sanitized_url to be: domain.tld/wat
# and (domain, path) to be : (domain.tld, /wat)
domain, path = split_domain_path(url)
sanitized_url = domain + path
if domain not in domains:
raise YunohostValidationError("domain_name_unknown", domain=domain)
_assert_no_conflicting_apps(domain, path, ignore_app=app)
return sanitized_url
def settings_set(key, value):
"""
Set an entry value in the settings
Keyword argument:
key -- Settings key
value -- New value
"""
settings = _get_settings()
if key not in settings:
raise YunohostValidationError("global_settings_key_doesnt_exists",
settings_key=key)
key_type = settings[key]["type"]
if key_type == "bool":
boolean_value = is_boolean(value)
if boolean_value[0]:
value = boolean_value[1]
else:
raise YunohostValidationError(
"global_settings_bad_type_for_setting",
setting=key,
received_type=type(value).__name__,
expected_type=key_type,
)
elif key_type == "int":
if not isinstance(value, int) or isinstance(value, bool):
if isinstance(value, str):
try:
value = int(value)
except Exception:
raise YunohostValidationError(
"global_settings_bad_type_for_setting",
setting=key,
received_type=type(value).__name__,
expected_type=key_type,
)
else:
raise YunohostValidationError(
"global_settings_bad_type_for_setting",
setting=key,
received_type=type(value).__name__,
expected_type=key_type,
)
elif key_type == "string":
if not isinstance(value, str):
raise YunohostValidationError(
"global_settings_bad_type_for_setting",
setting=key,
received_type=type(value).__name__,
expected_type=key_type,
)
elif key_type == "enum":
if value not in settings[key]["choices"]:
raise YunohostValidationError(
"global_settings_bad_choice_for_enum",
setting=key,
choice=str(value),
available_choices=", ".join(settings[key]["choices"]),
)
else:
raise YunohostValidationError("global_settings_unknown_type",
setting=key,
unknown_type=key_type)
old_value = settings[key].get("value")
settings[key]["value"] = value
_save_settings(settings)
try:
trigger_post_change_hook(key, old_value, value)
except Exception as e:
logger.error("Post-change hook for setting %s failed : %s" % (key, e))
raise
