def diagnosis_run(categories=[], force=False, except_if_never_ran_yet=False, email=False): if (email or except_if_never_ran_yet) and not os.path.exists(DIAGNOSIS_CACHE): return # Get all the categories all_categories = _list_diagnosis_categories() all_categories_names = [category for category, _ in all_categories] # Check the requested category makes sense if categories == []: categories = all_categories_names else: unknown_categories = [ c for c in categories if c not in all_categories_names ] if unknown_categories: raise YunohostValidationError( "diagnosis_unknown_categories", categories=", ".join(unknown_categories)) issues = [] # Call the hook ... diagnosed_categories = [] for category in categories: logger.debug("Running diagnosis for %s ..." % category) path = [p for n, p in all_categories if n == category][0] try: code, report = hook_exec(path, args={"force": force}, env=None) except Exception: import traceback logger.error( m18n.n( "diagnosis_failed_for_category", category=category, error="\n" + traceback.format_exc(), )) else: diagnosed_categories.append(category) if report != {}: issues.extend([ item for item in report["items"] if item["status"] in ["WARNING", "ERROR"] ]) if email: _email_diagnosis_issues() if issues and msettings.get("interface") == "cli": logger.warning(m18n.n("diagnosis_display_tip"))
def domain_dns_conf(domain, ttl=None): """ Generate DNS configuration for a domain Keyword argument: domain -- Domain name ttl -- Time to live """ if domain not in domain_list()["domains"]: raise YunohostError("domain_name_unknown", domain=domain) ttl = 3600 if ttl is None else ttl dns_conf = _build_dns_conf(domain, ttl) result = "" result += "; Basic ipv4/ipv6 records" for record in dns_conf["basic"]: result += "\n{name} {ttl} IN {type} {value}".format(**record) result += "\n\n" result += "; XMPP" for record in dns_conf["xmpp"]: result += "\n{name} {ttl} IN {type} {value}".format(**record) result += "\n\n" result += "; Mail" for record in dns_conf["mail"]: result += "\n{name} {ttl} IN {type} {value}".format(**record) result += "\n\n" result += "; Extra" for record in dns_conf["extra"]: result += "\n{name} {ttl} IN {type} {value}".format(**record) for name, record_list in dns_conf.items(): if name not in ("basic", "xmpp", "mail", "extra") and record_list: result += "\n\n" result += "; " + name for record in record_list: result += "\n{name} {ttl} IN {type} {value}".format(**record) if msettings.get("interface") == "cli": logger.info(m18n.n("domain_dns_conf_is_just_a_recommendation")) return result
def close(self, error=None): """ Close properly the unit operation """ # When the error happen's in the is_unit_operation try/except, # we want to inject the log ref in the exception, such that it may be # transmitted to the webadmin which can then redirect to the appropriate # log page if isinstance(error, Exception) and not isinstance( error, YunohostValidationError): error.log_ref = self.name if self.ended_at is not None or self.started_at is None: return if error is not None and not isinstance(error, str): error = str(error) self.ended_at = datetime.utcnow() self._error = error self._success = error is None if self.logger is not None: self.logger.removeHandler(self.file_handler) self.file_handler.close() is_api = msettings.get("interface") == "api" desc = _get_description_from_name(self.name) if error is None: if is_api: msg = m18n.n("log_link_to_log", name=self.name, desc=desc) else: msg = m18n.n("log_help_to_get_log", name=self.name, desc=desc) logger.debug(msg) else: if is_api: msg = ("<strong>" + m18n.n( "log_link_to_failed_log", name=self.name, desc=desc) + "</strong>") else: msg = m18n.n("log_help_to_get_failed_log", name=self.name, desc=desc) logger.info(msg) self.flush() return msg
def m18n_(info): if not isinstance(info, tuple) and not isinstance(info, list): info = (info, {}) info[1].update(meta_data) s = m18n.n(info[0], **(info[1])) # In cli, we remove the html tags if msettings.get( "interface") != "api" or force_remove_html_tags: s = s.replace("<cmd>", "'").replace("</cmd>", "'") s = html_tags.sub("", s.replace("<br>", "\n")) else: s = s.replace("<cmd>", "<code class='cmd'>").replace( "</cmd>", "</code>") # Make it so that links open in new tabs s = s.replace( "<a href=", "<a target='_blank' rel='noopener noreferrer' href=") return s
def metadata(self): """ Dictionnary of all metadata collected """ data = { "started_at": self.started_at, "operation": self.operation, "parent": self.parent, "yunohost_version": get_ynh_package_version("yunohost")["version"], "interface": msettings.get("interface"), } if self.related_to is not None: data["related_to"] = self.related_to if self.ended_at is not None: data["ended_at"] = self.ended_at data["success"] = self._success if self.error is not None: data["error"] = self._error # TODO: detect if 'extra' erase some key of 'data' data.update(self.extra) return data
def metadata(self): """ Dictionnary of all metadata collected """ data = { 'started_at': self.started_at, 'operation': self.operation, 'parent': self.parent, 'yunohost_version': get_ynh_package_version("yunohost")["version"], 'interface': msettings.get('interface'), } if self.related_to is not None: data['related_to'] = self.related_to if self.ended_at is not None: data['ended_at'] = self.ended_at data['success'] = self._success if self.error is not None: data['error'] = self._error # TODO: detect if 'extra' erase some key of 'data' data.update(self.extra) return data
def close(self, error=None): """ Close properly the unit operation """ if self.ended_at is not None or self.started_at is None: return if error is not None and not isinstance(error, str): error = str(error) self.ended_at = datetime.utcnow() self._error = error self._success = error is None if self.logger is not None: self.logger.removeHandler(self.file_handler) self.file_handler.close() is_api = msettings.get("interface") == "api" desc = _get_description_from_name(self.name) if error is None: if is_api: msg = m18n.n("log_link_to_log", name=self.name, desc=desc) else: msg = m18n.n("log_help_to_get_log", name=self.name, desc=desc) logger.debug(msg) else: if is_api: msg = ("<strong>" + m18n.n( "log_link_to_failed_log", name=self.name, desc=desc) + "</strong>") else: msg = m18n.n("log_help_to_get_failed_log", name=self.name, desc=desc) logger.info(msg) self.flush() return msg
def user_create(operation_logger, username, firstname, lastname, domain, password, mailbox_quota="0", mail=None): from yunohost.domain import domain_list, _get_maindomain from yunohost.hook import hook_callback from yunohost.utils.password import assert_password_is_strong_enough from yunohost.utils.ldap import _get_ldap_interface # Ensure sufficiently complex password assert_password_is_strong_enough("user", password) if mail is not None: logger.warning( "Packagers ! Using --mail in 'yunohost user create' is deprecated ... please use --domain instead." ) domain = mail.split("@")[-1] # Validate domain used for email address/xmpp account if domain is None: if msettings.get('interface') == 'api': raise YunohostError('Invalide usage, specify domain argument') else: # On affiche les differents domaines possibles msignals.display(m18n.n('domains_available')) for domain in domain_list()['domains']: msignals.display("- {}".format(domain)) maindomain = _get_maindomain() domain = msignals.prompt( m18n.n('ask_user_domain') + ' (default: %s)' % maindomain) if not domain: domain = maindomain # Check that the domain exists if domain not in domain_list()['domains']: raise YunohostError('domain_name_unknown', domain=domain) mail = username + '@' + domain ldap = _get_ldap_interface() if username in user_list()["users"]: raise YunohostError("user_already_exists", user=username) # Validate uniqueness of username and mail in LDAP try: ldap.validate_uniqueness({ 'uid': username, 'mail': mail, 'cn': username }) except Exception as e: raise YunohostError('user_creation_failed', user=username, error=e) # Validate uniqueness of username in system users all_existing_usernames = {x.pw_name for x in pwd.getpwall()} if username in all_existing_usernames: raise YunohostError('system_username_exists') main_domain = _get_maindomain() aliases = [ 'root@' + main_domain, 'admin@' + main_domain, 'webmaster@' + main_domain, 'postmaster@' + main_domain, 'abuse@' + main_domain, ] if mail in aliases: raise YunohostError('mail_unavailable') operation_logger.start() # Get random UID/GID all_uid = {str(x.pw_uid) for x in pwd.getpwall()} all_gid = {str(x.gr_gid) for x in grp.getgrall()} uid_guid_found = False while not uid_guid_found: # LXC uid number is limited to 65536 by default uid = str(random.randint(200, 65000)) uid_guid_found = uid not in all_uid and uid not in all_gid # Adapt values for LDAP fullname = '%s %s' % (firstname, lastname) attr_dict = { 'objectClass': ['mailAccount', 'inetOrgPerson', 'posixAccount', 'userPermissionYnh'], 'givenName': [firstname], 'sn': [lastname], 'displayName': [fullname], 'cn': [fullname], 'uid': [username], 'mail': mail, # NOTE: this one seems to be already a list 'maildrop': [username], 'mailuserquota': [mailbox_quota], 'userPassword': [_hash_user_password(password)], 'gidNumber': [uid], 'uidNumber': [uid], 'homeDirectory': ['/home/' + username], 'loginShell': ['/bin/false'] } # If it is the first user, add some aliases if not ldap.search(base='ou=users,dc=yunohost,dc=org', filter='uid=*'): attr_dict['mail'] = [attr_dict['mail']] + aliases try: ldap.add('uid=%s,ou=users' % username, attr_dict) except Exception as e: raise YunohostError('user_creation_failed', user=username, error=e) # Invalidate passwd and group to take user and group creation into account subprocess.call(['nscd', '-i', 'passwd']) subprocess.call(['nscd', '-i', 'group']) try: # Attempt to create user home folder subprocess.check_call(["mkhomedir_helper", username]) except subprocess.CalledProcessError: if not os.path.isdir('/home/{0}'.format(username)): logger.warning(m18n.n('user_home_creation_failed'), exc_info=1) # Create group for user and add to group 'all_users' user_group_create(groupname=username, gid=uid, primary_group=True, sync_perm=False) user_group_update(groupname='all_users', add=username, force=True, sync_perm=True) # Trigger post_user_create hooks env_dict = { "YNH_USER_USERNAME": username, "YNH_USER_MAIL": mail, "YNH_USER_PASSWORD": password, "YNH_USER_FIRSTNAME": firstname, "YNH_USER_LASTNAME": lastname } hook_callback('post_user_create', args=[username, mail], env=env_dict) # TODO: Send a welcome mail to user logger.success(m18n.n('user_created')) return {'fullname': fullname, 'username': username, 'mail': mail}
def user_update(operation_logger, username, firstname=None, lastname=None, mail=None, change_password=None, add_mailforward=None, remove_mailforward=None, add_mailalias=None, remove_mailalias=None, mailbox_quota=None): """ Update user informations Keyword argument: lastname mail firstname add_mailalias -- Mail aliases to add remove_mailforward -- Mailforward addresses to remove username -- Username of user to update add_mailforward -- Mailforward addresses to add change_password -- New password to set remove_mailalias -- Mail aliases to remove """ from yunohost.domain import domain_list, _get_maindomain from yunohost.app import app_ssowatconf from yunohost.utils.password import assert_password_is_strong_enough from yunohost.utils.ldap import _get_ldap_interface from yunohost.hook import hook_callback domains = domain_list()['domains'] # Populate user informations ldap = _get_ldap_interface() attrs_to_fetch = ['givenName', 'sn', 'mail', 'maildrop'] result = ldap.search(base='ou=users,dc=yunohost,dc=org', filter='uid=' + username, attrs=attrs_to_fetch) if not result: raise YunohostError('user_unknown', user=username) user = result[0] env_dict = {"YNH_USER_USERNAME": username} # Get modifications from arguments new_attr_dict = {} if firstname: new_attr_dict['givenName'] = [firstname] # TODO: Validate new_attr_dict['cn'] = new_attr_dict['displayName'] = [ firstname + ' ' + user['sn'][0] ] env_dict["YNH_USER_FIRSTNAME"] = firstname if lastname: new_attr_dict['sn'] = [lastname] # TODO: Validate new_attr_dict['cn'] = new_attr_dict['displayName'] = [ user['givenName'][0] + ' ' + lastname ] env_dict["YNH_USER_LASTNAME"] = lastname if lastname and firstname: new_attr_dict['cn'] = new_attr_dict['displayName'] = [ firstname + ' ' + lastname ] # change_password is None if user_update is not called to change the password if change_password is not None: # when in the cli interface if the option to change the password is called # without a specified value, change_password will be set to the const 0. # In this case we prompt for the new password. if msettings.get('interface') == 'cli' and not change_password: change_password = msignals.prompt(m18n.n("ask_password"), True, True) # Ensure sufficiently complex password assert_password_is_strong_enough("user", change_password) new_attr_dict['userPassword'] = [_hash_user_password(change_password)] env_dict["YNH_USER_PASSWORD"] = change_password if mail: main_domain = _get_maindomain() aliases = [ 'root@' + main_domain, 'admin@' + main_domain, 'webmaster@' + main_domain, 'postmaster@' + main_domain, ] try: ldap.validate_uniqueness({'mail': mail}) except Exception as e: raise YunohostError('user_update_failed', user=username, error=e) if mail[mail.find('@') + 1:] not in domains: raise YunohostError('mail_domain_unknown', domain=mail[mail.find('@') + 1:]) if mail in aliases: raise YunohostError('mail_unavailable') del user['mail'][0] new_attr_dict['mail'] = [mail] + user['mail'] if add_mailalias: if not isinstance(add_mailalias, list): add_mailalias = [add_mailalias] for mail in add_mailalias: try: ldap.validate_uniqueness({'mail': mail}) except Exception as e: raise YunohostError('user_update_failed', user=username, error=e) if mail[mail.find('@') + 1:] not in domains: raise YunohostError('mail_domain_unknown', domain=mail[mail.find('@') + 1:]) user['mail'].append(mail) new_attr_dict['mail'] = user['mail'] if remove_mailalias: if not isinstance(remove_mailalias, list): remove_mailalias = [remove_mailalias] for mail in remove_mailalias: if len(user['mail']) > 1 and mail in user['mail'][1:]: user['mail'].remove(mail) else: raise YunohostError('mail_alias_remove_failed', mail=mail) new_attr_dict['mail'] = user['mail'] if 'mail' in new_attr_dict: env_dict["YNH_USER_MAILS"] = ','.join(new_attr_dict['mail']) if add_mailforward: if not isinstance(add_mailforward, list): add_mailforward = [add_mailforward] for mail in add_mailforward: if mail in user['maildrop'][1:]: continue user['maildrop'].append(mail) new_attr_dict['maildrop'] = user['maildrop'] if remove_mailforward: if not isinstance(remove_mailforward, list): remove_mailforward = [remove_mailforward] for mail in remove_mailforward: if len(user['maildrop']) > 1 and mail in user['maildrop'][1:]: user['maildrop'].remove(mail) else: raise YunohostError('mail_forward_remove_failed', mail=mail) new_attr_dict['maildrop'] = user['maildrop'] if 'maildrop' in new_attr_dict: env_dict["YNH_USER_MAILFORWARDS"] = ','.join(new_attr_dict['maildrop']) if mailbox_quota is not None: new_attr_dict['mailuserquota'] = [mailbox_quota] env_dict["YNH_USER_MAILQUOTA"] = mailbox_quota operation_logger.start() try: ldap.update('uid=%s,ou=users' % username, new_attr_dict) except Exception as e: raise YunohostError('user_update_failed', user=username, error=e) # Trigger post_user_update hooks hook_callback('post_user_update', env=env_dict) logger.success(m18n.n('user_updated')) app_ssowatconf() return user_info(username)
def diagnosis_show(categories=[], issues=False, full=False, share=False, human_readable=False): if not os.path.exists(DIAGNOSIS_CACHE): logger.warning(m18n.n("diagnosis_never_ran_yet")) return # Get all the categories all_categories = _list_diagnosis_categories() all_categories_names = [category for category, _ in all_categories] # Check the requested category makes sense if categories == []: categories = all_categories_names else: unknown_categories = [ c for c in categories if c not in all_categories_names ] if unknown_categories: raise YunohostValidationError( "diagnosis_unknown_categories", categories=", ".join(unknown_categories)) # Fetch all reports all_reports = [] for category in categories: try: report = Diagnoser.get_cached_report(category) except Exception as e: logger.error( m18n.n("diagnosis_failed", category=category, error=str(e))) continue Diagnoser.i18n(report, force_remove_html_tags=share or human_readable) add_ignore_flag_to_issues(report) if not full: del report["timestamp"] del report["cached_for"] report["items"] = [ item for item in report["items"] if not item["ignored"] ] for item in report["items"]: del item["meta"] del item["ignored"] if "data" in item: del item["data"] if issues: report["items"] = [ item for item in report["items"] if item["status"] in ["WARNING", "ERROR"] ] # Ignore this category if no issue was found if not report["items"]: continue all_reports.append(report) if share: from yunohost.utils.yunopaste import yunopaste content = _dump_human_readable_reports(all_reports) url = yunopaste(content) logger.info(m18n.n("log_available_on_yunopaste", url=url)) if msettings.get("interface") == "api": return {"url": url} else: return elif human_readable: print(_dump_human_readable_reports(all_reports)) else: return {"reports": all_reports}
def domain_remove(operation_logger, domain, remove_apps=False, force=False): """ Delete domains Keyword argument: domain -- Domain to delete remove_apps -- Remove applications installed on the domain force -- Force the domain removal and don't not ask confirmation to remove apps if remove_apps is specified """ from yunohost.hook import hook_callback from yunohost.app import app_ssowatconf, app_info, app_remove from yunohost.utils.ldap import _get_ldap_interface # the 'force' here is related to the exception happening in domain_add ... # we don't want to check the domain exists because the ldap add may have # failed if not force and domain not in domain_list()['domains']: raise YunohostError('domain_name_unknown', domain=domain) # Check domain is not the main domain if domain == _get_maindomain(): other_domains = domain_list()["domains"] other_domains.remove(domain) if other_domains: raise YunohostError( "domain_cannot_remove_main", domain=domain, other_domains="\n * " + ("\n * ".join(other_domains)), ) else: raise YunohostError("domain_cannot_remove_main_add_new_one", domain=domain) # Check if apps are installed on the domain apps_on_that_domain = [] for app in _installed_apps(): settings = _get_app_settings(app) label = app_info(app)["name"] if settings.get("domain") == domain: apps_on_that_domain.append( (app, " - %s \"%s\" on https://%s%s" % (app, label, domain, settings["path"]) if "path" in settings else app)) if apps_on_that_domain: if remove_apps: if msettings.get('interface') == "cli" and not force: answer = msignals.prompt(m18n.n( 'domain_remove_confirm_apps_removal', apps="\n".join([x[1] for x in apps_on_that_domain]), answers='y/N'), color="yellow") if answer.upper() != "Y": raise YunohostError("aborting") for app, _ in apps_on_that_domain: app_remove(app) else: raise YunohostError('domain_uninstall_app_first', apps="\n".join( [x[1] for x in apps_on_that_domain])) operation_logger.start() ldap = _get_ldap_interface() try: ldap.remove("virtualdomain=" + domain + ",ou=domains") except Exception as e: raise YunohostError("domain_deletion_failed", domain=domain, error=e) os.system("rm -rf /etc/yunohost/certs/%s" % domain) # Sometime we have weird issues with the regenconf where some files # appears as manually modified even though they weren't touched ... # There are a few ideas why this happens (like backup/restore nginx # conf ... which we shouldnt do ...). This in turns creates funky # situation where the regenconf may refuse to re-create the conf # (when re-creating a domain..) # # So here we force-clear the has out of the regenconf if it exists. # This is a pretty ad hoc solution and only applied to nginx # because it's one of the major service, but in the long term we # should identify the root of this bug... _force_clear_hashes(["/etc/nginx/conf.d/%s.conf" % domain]) # And in addition we even force-delete the file Otherwise, if the file was # manually modified, it may not get removed by the regenconf which leads to # catastrophic consequences of nginx breaking because it can't load the # cert file which disappeared etc.. if os.path.exists("/etc/nginx/conf.d/%s.conf" % domain): _process_regen_conf("/etc/nginx/conf.d/%s.conf" % domain, new_conf=None, save=True) regen_conf(names=["nginx", "metronome", "dnsmasq", "postfix"]) app_ssowatconf() hook_callback("post_domain_remove", args=[domain]) logger.success(m18n.n("domain_deleted"))
def user_create( operation_logger, username, firstname, lastname, domain, password, mailbox_quota="0", mail=None, ): from yunohost.domain import domain_list, _get_maindomain from yunohost.hook import hook_callback from yunohost.utils.password import assert_password_is_strong_enough from yunohost.utils.ldap import _get_ldap_interface # Ensure sufficiently complex password assert_password_is_strong_enough("user", password) if mail is not None: logger.warning( "Packagers ! Using --mail in 'yunohost user create' is deprecated ... please use --domain instead." ) domain = mail.split("@")[-1] # Validate domain used for email address/xmpp account if domain is None: if msettings.get("interface") == "api": raise YunohostValidationError( "Invalid usage, you should specify a domain argument") else: # On affiche les differents domaines possibles msignals.display(m18n.n("domains_available")) for domain in domain_list()["domains"]: msignals.display("- {}".format(domain)) maindomain = _get_maindomain() domain = msignals.prompt( m18n.n("ask_user_domain") + " (default: %s)" % maindomain) if not domain: domain = maindomain # Check that the domain exists if domain not in domain_list()["domains"]: raise YunohostValidationError("domain_name_unknown", domain=domain) mail = username + "@" + domain ldap = _get_ldap_interface() if username in user_list()["users"]: raise YunohostValidationError("user_already_exists", user=username) # Validate uniqueness of username and mail in LDAP try: ldap.validate_uniqueness({ "uid": username, "mail": mail, "cn": username }) except Exception as e: raise YunohostValidationError("user_creation_failed", user=username, error=e) # Validate uniqueness of username in system users all_existing_usernames = {x.pw_name for x in pwd.getpwall()} if username in all_existing_usernames: raise YunohostValidationError("system_username_exists") main_domain = _get_maindomain() aliases = [ "root@" + main_domain, "admin@" + main_domain, "webmaster@" + main_domain, "postmaster@" + main_domain, "abuse@" + main_domain, ] if mail in aliases: raise YunohostValidationError("mail_unavailable") operation_logger.start() # Get random UID/GID all_uid = {str(x.pw_uid) for x in pwd.getpwall()} all_gid = {str(x.gr_gid) for x in grp.getgrall()} uid_guid_found = False while not uid_guid_found: # LXC uid number is limited to 65536 by default uid = str(random.randint(1001, 65000)) uid_guid_found = uid not in all_uid and uid not in all_gid # Adapt values for LDAP fullname = "%s %s" % (firstname, lastname) attr_dict = { "objectClass": [ "mailAccount", "inetOrgPerson", "posixAccount", "userPermissionYnh", ], "givenName": [firstname], "sn": [lastname], "displayName": [fullname], "cn": [fullname], "uid": [username], "mail": mail, # NOTE: this one seems to be already a list "maildrop": [username], "mailuserquota": [mailbox_quota], "userPassword": [_hash_user_password(password)], "gidNumber": [uid], "uidNumber": [uid], "homeDirectory": ["/home/" + username], "loginShell": ["/bin/false"], } # If it is the first user, add some aliases if not ldap.search(base="ou=users,dc=yunohost,dc=org", filter="uid=*"): attr_dict["mail"] = [attr_dict["mail"]] + aliases try: ldap.add("uid=%s,ou=users" % username, attr_dict) except Exception as e: raise YunohostError("user_creation_failed", user=username, error=e) # Invalidate passwd and group to take user and group creation into account subprocess.call(["nscd", "-i", "passwd"]) subprocess.call(["nscd", "-i", "group"]) try: # Attempt to create user home folder subprocess.check_call(["mkhomedir_helper", username]) except subprocess.CalledProcessError: if not os.path.isdir("/home/{0}".format(username)): logger.warning(m18n.n("user_home_creation_failed"), exc_info=1) # Create group for user and add to group 'all_users' user_group_create(groupname=username, gid=uid, primary_group=True, sync_perm=False) user_group_update(groupname="all_users", add=username, force=True, sync_perm=True) # Trigger post_user_create hooks env_dict = { "YNH_USER_USERNAME": username, "YNH_USER_MAIL": mail, "YNH_USER_PASSWORD": password, "YNH_USER_FIRSTNAME": firstname, "YNH_USER_LASTNAME": lastname, } hook_callback("post_user_create", args=[username, mail], env=env_dict) # TODO: Send a welcome mail to user logger.success(m18n.n("user_created")) return {"fullname": fullname, "username": username, "mail": mail}
def log_list(limit=None, with_details=False, with_suboperations=False): """ List available logs Keyword argument: limit -- Maximum number of logs with_details -- Include details (e.g. if the operation was a success). Likely to increase the command time as it needs to open and parse the metadata file for each log... with_suboperations -- Include operations that are not the "main" operation but are sub-operations triggered by another ongoing operation ... (e.g. initializing groups/permissions when installing an app) """ operations = {} logs = [ x for x in os.listdir(OPERATIONS_PATH) if x.endswith(METADATA_FILE_EXT) ] logs = list(reversed(sorted(logs))) if limit is not None: logs = logs[:limit] for log in logs: base_filename = log[:-len(METADATA_FILE_EXT)] md_path = os.path.join(OPERATIONS_PATH, log) entry = { "name": base_filename, "path": md_path, "description": _get_description_from_name(base_filename), } try: entry["started_at"] = _get_datetime_from_name(base_filename) except ValueError: pass try: metadata = (read_yaml(md_path) or {}) # Making sure this is a dict and not None..? except Exception as e: # If we can't read the yaml for some reason, report an error and ignore this entry... logger.error( m18n.n("log_corrupted_md_file", md_file=md_path, error=e)) continue if with_details: entry["success"] = metadata.get("success", "?") entry["parent"] = metadata.get("parent") if with_suboperations: entry["parent"] = metadata.get("parent") entry["suboperations"] = [] elif metadata.get("parent") is not None: continue operations[base_filename] = entry # When displaying suboperations, we build a tree-like structure where # "suboperations" is a list of suboperations (each of them may also have a list of # "suboperations" suboperations etc... if with_suboperations: suboperations = [ o for o in operations.values() if o["parent"] is not None ] for suboperation in suboperations: parent = operations.get(suboperation["parent"]) if not parent: continue parent["suboperations"].append(suboperation) operations = [o for o in operations.values() if o["parent"] is None] else: operations = [o for o in operations.values()] operations = list(reversed(sorted(operations, key=lambda o: o["name"]))) # Reverse the order of log when in cli, more comfortable to read (avoid # unecessary scrolling) is_api = msettings.get("interface") == "api" if not is_api: operations = list(reversed(operations)) return {"operation": operations}
def log_show(path, number=None, share=False, filter_irrelevant=False, with_suboperations=False): """ Display a log file enriched with metadata if any. If the file_name is not an absolute path, it will try to search the file in the unit operations log path (see OPERATIONS_PATH). Argument: file_name number share """ if share: filter_irrelevant = True if filter_irrelevant: filters = [ r"set [+-]x$", r"set [+-]o xtrace$", r"local \w+$", r"local legacy_args=.*$", r".*Helper used in legacy mode.*", r"args_array=.*$", r"local -A args_array$", r"ynh_handle_getopts_args", r"ynh_script_progression", ] else: filters = [] def _filter_lines(lines, filters=[]): filters = [re.compile(f) for f in filters] return [ line for line in lines if not any(f.search(line.strip()) for f in filters) ] # Normalize log/metadata paths and filenames abs_path = path log_path = None if not path.startswith("/"): abs_path = os.path.join(OPERATIONS_PATH, path) if os.path.exists(abs_path) and not path.endswith(METADATA_FILE_EXT): log_path = abs_path if abs_path.endswith(METADATA_FILE_EXT) or abs_path.endswith(LOG_FILE_EXT): base_path = "".join(os.path.splitext(abs_path)[:-1]) else: base_path = abs_path base_filename = os.path.basename(base_path) md_path = base_path + METADATA_FILE_EXT if log_path is None: log_path = base_path + LOG_FILE_EXT if not os.path.exists(md_path) and not os.path.exists(log_path): raise YunohostValidationError("log_does_exists", log=path) infos = {} # If it's a unit operation, display the name and the description if base_path.startswith(CATEGORIES_PATH): infos["description"] = _get_description_from_name(base_filename) infos["name"] = base_filename if share: from yunohost.utils.yunopaste import yunopaste content = "" if os.path.exists(md_path): content += read_file(md_path) content += "\n============\n\n" if os.path.exists(log_path): actual_log = read_file(log_path) content += "\n".join(_filter_lines(actual_log.split("\n"), filters)) url = yunopaste(content) logger.info(m18n.n("log_available_on_yunopaste", url=url)) if msettings.get("interface") == "api": return {"url": url} else: return # Display metadata if exist if os.path.exists(md_path): try: metadata = read_yaml(md_path) except MoulinetteError as e: error = m18n.n("log_corrupted_md_file", md_file=md_path, error=e) if os.path.exists(log_path): logger.warning(error) else: raise YunohostError(error) else: infos["metadata_path"] = md_path infos["metadata"] = metadata if "log_path" in metadata: log_path = metadata["log_path"] if with_suboperations: def suboperations(): try: log_start = _get_datetime_from_name(base_filename) except ValueError: return for filename in os.listdir(OPERATIONS_PATH): if not filename.endswith(METADATA_FILE_EXT): continue # We first retrict search to a ~48h time window to limit the number # of .yml we look into try: date = _get_datetime_from_name(base_filename) except ValueError: continue if (date < log_start) or ( date > log_start + timedelta(hours=48)): continue try: submetadata = read_yaml( os.path.join(OPERATIONS_PATH, filename)) except Exception: continue if submetadata and submetadata.get( "parent") == base_filename: yield { "name": filename[:-len(METADATA_FILE_EXT)], "description": _get_description_from_name( filename[:-len(METADATA_FILE_EXT)]), "success": submetadata.get("success", "?"), } metadata["suboperations"] = list(suboperations()) # Display logs if exist if os.path.exists(log_path): from yunohost.service import _tail if number and filters: logs = _tail(log_path, int(number * 4)) elif number: logs = _tail(log_path, int(number)) else: logs = read_file(log_path) logs = _filter_lines(logs, filters) if number: logs = logs[-number:] infos["log_path"] = log_path infos["logs"] = logs return infos
def _hook_exec_bash(path, args, chdir, env, return_format, loggers): from moulinette.utils.process import call_async_output # Construct command variables cmd_args = "" if args and isinstance(args, list): # Concatenate escaped arguments cmd_args = " ".join(shell_quote(s) for s in args) if not chdir: # use the script directory as current one chdir, cmd_script = os.path.split(path) cmd_script = "./{0}".format(cmd_script) else: cmd_script = path # Add Execution dir to environment var if env is None: env = {} env["YNH_CWD"] = chdir env["YNH_INTERFACE"] = msettings.get("interface") stdreturn = os.path.join(tempfile.mkdtemp(), "stdreturn") with open(stdreturn, "w") as f: f.write("") env["YNH_STDRETURN"] = stdreturn # use xtrace on fd 7 which is redirected to stdout env["BASH_XTRACEFD"] = "7" cmd = '/bin/bash -x "{script}" {args} 7>&1' cmd = cmd.format(script=cmd_script, args=cmd_args) logger.debug("Executing command '%s'" % cmd) _env = os.environ.copy() _env.update(env) returncode = call_async_output(cmd, loggers, shell=True, cwd=chdir, env=_env) raw_content = None try: with open(stdreturn, "r") as f: raw_content = f.read() returncontent = {} if return_format == "json": if raw_content != "": try: returncontent = read_json(stdreturn) except Exception as e: raise YunohostError( "hook_json_return_error", path=path, msg=str(e), raw_content=raw_content, ) elif return_format == "plain_dict": for line in raw_content.split("\n"): if "=" in line: key, value = line.strip().split("=", 1) returncontent[key] = value else: raise YunohostError( "Expected value for return_format is either 'json' or 'plain_dict', got '%s'" % return_format) finally: stdreturndir = os.path.split(stdreturn)[0] os.remove(stdreturn) os.rmdir(stdreturndir) return returncode, returncontent