def test_read_yaml_cannot_read(test_yaml, mocker): error = "foobar" mocker.patch("yaml.safe_load", side_effect=Exception(error)) with pytest.raises(MoulinetteError) as exception: read_yaml(str(test_yaml)) translation = m18n.g("corrupted_yaml", ressource=str(test_yaml), error=error) expected_msg = translation.format(ressource=str(test_yaml), error=error) assert expected_msg in str(exception)
def generate_cache(): logger.debug("generating cache for actions map") # Read actions map from yaml file actionsmap = read_yaml(actionsmap_yml) if not actionsmap["_global"].get("cache", True): return actionsmap # Delete old cache files for old_cache in glob.glob( f"{actionsmap_yml_dir}/.{actionsmap_yml_file}.*.pkl" ): os.remove(old_cache) # at installation, cachedir might not exists dir_ = os.path.dirname(actionsmap_pkl) if not os.path.isdir(dir_): os.makedirs(dir_) # Cache actions map into pickle file with open(actionsmap_pkl, "wb") as f: pickle.dump(actionsmap, f) return actionsmap
def tools_migrations_state(): """ Show current migration state """ if not os.path.exists(MIGRATIONS_STATE_PATH): return {"migrations": {}} return read_yaml(MIGRATIONS_STATE_PATH)
def text_write_list_to_yaml(tmp_path): new_file = tmp_path / "newfile.yaml" dummy_list = ["foo", "bar", "baz"] write_to_yaml(str(new_file), dummy_list) _yaml = read_yaml(str(new_file)) assert _yaml == ["foo", "bar", "baz"]
def run(self, *args): from yunohost.utils.ldap import _get_ldap_interface ldap = _get_ldap_interface() existing_perms_raw = ldap.search( "ou=permission,dc=yunohost,dc=org", "(objectclass=permissionYnh)", ["cn"] ) existing_perms = [perm["cn"][0] for perm in existing_perms_raw] # Add SSH and SFTP permissions ldap_map = read_yaml( "/usr/share/yunohost/yunohost-config/moulinette/ldap_scheme.yml" ) if "sftp.main" not in existing_perms: ldap.add( "cn=sftp.main,ou=permission", ldap_map["depends_children"]["cn=sftp.main,ou=permission"], ) if "ssh.main" not in existing_perms: ldap.add( "cn=ssh.main,ou=permission", ldap_map["depends_children"]["cn=ssh.main,ou=permission"], ) # Add a bash terminal to each users users = ldap.search( "ou=users,dc=yunohost,dc=org", filter="(loginShell=*)", attrs=["dn", "uid", "loginShell"], ) for user in users: if user["loginShell"][0] == "/bin/false": dn = user["dn"][0].replace(",dc=yunohost,dc=org", "") ldap.update(dn, {"loginShell": ["/bin/bash"]}) else: user_permission_update( "ssh.main", add=user["uid"][0], sync_perm=False ) permission_sync_to_user() # Somehow this is needed otherwise the PAM thing doesn't forget about the # old loginShell value ? subprocess.call(["nscd", "-i", "passwd"]) if ( "/etc/ssh/sshd_config" in manually_modified_files() and os.system( "grep -q '^ *AllowGroups\\|^ *AllowUsers' /etc/ssh/sshd_config" ) != 0 ): logger.error(m18n.n("diagnosis_sshd_config_insecure"))
def migrate_LDAP_db(): logger.info(m18n.n("migration_0011_update_LDAP_database")) from yunohost.utils.ldap import _get_ldap_interface ldap = _get_ldap_interface() ldap_map = read_yaml( '/usr/share/yunohost/yunohost-config/moulinette/ldap_scheme.yml') try: SetupGroupPermissions.remove_if_exists("ou=permission") SetupGroupPermissions.remove_if_exists('ou=groups') attr_dict = ldap_map['parents']['ou=permission'] ldap.add('ou=permission', attr_dict) attr_dict = ldap_map['parents']['ou=groups'] ldap.add('ou=groups', attr_dict) attr_dict = ldap_map['children']['cn=all_users,ou=groups'] ldap.add('cn=all_users,ou=groups', attr_dict) attr_dict = ldap_map['children']['cn=visitors,ou=groups'] ldap.add('cn=visitors,ou=groups', attr_dict) for rdn, attr_dict in ldap_map['depends_children'].items(): ldap.add(rdn, attr_dict) except Exception as e: raise YunohostError("migration_0011_LDAP_update_failed", error=e) logger.info(m18n.n("migration_0011_create_group")) # Create a group for each yunohost user user_list = ldap.search( 'ou=users,dc=yunohost,dc=org', '(&(objectclass=person)(!(uid=root))(!(uid=nobody)))', ['uid', 'uidNumber']) for user_info in user_list: username = user_info['uid'][0] ldap.update( 'uid=%s,ou=users' % username, { 'objectClass': [ 'mailAccount', 'inetOrgPerson', 'posixAccount', 'userPermissionYnh' ] }) user_group_create(username, gid=user_info['uidNumber'][0], primary_group=True, sync_perm=False) user_group_update(groupname='all_users', add=username, force=True, sync_perm=False)
def check_blacklist(self): """ Check with dig onto blacklist DNS server This check is ran on IPs and domains we could used to send mail. """ dns_blacklists = read_yaml(DEFAULT_DNS_BLACKLIST) for item in self.ips + self.mail_domains: for blacklist in dns_blacklists: item_type = "domain" if ":" in item: item_type = "ipv6" elif re.match(r"^\d+\.\d+\.\d+\.\d+$", item): item_type = "ipv4" if not blacklist[item_type]: continue # Build the query for DNSBL subdomain = item if item_type != "domain": rev = dns.reversename.from_address(item) subdomain = str(rev.split(3)[0]) query = subdomain + "." + blacklist["dns_server"] # Do the DNS Query status, _ = dig(query, "A") if status != "ok": continue # Try to get the reason details = [] status, answers = dig(query, "TXT") reason = "-" if status == "ok": reason = ", ".join(answers) details.append("diagnosis_mail_blacklist_reason") details.append("diagnosis_mail_blacklist_website") yield dict( meta={ "test": "mail_blacklist", "item": item, "blacklist": blacklist["dns_server"], }, data={ "blacklist_name": blacklist["name"], "blacklist_website": blacklist["website"], "reason": reason, }, status="ERROR", summary="diagnosis_mail_blacklist_listed_by", details=details, )
def check_blacklist(self): """ Check with dig onto blacklist DNS server This check is ran on IPs and domains we could used to send mail. """ dns_blacklists = read_yaml(DEFAULT_DNS_BLACKLIST) for item in self.ips + self.mail_domains: for blacklist in dns_blacklists: item_type = "domain" if ":" in item: item_type = 'ipv6' elif re.match(r'^\d+\.\d+\.\d+\.\d+$', item): item_type = 'ipv4' if not blacklist[item_type]: continue # Build the query for DNSBL subdomain = item if item_type != "domain": rev = dns.reversename.from_address(item) subdomain = str(rev.split(3)[0]) query = subdomain + '.' + blacklist['dns_server'] # Do the DNS Query status, _ = dig(query, 'A') if status != 'ok': continue # Try to get the reason details = [] status, answers = dig(query, 'TXT') reason = "-" if status == 'ok': reason = ', '.join(answers) details.append("diagnosis_mail_blacklist_reason") details.append("diagnosis_mail_blacklist_website") yield dict(meta={ "test": "mail_blacklist", "item": item, "blacklist": blacklist["dns_server"] }, data={ 'blacklist_name': blacklist['name'], 'blacklist_website': blacklist['website'], 'reason': reason }, status="ERROR", summary='diagnosis_mail_blacklist_listed_by', details=details)
def test_write_dict_to_yaml(tmp_path): new_file = tmp_path / "newfile.yaml" dummy_dict = {"foo": 42, "bar": ["a", "b", "c"]} write_to_yaml(str(new_file), dummy_dict) _yaml = read_yaml(str(new_file)) assert "foo" in _yaml.keys() assert "bar" in _yaml.keys() assert _yaml["foo"] == 42 assert _yaml["bar"] == ["a", "b", "c"]
def suboperations(): try: log_start = _get_datetime_from_name(base_filename) except ValueError: return for filename in os.listdir(OPERATIONS_PATH): if not filename.endswith(METADATA_FILE_EXT): continue # We first retrict search to a ~48h time window to limit the number # of .yml we look into try: date = _get_datetime_from_name(base_filename) except ValueError: continue if (date < log_start) or ( date > log_start + timedelta(hours=48)): continue try: submetadata = read_yaml( os.path.join(OPERATIONS_PATH, filename)) except Exception: continue if submetadata and submetadata.get( "parent") == base_filename: yield { "name": filename[:-len(METADATA_FILE_EXT)], "description": _get_description_from_name( filename[:-len(METADATA_FILE_EXT)]), "success": submetadata.get("success", "?"), }
def _diagnosis_read_configuration(): if not os.path.exists(DIAGNOSIS_CONFIG_FILE): return {} return read_yaml(DIAGNOSIS_CONFIG_FILE)
def log_list(limit=None, with_details=False, with_suboperations=False): """ List available logs Keyword argument: limit -- Maximum number of logs with_details -- Include details (e.g. if the operation was a success). Likely to increase the command time as it needs to open and parse the metadata file for each log... with_suboperations -- Include operations that are not the "main" operation but are sub-operations triggered by another ongoing operation ... (e.g. initializing groups/permissions when installing an app) """ operations = {} logs = [ x for x in os.listdir(OPERATIONS_PATH) if x.endswith(METADATA_FILE_EXT) ] logs = list(reversed(sorted(logs))) if limit is not None: logs = logs[:limit] for log in logs: base_filename = log[:-len(METADATA_FILE_EXT)] md_path = os.path.join(OPERATIONS_PATH, log) entry = { "name": base_filename, "path": md_path, "description": _get_description_from_name(base_filename), } try: entry["started_at"] = _get_datetime_from_name(base_filename) except ValueError: pass try: metadata = (read_yaml(md_path) or {}) # Making sure this is a dict and not None..? except Exception as e: # If we can't read the yaml for some reason, report an error and ignore this entry... logger.error( m18n.n("log_corrupted_md_file", md_file=md_path, error=e)) continue if with_details: entry["success"] = metadata.get("success", "?") entry["parent"] = metadata.get("parent") if with_suboperations: entry["parent"] = metadata.get("parent") entry["suboperations"] = [] elif metadata.get("parent") is not None: continue operations[base_filename] = entry # When displaying suboperations, we build a tree-like structure where # "suboperations" is a list of suboperations (each of them may also have a list of # "suboperations" suboperations etc... if with_suboperations: suboperations = [ o for o in operations.values() if o["parent"] is not None ] for suboperation in suboperations: parent = operations.get(suboperation["parent"]) if not parent: continue parent["suboperations"].append(suboperation) operations = [o for o in operations.values() if o["parent"] is None] else: operations = [o for o in operations.values()] operations = list(reversed(sorted(operations, key=lambda o: o["name"]))) # Reverse the order of log when in cli, more comfortable to read (avoid # unecessary scrolling) is_api = msettings.get("interface") == "api" if not is_api: operations = list(reversed(operations)) return {"operation": operations}
def log_show(path, number=None, share=False, filter_irrelevant=False, with_suboperations=False): """ Display a log file enriched with metadata if any. If the file_name is not an absolute path, it will try to search the file in the unit operations log path (see OPERATIONS_PATH). Argument: file_name number share """ if share: filter_irrelevant = True if filter_irrelevant: filters = [ r"set [+-]x$", r"set [+-]o xtrace$", r"local \w+$", r"local legacy_args=.*$", r".*Helper used in legacy mode.*", r"args_array=.*$", r"local -A args_array$", r"ynh_handle_getopts_args", r"ynh_script_progression", ] else: filters = [] def _filter_lines(lines, filters=[]): filters = [re.compile(f) for f in filters] return [ line for line in lines if not any(f.search(line.strip()) for f in filters) ] # Normalize log/metadata paths and filenames abs_path = path log_path = None if not path.startswith("/"): abs_path = os.path.join(OPERATIONS_PATH, path) if os.path.exists(abs_path) and not path.endswith(METADATA_FILE_EXT): log_path = abs_path if abs_path.endswith(METADATA_FILE_EXT) or abs_path.endswith(LOG_FILE_EXT): base_path = "".join(os.path.splitext(abs_path)[:-1]) else: base_path = abs_path base_filename = os.path.basename(base_path) md_path = base_path + METADATA_FILE_EXT if log_path is None: log_path = base_path + LOG_FILE_EXT if not os.path.exists(md_path) and not os.path.exists(log_path): raise YunohostValidationError("log_does_exists", log=path) infos = {} # If it's a unit operation, display the name and the description if base_path.startswith(CATEGORIES_PATH): infos["description"] = _get_description_from_name(base_filename) infos["name"] = base_filename if share: from yunohost.utils.yunopaste import yunopaste content = "" if os.path.exists(md_path): content += read_file(md_path) content += "\n============\n\n" if os.path.exists(log_path): actual_log = read_file(log_path) content += "\n".join(_filter_lines(actual_log.split("\n"), filters)) url = yunopaste(content) logger.info(m18n.n("log_available_on_yunopaste", url=url)) if msettings.get("interface") == "api": return {"url": url} else: return # Display metadata if exist if os.path.exists(md_path): try: metadata = read_yaml(md_path) except MoulinetteError as e: error = m18n.n("log_corrupted_md_file", md_file=md_path, error=e) if os.path.exists(log_path): logger.warning(error) else: raise YunohostError(error) else: infos["metadata_path"] = md_path infos["metadata"] = metadata if "log_path" in metadata: log_path = metadata["log_path"] if with_suboperations: def suboperations(): try: log_start = _get_datetime_from_name(base_filename) except ValueError: return for filename in os.listdir(OPERATIONS_PATH): if not filename.endswith(METADATA_FILE_EXT): continue # We first retrict search to a ~48h time window to limit the number # of .yml we look into try: date = _get_datetime_from_name(base_filename) except ValueError: continue if (date < log_start) or ( date > log_start + timedelta(hours=48)): continue try: submetadata = read_yaml( os.path.join(OPERATIONS_PATH, filename)) except Exception: continue if submetadata and submetadata.get( "parent") == base_filename: yield { "name": filename[:-len(METADATA_FILE_EXT)], "description": _get_description_from_name( filename[:-len(METADATA_FILE_EXT)]), "success": submetadata.get("success", "?"), } metadata["suboperations"] = list(suboperations()) # Display logs if exist if os.path.exists(log_path): from yunohost.service import _tail if number and filters: logs = _tail(log_path, int(number * 4)) elif number: logs = _tail(log_path, int(number)) else: logs = read_file(log_path) logs = _filter_lines(logs, filters) if number: logs = logs[-number:] infos["log_path"] = log_path infos["logs"] = logs return infos
def translate_legacy_rules_in_ssowant_conf_json_persistent(): persistent_file_name = "/etc/ssowat/conf.json.persistent" if not os.path.exists(persistent_file_name): return # Ugly hack to try not to misarably fail migration persistent = read_yaml(persistent_file_name) legacy_rules = [ "skipped_urls", "unprotected_urls", "protected_urls", "skipped_regex", "unprotected_regex", "protected_regex" ] if not any(legacy_rule in persistent for legacy_rule in legacy_rules): return if not isinstance(persistent.get("permissions"), dict): persistent["permissions"] = {} skipped_urls = persistent.get("skipped_urls", []) + [ "re:" + r for r in persistent.get("skipped_regex", []) ] protected_urls = persistent.get("protected_urls", []) + [ "re:" + r for r in persistent.get("protected_regex", []) ] unprotected_urls = persistent.get("unprotected_urls", []) + [ "re:" + r for r in persistent.get("unprotected_regex", []) ] known_users = list(user_list()["users"].keys()) for legacy_rule in legacy_rules: if legacy_rule in persistent: del persistent[legacy_rule] if skipped_urls: persistent["permissions"]['custom_skipped'] = { "users": [], "label": "Custom permissions - skipped", "show_tile": False, "auth_header": False, "public": True, "uris": skipped_urls + persistent["permissions"].get("custom_skipped", {}).get("uris", []), } if unprotected_urls: persistent["permissions"]['custom_unprotected'] = { "users": [], "label": "Custom permissions - unprotected", "show_tile": False, "auth_header": True, "public": True, "uris": unprotected_urls + persistent["permissions"].get( "custom_unprotected", {}).get("uris", []), } if protected_urls: persistent["permissions"]['custom_protected'] = { "users": known_users, "label": "Custom permissions - protected", "show_tile": False, "auth_header": True, "public": False, "uris": protected_urls + persistent["permissions"].get( "custom_protected", {}).get("uris", []), } write_to_json(persistent_file_name, persistent, sort_keys=True, indent=4) logger.warning( "Yunohost automatically translated some legacy rules in /etc/ssowat/conf.json.persistent to match the new permission system" )
def test_read_yaml(test_yaml): content = read_yaml(str(test_yaml)) assert "foo" in content.keys() assert content["foo"] == "bar"
def migrate_LDAP_db(): logger.info(m18n.n("migration_0011_update_LDAP_database")) from yunohost.utils.ldap import _get_ldap_interface ldap = _get_ldap_interface() ldap_map = read_yaml( "/usr/share/yunohost/yunohost-config/moulinette/ldap_scheme.yml" ) try: SetupGroupPermissions.remove_if_exists("ou=permission") SetupGroupPermissions.remove_if_exists("ou=groups") attr_dict = ldap_map["parents"]["ou=permission"] ldap.add("ou=permission", attr_dict) attr_dict = ldap_map["parents"]["ou=groups"] ldap.add("ou=groups", attr_dict) attr_dict = ldap_map["children"]["cn=all_users,ou=groups"] ldap.add("cn=all_users,ou=groups", attr_dict) attr_dict = ldap_map["children"]["cn=visitors,ou=groups"] ldap.add("cn=visitors,ou=groups", attr_dict) for rdn, attr_dict in ldap_map["depends_children"].items(): ldap.add(rdn, attr_dict) except Exception as e: raise YunohostError("migration_0011_LDAP_update_failed", error=e) logger.info(m18n.n("migration_0011_create_group")) # Create a group for each yunohost user user_list = ldap.search( "ou=users,dc=yunohost,dc=org", "(&(objectclass=person)(!(uid=root))(!(uid=nobody)))", ["uid", "uidNumber"], ) for user_info in user_list: username = user_info["uid"][0] ldap.update( "uid=%s,ou=users" % username, { "objectClass": [ "mailAccount", "inetOrgPerson", "posixAccount", "userPermissionYnh", ] }, ) user_group_create( username, gid=user_info["uidNumber"][0], primary_group=True, sync_perm=False, ) user_group_update( groupname="all_users", add=username, force=True, sync_perm=False )