def cleanup_if_necessary(parent_dirname: str) -> None: """ Process the cleanup if it's necessary. """ cleanup_tool = FilesystemCleanup(parent_dirname) running_file_helper = FileHelper( os.path.join( cleanup_tool.get_output_basedir(), PyFunceble.cli.storage.TEST_RUNNING_FILE, )) if not running_file_helper.exists(): self.sessions_id[parent_dirname] = secrets.token_hex(12) cleanup_tool.clean_output_files() running_file_helper.write( f"{self.sessions_id[parent_dirname]} " f"{datetime.datetime.utcnow().isoformat()}", overwrite=True, ) else: possible_session_id = running_file_helper.read().split()[0] try: _ = datetime.datetime.fromisoformat(possible_session_id) self.sessions_id[parent_dirname] = None except (ValueError, TypeError): self.sessions_id[parent_dirname] = possible_session_id
def get_backup_data(self) -> dict: """ Provides the data which acts as a backup. """ result = {} base_dir = self.get_output_basedir() file_helper = FileHelper() hash_helper = HashHelper() for file in DirectoryHelper(base_dir).list_all_files(): file_helper.set_path(file) reduced_path = self.get_path_without_base_dir(file) directory, filename = reduced_path.rsplit(os.sep, 1) file_hash = hash_helper.hash_file(file_helper.path) dataset = { filename: { "content": file_helper.read(), "hash": file_hash } } if directory not in result: result[directory] = dataset else: result[directory].update(dataset) PyFunceble.facility.Logger.debug("Dir Structure to backup:\n%r", result) return result
def test_open(self) -> None: """ Tests the method which let us open the given file as we want. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() self.assertEqual(expected, actual) with file_helper.open("w") as file_stream: file_stream.write("Hello, World!") expected = True actual = file_helper.exists() self.assertEqual(expected, actual) expected = "Hello, World!" actual = file_helper.read() self.assertEqual(expected, actual)
def test_move(self) -> None: """ Tests of the method which let us move a file to another location. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) destination_file_helper = FileHelper(tempfile.gettempdir()) destination_file_helper.set_path( destination_file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() actual_destination = destination_file_helper.exists() self.assertEqual(expected, actual) self.assertEqual(expected, actual_destination) file_helper.write("Hello, World!") expected = True actual = file_helper.exists() self.assertEqual(expected, actual) expected = False actual_destination = destination_file_helper.exists() self.assertEqual(expected, actual_destination) file_helper.move(destination_file_helper.path) expected = True actual_destination = destination_file_helper.exists() self.assertEqual(expected, actual_destination) expected = "Hello, World!" actual = destination_file_helper.read() self.assertEqual(expected, actual) expected = False actual = file_helper.exists() self.assertEqual(expected, actual) expected = True actual_destination = destination_file_helper.exists() self.assertEqual(expected, actual_destination)
def test_copy(self) -> None: """ Tests the method which let us copy a file to another place. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) copy_file_helper = FileHelper(tempfile.gettempdir()) copy_file_helper.set_path( copy_file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() actual_copy = copy_file_helper.exists() self.assertEqual(expected, actual) self.assertEqual(expected, actual_copy) file_helper.write("Hello, World!") expected = True actual = file_helper.exists() self.assertEqual(expected, actual) expected = False actual_copy = copy_file_helper.exists() self.assertEqual(expected, actual_copy) file_helper.copy(copy_file_helper.path) expected = True actual_copy = copy_file_helper.exists() self.assertEqual(expected, actual_copy) expected = "Hello, World!" actual = copy_file_helper.read() self.assertEqual(expected, actual) expected = True actual = file_helper.exists() actual_copy = copy_file_helper.exists() self.assertEqual(expected, actual) self.assertEqual(expected, actual_copy)
def test_read(self) -> None: """ Tests the method which let us read a file. """ given = tempfile.NamedTemporaryFile(delete=False) file_helper = FileHelper(given.name) file_helper.write("Hello, World!") given.seek(0) expected = "Hello, World!" actual = file_helper.read() self.assertEqual(expected, actual)
def test_read_file_does_not_exists(self) -> None: """ Tests the method which let us read a file for the case that the given file does not exists. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() self.assertEqual(expected, actual) expected = None actual = file_helper.read() self.assertEqual(expected, actual)
class Administration: """ Provides the administration interface. """ info_file_location: Optional[str] = None info_file_helper: Optional[FileHelper] = None __our_info: dict = dict() def __init__(self) -> None: self.info_file_location = outputs.ADMINISTRATION_DESTINATION self.info_file_helper = FileHelper(self.info_file_location) self.__our_info.update(self.load()) def __contains__(self, index: str) -> bool: return index in self.__our_info def __getitem__(self, index: str) -> Any: if index in self.__our_info: return self.__our_info[index] raise KeyError(index) def __getattr__(self, index: str) -> Any: if index in self.__our_info: return self.__our_info[index] raise AttributeError(index) def __setattr__(self, index: str, value: Any) -> None: if self.__our_info: self.__our_info[index] = value else: super().__setattr__(index, value) def __delattr__(self, index: str) -> None: if index in self.info: del self.info[index] def __del__(self) -> None: self.save() @property def exists(self) -> bool: """ Checks if the administration file exists. """ return self.info_file_helper.exists() @staticmethod def convert_data_for_system(data: dict) -> dict: """ Given the content of the info file, we convert it into something our system may understand. .. warning:: This method also delete the keys that are scheduled or declared for deletion :param data: The data to work with. """ result = dict() for key, value in dict(data).items(): if key in infrastructure.ADMINISTRATION_INDEXES["delete"]: continue if key == "name": result[key] = ( CommandHelper("basename $(git rev-parse --show-toplevel)" ).execute().strip()) elif key in infrastructure.ADMINISTRATION_INDEXES["bool"]: result[key] = bool(int(value)) elif key in infrastructure.ADMINISTRATION_INDEXES["int"]: result[key] = int(value) elif key in infrastructure.ADMINISTRATION_INDEXES["dict"]: result[key] = dict(value) elif key in infrastructure.ADMINISTRATION_INDEXES["datetime"]: try: result[key] = datetime.fromisoformat(value) except ValueError: result[key] = datetime.utcnow() elif key in infrastructure.ADMINISTRATION_INDEXES["epoch"]: result[key] = datetime.fromtimestamp(value) else: result[key] = value if "pyfunceble" not in result: result["pyfunceble"] = { "config": {}, } for sanitize_type, keys in infrastructure.ADMINISTRATION_INDEXES.items( ): if sanitize_type == "delete": continue for key in keys: if key not in result: if sanitize_type == "bool": local_result = bool(None) elif sanitize_type == "int": local_result = 0 elif sanitize_type == "dict": local_result = dict() elif sanitize_type == "datetime": local_result = datetime.utcnow() - timedelta( days=365.25) elif sanitize_type == "epoch": local_result = datetime.utcnow() - timedelta( days=365.25) else: local_result = None result[key] = local_result return result @staticmethod def convert_data_for_file(data: dict) -> dict: """ Given the content of the info file, we convert it into something that can be saved into a JSON file. .. warning:: This method also delete the keys that are scheduled or declared for deletion :param data: The data to work with. """ result = dict() for key, value in dict(data).items(): if key in infrastructure.ADMINISTRATION_INDEXES["delete"]: continue if key in infrastructure.ADMINISTRATION_INDEXES["bool"]: result[key] = bool(value) elif key in infrastructure.ADMINISTRATION_INDEXES["int"]: result[key] = int(value) elif key in infrastructure.ADMINISTRATION_INDEXES["datetime"]: result[key] = value.isoformat() elif key in infrastructure.ADMINISTRATION_INDEXES["epoch"]: result[key] = value.timestamp() else: result[key] = value return result def load(self) -> dict: """ Loads and return the content of the administration file. """ if self.info_file_helper.read(): content = self.info_file_helper.read() logging.debug("Administration file content:\n%s", content) return self.convert_data_for_system(DictHelper().from_json( content, return_dict_on_error=False)) return dict() def save(self) -> None: """ Saves the loaded content of the administration file. """ if self.info_file_helper.exists(): DictHelper(self.convert_data_for_file( self.__our_info)).to_json_file(self.info_file_location)
repo_slug = f"{ORG_NAME}/{repo.name}" clone_url = CLONE_TEMPLATE.replace("%slug%", repo_slug) clone_destination = f"{CLONE_DIR}/{repo.name}" if not DirectoryHelper(clone_destination).exists(): response = CommandHelper(f"git clone {clone_url} {clone_destination}").run() else: response = CommandHelper(f"cd {clone_destination} && git pull").run() for line in response: print(line) config_file_instance = FileHelper( os.path.join(clone_destination, ".git", "config") ) config_file_content = config_file_instance.read() for root, _, files in os.walk(dir_helper.path): unformatted_root = root root = root.replace(dir_helper.path, "") if root.startswith("/"): root = root[1:] local_dir_helper = DirectoryHelper(os.path.join(clone_destination, root)) if not local_dir_helper.exists(): local_dir_helper.create() for file in files: file_helper.set_path(os.path.join(dir_helper.path, root, file)).copy(
class Orchestration: """ Orchester the test launch. """ info_manager: Optional[InfoManager] = None authorization_handler: Optional[Authorization] = None origin_file: Optional[FileHelper] = None output_file: Optional[FileHelper] = None def __init__(self, save: bool = False, end: bool = False) -> None: self.info_manager = InfoManager() git_name = EnvironmentVariableHelper("GIT_NAME") git_email = EnvironmentVariableHelper("GIT_EMAIL") if git_email.exists() and "funilrys" in git_email.get_value(): git_name.set_value(dead_hosts.launcher.defaults.envs.GIT_NAME) git_email.set_value(dead_hosts.launcher.defaults.envs.GIT_EMAIL) EnvironmentVariableHelper("PYFUNCEBLE_OUTPUT_LOCATION").set_value( self.info_manager.WORKSPACE_DIR) EnvironmentVariableHelper("PYFUNCEBLE_CONFIG_DIR").set_value( self.info_manager.PYFUNCEBLE_CONFIG_DIR) self.authorization_handler = Authorization(self.info_manager) self.origin_file = FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME, )) self.output_file = FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.OUTPUT_FILENAME, )) logging.info("Origin file: %r", self.origin_file.path) logging.info("Output file: %r", self.output_file.path) if not end and not save: logging.info("Checking authorization to run.") if self.authorization_handler.is_test_authorized(): execute_all_updater(self.info_manager) PyFunceble.facility.ConfigLoader.start() self.fetch_file_to_test() self.run_test() else: logging.info( "Not authorized to run a test until %r (current time) > %r", datetime.now(), self.authorization_handler.next_authorization_time, ) sys.exit(0) elif save: self.run_autosave() else: self.run_end() def fetch_file_to_test(self) -> "Orchestration": """ Provides the latest version of the file to test. """ if self.authorization_handler.is_refresh_authorized(): logging.info( "We are authorized to refresh the lists! Let's do that.") logging.info("Raw Link: %r", self.info_manager.raw_link) if self.info_manager.raw_link: DownloadHelper(self.info_manager.raw_link).download_text( destination=self.origin_file.path) logging.info( "Could get the new version of the list. Updating the download time." ) self.info_manager["last_download_datetime"] = datetime.utcnow() self.info_manager[ "last_download_timestamp"] = self.info_manager[ "last_download_datetime"].timestamp() elif self.origin_file.exists(): logging.info( "Raw link not given or is empty. Let's work with %r.", self.origin_file.path, ) self.origin_file.read() logging.info("Emptying the download time.") self.info_manager[ "last_download_datetime"] = datetime.fromtimestamp(0) self.info_manager[ "last_download_timestamp"] = self.info_manager[ "last_download_datetime"].timestamp() else: logging.info(f"Could not find {self.origin_file.path}. " "Generating empty content to test.") self.origin_file.write("# No content yet.", overwrite=True) logging.info("Emptying the download time.") self.info_manager[ "last_download_datetime"] = datetime.fromtimestamp(0) self.info_manager[ "last_download_timestamp"] = self.info_manager[ "last_download_datetime"].timestamp() logging.info("Updated %r.", self.origin_file.path) return self def run_test(self): """ Run a test of the input list. """ if not self.info_manager.currently_under_test: self.info_manager["currently_under_test"] = True self.info_manager["start_datetime"] = datetime.utcnow() self.info_manager["start_timestamp"] = self.info_manager[ "start_datetime"].timestamp() self.info_manager["finish_datetime"] = datetime.fromtimestamp(0) self.info_manager["finish_timestamp"] = self.info_manager[ "finish_datetime"].timestamp() self.info_manager["latest_part_start_datetime"] = datetime.utcnow() self.info_manager["latest_part_start_timestamp"] = self.info_manager[ "latest_part_start_datetime"].timestamp() self.info_manager[ "latest_part_finish_datetime"] = datetime.fromtimestamp(0) self.info_manager["latest_part_finish_timestamp"] = self.info_manager[ "latest_part_finish_datetime"].timestamp() logging.info("Updated all timestamps.") logging.info("Starting PyFunceble %r ...", PyFunceble.__version__) Command(f"PyFunceble -f {self.origin_file.path}").run_to_stdout() if not dead_hosts.launcher.defaults.envs.GITHUB_TOKEN: self.run_end() def run_autosave(self): """ Run the autosave logic of the administration file. .. warning:: This is just about the administration file not PyFunceble. """ self.info_manager["latest_part_finish_datetime"] = datetime.utcnow() self.info_manager["latest_part_finish_timestamp"] = self.info_manager[ "latest_part_finish_datetime"].timestamp() logging.info("Updated all timestamps.") def run_end(self): """ Run the end logic. """ self.info_manager["currently_under_test"] = False self.info_manager["latest_part_finish_datetime"] = datetime.utcnow() self.info_manager["latest_part_finish_timestamp"] = self.info_manager[ "latest_part_finish_datetime"].timestamp() self.info_manager["finish_datetime"] = self.info_manager[ "latest_part_finish_datetime"] self.info_manager["finish_timestamp"] = self.info_manager[ "finish_datetime"].timestamp() logging.info( "Updated all timestamps and indexes that needed to be updated.") pyfunceble_active_list = FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, "output", dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME, "domains", "ACTIVE", "list", )) clean_list = [ "# File generated by the Dead-Hosts project with the help of PyFunceble.", "# Dead-Hosts: https://github.com/dead-hosts", "# PyFunceble: https://pyfunceble.github.io", f"# Generation Time: {datetime.utcnow().isoformat()}", ] logging.info( f"PyFunceble ACTIVE list output: {pyfunceble_active_list.path}") if pyfunceble_active_list.exists(): logging.info( f"{pyfunceble_active_list.path} exists, getting and formatting its content." ) self.output_file.write("\n".join(clean_list) + "\n\n", overwrite=True) with pyfunceble_active_list.open("r", encoding="utf-8") as file_stream: for line in file_stream: if line.startswith("#"): continue self.output_file.write(line) self.output_file.write("\n") logging.info("Updated of the content of %r", self.output_file.path)
class PyFuncebleConfigUpdater(UpdaterBase): """ Provides the updated of the PyFunceble configuration. """ def __init__(self, info_manager: InfoManager) -> None: self.pyfunceble_config_file_instance = FileHelper( os.path.join(info_manager.PYFUNCEBLE_CONFIG_DIR, ".PyFunceble.yaml")) super().__init__(info_manager) @property def authorized(self) -> bool: return not self.info_manager.own_management @staticmethod def get_commit_message(message: str, ping: Optional[str] = None) -> str: """ Provides the commit message to use. """ if ping: return f"{message} | cc {ping} | " return message def pre(self) -> "PyFuncebleConfigUpdater": logging.info( "Started to update %r.", self.pyfunceble_config_file_instance.path, ) return self def post(self) -> "PyFuncebleConfigUpdater": logging.info( "Finished to update %r", self.pyfunceble_config_file_instance.path, ) return self def start(self) -> "PyFuncebleConfigUpdater": with importlib.resources.path( "PyFunceble.data.infrastructure", ".PyFunceble_production.yaml") as file_path: local_version = DictHelper(DictHelper().from_yaml_file( str(file_path))).flatten() local_version = Merge( dead_hosts.launcher.defaults.pyfunceble.CONFIGURATION).into( local_version, strict=True) if self.info_manager.custom_pyfunceble_config and isinstance( self.info_manager.custom_pyfunceble_config, dict): logging.info("Custom PyFunceble configuration given, " "appending them to the local configuration file.") local_version = Merge( self.info_manager.custom_pyfunceble_config).into(local_version, strict=True) if self.info_manager.ping: logging.info( "Ping names given, appending them to the commit message.") local_version[ "cli_testing.ci.end_commit_message"] = self.get_commit_message( local_version["cli_testing.ci.end_commit_message"], ping=self.info_manager.get_ping_for_commit(), ) local_version = Merge( dead_hosts.launcher.defaults.pyfunceble.PERSISTENT_CONFIG).into( local_version, strict=True) if FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.EXAMPLE_INFO_FILENAME, )).exists(): local_version["cli_testing.ci.active"] = False # Default behavior of PyFunceble since 4.0.0b12. local_version["cli_testing.autocontinue"] = False local_version = DictHelper(local_version).unflatten() DictHelper(local_version).to_yaml_file( self.pyfunceble_config_file_instance.path) logging.debug("Configuration:\n%s", self.pyfunceble_config_file_instance.read()) return self
class ReadmeUpdater(UpdaterBase): """ Provides the updater of the README file. """ def __init__(self, info_manager: InfoManager) -> None: self.destination_instance = FileHelper( os.path.join( info_manager.WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.README_FILENAME, )) super().__init__(info_manager) @property def authorized(self) -> bool: return self.destination_instance.exists() def pre(self) -> "ReadmeUpdater": logging.info("Started to update the content of %r!", self.destination_instance.path) return self def post(self) -> "ReadmeUpdater": logging.info("Finished to update the content of %r!", self.destination_instance.path) return self def start(self) -> "ReadmeUpdater": logging.info( "Started to update the `About PyFunceble` section of %r", self.destination_instance.path, ) with importlib.resources.path("dead_hosts.launcher.data.docs", "about_pyfunceble.md") as file_path: updated_version = RegexHelper( dead_hosts.launcher.defaults.markers.ABOUT_FUNCEBLE_REGEX ).replace_match( self.destination_instance.read(), FileHelper(str(file_path)).read() + "\n\n", ) logging.info( "Finished to update the `About PyFunceble` section of %r", self.destination_instance.path, ) logging.info( "Started to update the `About Dead-Hosts` section of %r", self.destination_instance.path, ) with importlib.resources.path("dead_hosts.launcher.data.docs", "about_dead_hosts.md") as file_path: updated_version = RegexHelper( dead_hosts.launcher.defaults.markers.ABOUT_DEAD_HOSTS_REGEX ).replace_match( self.destination_instance.read(), FileHelper(str(file_path)).read() + "\n\n", ) logging.info( "Finished to update the `About Dead-Hosts` section of %s", self.destination_instance.path, ) self.destination_instance.write(updated_version, overwrite=True) return self