def test_open(self) -> None: """ Tests the method which let us open the given file as we want. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() self.assertEqual(expected, actual) with file_helper.open("w") as file_stream: file_stream.write("Hello, World!") expected = True actual = file_helper.exists() self.assertEqual(expected, actual) expected = "Hello, World!" actual = file_helper.read() self.assertEqual(expected, actual)
def test_delete(self) -> None: """ Tests the method which let us delete a file. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() self.assertEqual(expected, actual) with open(file_helper.path, "w", encoding="utf-8") as file_stream: file_stream.write("") expected = True actual = file_helper.exists() self.assertEqual(expected, actual) file_helper.delete() expected = False actual = file_helper.exists() self.assertEqual(expected, actual)
def get_content(self) -> Optional[dict]: """ Provides the cached or the real contend of the dataset (after caching) :raise FileNotFoundError: When the declared file does not exists. """ if ( bool(self.STORAGE_INDEX) and hasattr(PyFunceble.storage, self.STORAGE_INDEX) and bool(getattr(PyFunceble.storage, self.STORAGE_INDEX)) ): return getattr(PyFunceble.storage, self.STORAGE_INDEX) file_helper = FileHelper(self.source_file) if not file_helper.exists() and bool( self.DOWNLOADER ): # pragma: no cover ## This is just a safety endpoint. self.DOWNLOADER.start() if not file_helper.exists(): raise FileNotFoundError(file_helper.path) content = DictHelper().from_json_file( self.source_file, return_dict_on_error=False ) setattr(PyFunceble.storage, self.STORAGE_INDEX, content) return content
def test_get_size(self) -> None: """ Tests the method which let us get the size of a file. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() self.assertEqual(expected, actual) with open(file_helper.path, "w", encoding="utf-8") as file_stream: file_stream.write("Hello, World!") expected = True actual = file_helper.exists() self.assertEqual(expected, actual) expected = 13 actual = file_helper.get_size() self.assertEqual(expected, actual) os.remove(file_helper.path)
def update_volatile_list(self) -> "UHBPyFuncebleSystemLauncher": """ Updates the content of the :code:`volatile.list` file. """ input_file = FileHelper(outputs.TEMP_VOLATIVE_DESTINATION) volatile_file = FileHelper(outputs.VOLATILE_DESTINATION) clean_file = FileHelper(outputs.CLEAN_DESTINATION) logging.info("Started generation of %r.", volatile_file.path) with volatile_file.open("w", encoding="utf-8") as volatile_file_stream: if clean_file.exists(): with clean_file.open("r", encoding="utf-8") as clean_file_stream: for line in clean_file_stream: line = line.strip() if not line or line.startswith("#") or "." not in line: continue if line.endswith("."): line = line[:-1] volatile_file_stream.write(line + "\n") if input_file.exists(): with input_file.open("r", encoding="utf-8") as input_file_stream: for line in input_file_stream: line = line.strip() if not line or line.startswith("#") or "." not in line: continue if line.endswith("."): line = line[:-1] volatile_file_stream.write(line + "\n") volatile_file.write("\n") whitelist_core_tool( output_file=volatile_file.path, use_official=True, processes=os.cpu_count(), ).filter(file=volatile_file.path, already_formatted=True, standard_sort=False) logging.info("Finished generation of %r.", volatile_file.path) return self
def start(self) -> "OurInfrastructureUpdater": file_helper = FileHelper() dir_helper = DirectoryHelper() for file in self.FILES_TO_REMOVE: file_helper.set_path( os.path.join(self.info_manager.WORKSPACE_DIR, file)) if file_helper.exists(): logging.info("Starting deletion of %r", file_helper.path) file_helper.delete() logging.info("Finished deletion of %r", file_helper.path) for directory in self.DIRS_TO_REMOVE: dir_helper.set_path( os.path.join(self.info_manager.WORKSPACE_DIR, directory)) if dir_helper.exists(): logging.info("Starting deletion of %r", dir_helper.path) dir_helper.delete() logging.info("Finished deletion of %r", dir_helper.path) return self
def cleanup_if_necessary(parent_dirname: str) -> None: """ Process the cleanup if it's necessary. """ cleanup_tool = FilesystemCleanup(parent_dirname) running_file_helper = FileHelper( os.path.join( cleanup_tool.get_output_basedir(), PyFunceble.cli.storage.TEST_RUNNING_FILE, )) if not running_file_helper.exists(): self.sessions_id[parent_dirname] = secrets.token_hex(12) cleanup_tool.clean_output_files() running_file_helper.write( f"{self.sessions_id[parent_dirname]} " f"{datetime.datetime.utcnow().isoformat()}", overwrite=True, ) else: possible_session_id = running_file_helper.read().split()[0] try: _ = datetime.datetime.fromisoformat(possible_session_id) self.sessions_id[parent_dirname] = None except (ValueError, TypeError): self.sessions_id[parent_dirname] = possible_session_id
def update_ip_list() -> "UHBPyFuncebleSystemLauncher": """ Updates the content of the :code:`ip.list` file. """ input_file = FileHelper(outputs.IP_SUBJECTS_DESTINATION) ip_file = FileHelper(outputs.IP_DESTINATION) if input_file.exists(): logging.info("Started generation of %r.", ip_file.path) with input_file.open( "r", encoding="utf-8") as input_file_stream, ip_file.open( "w", encoding="utf-8") as ip_file_stream: for line in input_file_stream: if not line.strip() or line.startswith("#"): continue ip_file_stream.write("\n".join(line.split()[1:]) + "\n") ip_file_stream.write("\n") whitelist_core_tool( output_file=ip_file.path, use_official=True, processes=os.cpu_count(), ).filter(file=ip_file.path, already_formatted=True, standard_sort=False) logging.info("Finished generation of %r.", ip_file.path)
def update_clean_list(self) -> "UHBPyFuncebleSystemLauncher": """ Updates the content of the :code:`clean.list` file. """ input_file = FileHelper(outputs.ACTIVE_SUBJECTS_DESTINATION) clean_file = FileHelper(outputs.CLEAN_DESTINATION) if input_file.exists(): logging.info("Started generation of %r.", clean_file.path) with input_file.open( "r", encoding="utf-8") as input_file_stream, clean_file.open( "w", encoding="utf-8") as clean_file_stream: for line in input_file_stream: line = line.strip() if not line or line.startswith("#") or "." not in line: continue if line.endswith("."): line = line[:-1] clean_file_stream.write("\n" + line) logging.info("Finished generation of %r.", clean_file.path) return self
def run_end(self): """ Run the end logic. """ self.info_manager["currently_under_test"] = False self.info_manager["latest_part_finish_datetime"] = datetime.utcnow() self.info_manager["latest_part_finish_timestamp"] = self.info_manager[ "latest_part_finish_datetime"].timestamp() self.info_manager["finish_datetime"] = self.info_manager[ "latest_part_finish_datetime"] self.info_manager["finish_timestamp"] = self.info_manager[ "finish_datetime"].timestamp() logging.info( "Updated all timestamps and indexes that needed to be updated.") pyfunceble_active_list = FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, "output", dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME, "domains", "ACTIVE", "list", )) clean_list = [ "# File generated by the Dead-Hosts project with the help of PyFunceble.", "# Dead-Hosts: https://github.com/dead-hosts", "# PyFunceble: https://pyfunceble.github.io", f"# Generation Time: {datetime.utcnow().isoformat()}", ] logging.info( f"PyFunceble ACTIVE list output: {pyfunceble_active_list.path}") if pyfunceble_active_list.exists(): logging.info( f"{pyfunceble_active_list.path} exists, getting and formatting its content." ) self.output_file.write("\n".join(clean_list) + "\n\n", overwrite=True) with pyfunceble_active_list.open("r", encoding="utf-8") as file_stream: for line in file_stream: if line.startswith("#"): continue self.output_file.write(line) self.output_file.write("\n") logging.info("Updated of the content of %r", self.output_file.path)
def start(self) -> "InfrastructureCleaner": analytic_directory = DirectoryHelper( os.path.join(outputs.OUTPUT_ROOT_DIRECTORY, "json")) if analytic_directory.exists(): for element in os.listdir(outputs.OUTPUT_ROOT_DIRECTORY): if any(x in element for x in self.STD_FILES_TO_IGNORE): continue dir_helper = DirectoryHelper( os.path.join(outputs.OUTPUT_ROOT_DIRECTORY, element)) if dir_helper.exists(): dir_helper.delete() logging.info("Deleted: %r", dir_helper.path) del analytic_directory for file in self.FILES_TO_REMOVE: if not isinstance(file, list): file_helper = FileHelper( os.path.join(outputs.CURRENT_DIRECTORY, file)) else: file_helper = FileHelper( os.path.join(outputs.CURRENT_DIRECTORY, *file)) if file_helper.exists(): file_helper.delete() logging.info("Deleted: %r", file_helper.path) for file in self.FILES_TO_MOVE_TO_PYFUNCEBLE_CONFIG: file_helper = FileHelper( os.path.join(outputs.CURRENT_DIRECTORY, file)) if file_helper.exists(): file_helper.move( os.path.join(outputs.PYFUNCEBLE_CONFIG_DIRECTORY, file)) logging.info("Moved: %r", file_helper.path) return self
def get_content(self) -> open: """ Provides a file handler which does let you read the content line by line. :raise FileNotFoundError: When the declared file does not exists. """ file_helper = FileHelper(self.source_file) if not file_helper.exists() and bool( self.DOWNLOADER): # pragma: no cover ## pragma reason: Safety. self.DOWNLOADER.start() if not file_helper.exists(): raise FileNotFoundError(file_helper.path) return file_helper.open("r", encoding="utf-8")
def fetch_dataset(self) -> "FilesystemCounter": """ Fetches the source file into the current instance. """ file_helper = FileHelper(self.source_file) if file_helper.exists(): self.dataset = DictHelper().from_json_file(file_helper.path) else: self.dataset = copy.deepcopy(self.STD_DATASET) return self
def test_move(self) -> None: """ Tests of the method which let us move a file to another location. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) destination_file_helper = FileHelper(tempfile.gettempdir()) destination_file_helper.set_path( destination_file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() actual_destination = destination_file_helper.exists() self.assertEqual(expected, actual) self.assertEqual(expected, actual_destination) file_helper.write("Hello, World!") expected = True actual = file_helper.exists() self.assertEqual(expected, actual) expected = False actual_destination = destination_file_helper.exists() self.assertEqual(expected, actual_destination) file_helper.move(destination_file_helper.path) expected = True actual_destination = destination_file_helper.exists() self.assertEqual(expected, actual_destination) expected = "Hello, World!" actual = destination_file_helper.read() self.assertEqual(expected, actual) expected = False actual = file_helper.exists() self.assertEqual(expected, actual) expected = True actual_destination = destination_file_helper.exists() self.assertEqual(expected, actual_destination)
def start(self) -> "PyFuncebleConfigLocationUpdater": for file in self.FILES_TO_MOVE: source_file = FileHelper( os.path.join(self.info_manager.WORKSPACE_DIR, file)) destination_file = FileHelper( os.path.join(self.info_manager.PYFUNCEBLE_CONFIG_DIR, file)) if source_file.exists(): logging.info( "Starting to move %r into %r.", source_file.path, destination_file.path, ) destination_file.delete() source_file.move(destination_file.path) logging.info( "Finished to move %r into %r", source_file.path, destination_file.path, ) else: logging.info( "Did not moved move %r into %r: It does not exists.", source_file.path, destination_file.path, ) if (all( FileHelper( os.path.join(self.info_manager.PYFUNCEBLE_CONFIG_DIR, x)).exists() for x in self.INACTIVE_FILES_TO_DELETE) or FileHelper( os.path.join( self.info_manager.PYFUNCEBLE_CONFIG_DIR, self.INACTIVE_FILES_TO_DELETE[0], )).exists()): logging.info("Starting to delete inactive files.", ) for inactive_file in self.INACTIVE_FILES_TO_DELETE: FileHelper( os.path.join(self.info_manager.PYFUNCEBLE_CONFIG_DIR, inactive_file)).delete() logging.info("Finished to delete inactive files.", ) return self
def test_copy(self) -> None: """ Tests the method which let us copy a file to another place. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) copy_file_helper = FileHelper(tempfile.gettempdir()) copy_file_helper.set_path( copy_file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() actual_copy = copy_file_helper.exists() self.assertEqual(expected, actual) self.assertEqual(expected, actual_copy) file_helper.write("Hello, World!") expected = True actual = file_helper.exists() self.assertEqual(expected, actual) expected = False actual_copy = copy_file_helper.exists() self.assertEqual(expected, actual_copy) file_helper.copy(copy_file_helper.path) expected = True actual_copy = copy_file_helper.exists() self.assertEqual(expected, actual_copy) expected = "Hello, World!" actual = copy_file_helper.read() self.assertEqual(expected, actual) expected = True actual = file_helper.exists() actual_copy = copy_file_helper.exists() self.assertEqual(expected, actual) self.assertEqual(expected, actual_copy)
def get_csv_writer(self) -> Tuple[csv.DictWriter, open]: """ Provides the standard and initiated CSV Dict writer along with the file that was open with it. """ file_helper = FileHelper(self.source_file) add_header = not file_helper.exists() file_handler = file_helper.open("a+", newline="") writer = csv.DictWriter(file_handler, fieldnames=self.FIELDS) if add_header: writer.writeheader() return writer, file_handler
def produce_diff(self) -> None: """ Produce the difference from teh downloaded file. """ file_helper = FileHelper(self.final_destination) new = set() kept = set() removed = set() if file_helper.exists(): with file_helper.open("r", encoding="utf-8") as file_stream: current_content = set(x.strip() for x in file_stream) else: current_content = set() downloaded_empty = True for line in self.download_temp_file: if downloaded_empty: downloaded_empty = False line = line.strip() if not line: continue kept_kept, new_new = self.__get_diff_data( current_content, get_subjects_from_line(line, "availability")) new.update(new_new) kept.update(kept_kept) if downloaded_empty: kept = current_content else: compare_base = kept.copy() compare_base.update(new) removed = current_content - compare_base self.download_temp_file.seek(0) return kept, removed, new
def migrate(self) -> "MigratorBase": """ Provides the migrator (itself). """ file_helper = FileHelper(self.source_file) if file_helper.exists(): with file_helper.open("r", encoding="utf-8") as file_stream: first_line = next(file_stream) if any(x in first_line for x in self.TO_DELETE): temp_destination = tempfile.NamedTemporaryFile( "a+", newline="", encoding="utf-8", delete=False ) file_handler = file_helper.open(newline="") reader = csv.DictReader(file_handler) writer = csv.DictWriter( temp_destination, fieldnames=[x for x in self.FIELDS if x not in self.TO_DELETE], ) writer.writeheader() keys_found = False for row in reader: row = dict(row) for key in self.TO_DELETE: if key in row: del row[key] keys_found = True if not keys_found: break writer.writerow(row) if self.print_action_to_stdout: print_single_line() temp_destination.seek(0) FileHelper(temp_destination.name).move(self.source_file) self.done = True
def test_read_bytes_file_does_not_exists(self) -> None: """ Tests the method which let us read (bytes) a file for the case that the given file does not exists. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() self.assertEqual(expected, actual) expected = None actual = file_helper.read_bytes() self.assertEqual(expected, actual)
def csv_file_delete_source_column_target( continuous_integration: ContinuousIntegrationBase, ) -> None: """ Provides the target for the deletion of the source column. """ migrator = InactiveDatasetDeleteSourceColumnMigrator( print_action_to_stdout=True ) migrator.continuous_integration = continuous_integration file_helper = FileHelper(migrator.source_file) if file_helper.exists(): with file_helper.open("r", encoding="utf-8") as file_stream: first_line = next(file_stream) if any(x in first_line for x in migrator.TO_DELETE): print( f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}" "Started deletion of the 'source' column into " f"{migrator.source_file!r}." ) migrator.start() if migrator.done: print( f"{colorama.Fore.GREEN}{colorama.Style.BRIGHT}" "Finished deletion of the 'source' column into " f"{migrator.source_file!r}." ) else: print( f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}" "unfinished deletion of the 'source' column into " f"{migrator.source_file!r}." ) else: PyFunceble.facility.Logger.info( "Stopped csv_file_delete_source_column_target. File does not exist." )
def update_whitelisted_list(self) -> "UHBPyFuncebleSystemLauncher": """ Updates the content of the :code:`whitelist.list` file. """ input_file = FileHelper(outputs.CLEAN_DESTINATION) whitelist_file = FileHelper(outputs.WHITELISTED_DESTINATION) if input_file.exists(): logging.info("Started generation of %r.", whitelist_file.path) whitelist_core_tool( output_file=whitelist_file.path, use_official=True, processes=os.cpu_count(), ).filter(file=input_file.path, already_formatted=True, standard_sort=False) logging.info("Finished generation of %r.", whitelist_file.path) return self
def get_content(self) -> Generator[Optional[dict], None, None]: """ Provides a generator which provides the next line to read. """ file_helper = FileHelper(self.source_file) if file_helper.exists(): file_handler = file_helper.open(newline="") reader = csv.DictReader(file_handler) for row in reader: if "tested_at" in row: try: row["tested_at"] = datetime.fromisoformat( row["tested_at"]) except (TypeError, ValueError): row["tested_at"] = datetime.utcnow() - timedelta( days=365) yield row file_handler.close()
def migrate(self) -> "InactiveJSON2CSVMigrator": """ Starts the migration. """ file_helper = FileHelper(self.source_file) if file_helper.exists(): self.dataset.set_authorized(True) dataset = { "idna_subject": None, "status": None, "status_source": None, "checker_type": "AVAILABILITY", "destination": None, "source": None, "tested_at": None, "session_id": None, } delete_file = True with file_helper.open("r", encoding="utf-8") as file_stream: for line in file_stream: if (self.continuous_integration and self.continuous_integration.is_time_exceeded()): delete_file = False break line = (line.strip().replace('"', "").replace(",", "").replace( "{", "", ).replace("}", "")) if ":" not in line: continue index, value = [x.strip() for x in line.rsplit(":", 1)] if not value: if index.isdigit(): dataset[ "tested_at"] = datetime.datetime.fromtimestamp( float(index)).isoformat() else: dataset["source"] = os.path.abspath(index) dataset[ "destination"] = get_destination_from_origin( dataset["source"]) continue dataset["idna_subject"] = domain2idna.domain2idna(index) dataset["status"] = value if not dataset["tested_at"]: dataset["tested_at"] = datetime.datetime.utcnow( ).isoformat() PyFunceble.facility.Logger.debug("Decoded dataset:\n%r.", dataset) self.dataset.update(dataset) if self.print_action_to_stdout: print_single_line() PyFunceble.facility.Logger.info("Added %r into %r", dataset["idna_subject"], self.dataset) if delete_file: file_helper.delete() self.done = True return self
class InfoManager: """ Provides an interface for the management of the :code:`info.json` file. .. warning:: Keep in mind that this interface provides everything that may be needed by other interfaces. """ WORKSPACE_DIR: str = dead_hosts.launcher.defaults.envs.WORKSPACE_DIR PYFUNCEBLE_CONFIG_DIR: str = ( dead_hosts.launcher.defaults.paths.PYFUNCEBLE_CONFIG_DIRECTORY ) GHA_WORKFLOWS_DIR: str = os.path.join( WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.GHA_WORKFLOW_DIR ) INFO_FILE = os.path.join( WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.INFO_FILENAME ) def __init__(self) -> None: self.info_file_instance = FileHelper(self.INFO_FILE) if self.info_file_instance.exists(): self.content = DictHelper().from_json_file(self.info_file_instance.path) else: self.content = {} logging.debug("Administration file path: %r", self.INFO_FILE) logging.debug( "Administration file exists: %r", self.info_file_instance.exists() ) logging.debug("Administration file content:\n%r", self.content) self.update() self.create_missing_index() self.clean() self.store() def __getattr__(self, index: str) -> Any: if index in self.content: return self.content[index] raise AttributeError(index) def __getitem__(self, index: str) -> Any: if index in self.content: return self.content[index] raise AttributeError(index) def __setitem__(self, index: str, value: Any): self.content[index] = value def __del__(self): self.store() def store(self) -> "InfoManager": """ Stores the current state. """ local_copy = {} for index, value in self.content.items(): if index.endswith("_timestamp") and isinstance(value, datetime): local_copy[index] = value.timestamp() elif index.endswith("_datetime") and isinstance(value, datetime): local_copy[index] = value.isoformat() else: local_copy[index] = copy.deepcopy(value) DictHelper(local_copy).to_json_file(self.info_file_instance.path) return self def clean(self) -> "InfoManager": """ Cleans the unneeded indexes. """ for index in [ "arguments", "clean_list_file", "clean_original", "commit_autosave_message", "last_test", "list_name", "stable", ]: if index in self.content: del self.content[index] logging.debug( "Deleted the %r index of the administration file, " "it is not needed anymore.", index, ) return self def create_missing_index(self) -> "InfoManager": """ Creates the missing indexes. """ default_datetime = datetime.utcnow() - timedelta(days=15) indexes = { "currently_under_test": False, "custom_pyfunceble_config": {}, "days_until_next_test": 2, "finish_datetime": default_datetime, "finish_timestamp": default_datetime.timestamp(), "last_download_datetime": default_datetime, "last_download_timestamp": default_datetime.timestamp(), "latest_part_finish_timestamp": default_datetime.timestamp(), "latest_part_start_timestamp": default_datetime.timestamp(), "latest_part_finish_datetime": default_datetime, "latest_part_start_datetime": default_datetime, "name": dead_hosts.launcher.defaults.paths.GIT_BASE_NAME, "own_management": False, "ping": [], "raw_link": None, "start_datetime": default_datetime, "start_timestamp": default_datetime.timestamp(), "live_update": True, } for index, value in indexes.items(): if index not in self.content: self.content[index] = value logging.debug( "Created the %r index of the administration file, it was not found.", index, ) def update(self) -> "InfoManager": """ Updates and filters the new content. """ # pylint: disable=too-many-branches self.content["name"] = dead_hosts.launcher.defaults.paths.GIT_BASE_NAME logging.debug("Updated the `name` index of the administration file.") to_delete = [ FileHelper(os.path.join(self.WORKSPACE_DIR, ".administrators")), FileHelper(os.path.join(self.WORKSPACE_DIR, "update_me.py")), FileHelper(os.path.join(self.WORKSPACE_DIR, "admin.py")), ] if "list_name" in self.content: to_delete.append( FileHelper(os.path.join(self.WORKSPACE_DIR, self.content["list_name"])) ) if "ping" in self.content: local_ping_result = [] for username in self.content["ping"]: if username.startswith("@"): local_ping_result.append(username) else: local_ping_result.append(f"@{username}") self.content["ping"] = local_ping_result logging.debug( "Updated the `ping` index of the administration file, " "the format has to stay the same everywhere." ) if ( "raw_link" in self.content and isinstance(self.content["raw_link"], str) and not self.content["raw_link"] ): self.content["raw_link"] = None logging.debug( "Updated the `raw_link` index of the administration file, " "empty string not accepted." ) if "custom_pyfunceble_config" in self.content: if self.content["custom_pyfunceble_config"]: if not isinstance(self.content["custom_pyfunceble_config"], dict): self.content["custom_pyfunceble_config"] = {} else: self.content["custom_pyfunceble_config"] = DictHelper( self.content["custom_pyfunceble_config"] ).flatten() else: self.content["custom_pyfunceble_config"] = {} logging.debug( "Updated the `custom_pyfunceble_config` index of the " "administration file, it should be a %r.", dict, ) if ( "custom_pyfunceble_config" in self.content and self.content["custom_pyfunceble_config"] and not isinstance(self.content["custom_pyfunceble_config"], dict) ): self.content["custom_pyfunceble_config"] = {} logging.debug( "Updated the `custom_pyfunceble_config` index of the " "administration file, it should be a %r.", dict, ) for index in ["currently_under_test"]: if index in self.content and not isinstance(self.content[index], bool): self.content[index] = bool(int(self.content[index])) logging.debug( "Updated the %r index of the administration file, " "it should be a %r.", index, bool, ) for index in [ "days_until_next_test", "finish_timestamp", "last_download_datetime" "last_download_timestamp" "lastest_part_finish_timestamp", "lastest_part_start_timestamp", "start_timestamp", ]: if index in self.content and not isinstance(self.content[index], float): self.content[index] = float(self.content[index]) logging.debug( "Updated the %r index of the administration file, " "it should be a %r.", index, float, ) for index in [ "finish_timestamp", "last_download_timestamp", "lastest_part_finish_timestamp", "lastest_part_start_timestamp", "start_timestamp", ]: if index in self.content and not isinstance(self.content[index], datetime): self.content[index] = datetime.fromtimestamp(self.content[index]) logging.debug( "Updated the %r index of the administration file, " "the system understands %r only." " (JSON => %s).", index, datetime, dict, ) for index in [ "finish_datetime", "last_download_datetime", "lastest_part_finish_datetime", "lastest_part_start_datetime", "start_datetime", ]: if index in self.content: if self.content[index] and not isinstance( self.content[index], datetime ): self.content[index] = datetime.fromisoformat(self.content[index]) logging.debug( "Updated the %r index of the administration file, " "the system understands %r only." " (JSON => %r.", index, datetime, dict, ) else: self.content[index] = datetime.fromtimestamp(0) logging.debug( "Set the %r index of the administration file, " "it was not previously set.", repr(index), ) for file in to_delete: if file.exists(): file.delete() logging.debug( "Deleted the %r file, it is not needed anymore.", file.path, ) def get_ping_for_commit(self) -> str: """ Provides the string to append in order to mention the users to ping. """ if "ping" in self.content: return " ".join(self.content["ping"]) return ""
class ReadmeUpdater(UpdaterBase): """ Provides the updater of the README file. """ def __init__(self, info_manager: InfoManager) -> None: self.destination_instance = FileHelper( os.path.join( info_manager.WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.README_FILENAME, )) super().__init__(info_manager) @property def authorized(self) -> bool: return self.destination_instance.exists() def pre(self) -> "ReadmeUpdater": logging.info("Started to update the content of %r!", self.destination_instance.path) return self def post(self) -> "ReadmeUpdater": logging.info("Finished to update the content of %r!", self.destination_instance.path) return self def start(self) -> "ReadmeUpdater": logging.info( "Started to update the `About PyFunceble` section of %r", self.destination_instance.path, ) with importlib.resources.path("dead_hosts.launcher.data.docs", "about_pyfunceble.md") as file_path: updated_version = RegexHelper( dead_hosts.launcher.defaults.markers.ABOUT_FUNCEBLE_REGEX ).replace_match( self.destination_instance.read(), FileHelper(str(file_path)).read() + "\n\n", ) logging.info( "Finished to update the `About PyFunceble` section of %r", self.destination_instance.path, ) logging.info( "Started to update the `About Dead-Hosts` section of %r", self.destination_instance.path, ) with importlib.resources.path("dead_hosts.launcher.data.docs", "about_dead_hosts.md") as file_path: updated_version = RegexHelper( dead_hosts.launcher.defaults.markers.ABOUT_DEAD_HOSTS_REGEX ).replace_match( self.destination_instance.read(), FileHelper(str(file_path)).read() + "\n\n", ) logging.info( "Finished to update the `About Dead-Hosts` section of %s", self.destination_instance.path, ) self.destination_instance.write(updated_version, overwrite=True) return self
class Administration: """ Provides the administration interface. """ info_file_location: Optional[str] = None info_file_helper: Optional[FileHelper] = None __our_info: dict = dict() def __init__(self) -> None: self.info_file_location = outputs.ADMINISTRATION_DESTINATION self.info_file_helper = FileHelper(self.info_file_location) self.__our_info.update(self.load()) def __contains__(self, index: str) -> bool: return index in self.__our_info def __getitem__(self, index: str) -> Any: if index in self.__our_info: return self.__our_info[index] raise KeyError(index) def __getattr__(self, index: str) -> Any: if index in self.__our_info: return self.__our_info[index] raise AttributeError(index) def __setattr__(self, index: str, value: Any) -> None: if self.__our_info: self.__our_info[index] = value else: super().__setattr__(index, value) def __delattr__(self, index: str) -> None: if index in self.info: del self.info[index] def __del__(self) -> None: self.save() @property def exists(self) -> bool: """ Checks if the administration file exists. """ return self.info_file_helper.exists() @staticmethod def convert_data_for_system(data: dict) -> dict: """ Given the content of the info file, we convert it into something our system may understand. .. warning:: This method also delete the keys that are scheduled or declared for deletion :param data: The data to work with. """ result = dict() for key, value in dict(data).items(): if key in infrastructure.ADMINISTRATION_INDEXES["delete"]: continue if key == "name": result[key] = ( CommandHelper("basename $(git rev-parse --show-toplevel)" ).execute().strip()) elif key in infrastructure.ADMINISTRATION_INDEXES["bool"]: result[key] = bool(int(value)) elif key in infrastructure.ADMINISTRATION_INDEXES["int"]: result[key] = int(value) elif key in infrastructure.ADMINISTRATION_INDEXES["dict"]: result[key] = dict(value) elif key in infrastructure.ADMINISTRATION_INDEXES["datetime"]: try: result[key] = datetime.fromisoformat(value) except ValueError: result[key] = datetime.utcnow() elif key in infrastructure.ADMINISTRATION_INDEXES["epoch"]: result[key] = datetime.fromtimestamp(value) else: result[key] = value if "pyfunceble" not in result: result["pyfunceble"] = { "config": {}, } for sanitize_type, keys in infrastructure.ADMINISTRATION_INDEXES.items( ): if sanitize_type == "delete": continue for key in keys: if key not in result: if sanitize_type == "bool": local_result = bool(None) elif sanitize_type == "int": local_result = 0 elif sanitize_type == "dict": local_result = dict() elif sanitize_type == "datetime": local_result = datetime.utcnow() - timedelta( days=365.25) elif sanitize_type == "epoch": local_result = datetime.utcnow() - timedelta( days=365.25) else: local_result = None result[key] = local_result return result @staticmethod def convert_data_for_file(data: dict) -> dict: """ Given the content of the info file, we convert it into something that can be saved into a JSON file. .. warning:: This method also delete the keys that are scheduled or declared for deletion :param data: The data to work with. """ result = dict() for key, value in dict(data).items(): if key in infrastructure.ADMINISTRATION_INDEXES["delete"]: continue if key in infrastructure.ADMINISTRATION_INDEXES["bool"]: result[key] = bool(value) elif key in infrastructure.ADMINISTRATION_INDEXES["int"]: result[key] = int(value) elif key in infrastructure.ADMINISTRATION_INDEXES["datetime"]: result[key] = value.isoformat() elif key in infrastructure.ADMINISTRATION_INDEXES["epoch"]: result[key] = value.timestamp() else: result[key] = value return result def load(self) -> dict: """ Loads and return the content of the administration file. """ if self.info_file_helper.read(): content = self.info_file_helper.read() logging.debug("Administration file content:\n%s", content) return self.convert_data_for_system(DictHelper().from_json( content, return_dict_on_error=False)) return dict() def save(self) -> None: """ Saves the loaded content of the administration file. """ if self.info_file_helper.exists(): DictHelper(self.convert_data_for_file( self.__our_info)).to_json_file(self.info_file_location)
class Orchestration: """ Orchester the test launch. """ info_manager: Optional[InfoManager] = None authorization_handler: Optional[Authorization] = None origin_file: Optional[FileHelper] = None output_file: Optional[FileHelper] = None def __init__(self, save: bool = False, end: bool = False) -> None: self.info_manager = InfoManager() git_name = EnvironmentVariableHelper("GIT_NAME") git_email = EnvironmentVariableHelper("GIT_EMAIL") if git_email.exists() and "funilrys" in git_email.get_value(): git_name.set_value(dead_hosts.launcher.defaults.envs.GIT_NAME) git_email.set_value(dead_hosts.launcher.defaults.envs.GIT_EMAIL) EnvironmentVariableHelper("PYFUNCEBLE_OUTPUT_LOCATION").set_value( self.info_manager.WORKSPACE_DIR) EnvironmentVariableHelper("PYFUNCEBLE_CONFIG_DIR").set_value( self.info_manager.PYFUNCEBLE_CONFIG_DIR) self.authorization_handler = Authorization(self.info_manager) self.origin_file = FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME, )) self.output_file = FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.OUTPUT_FILENAME, )) logging.info("Origin file: %r", self.origin_file.path) logging.info("Output file: %r", self.output_file.path) if not end and not save: logging.info("Checking authorization to run.") if self.authorization_handler.is_test_authorized(): execute_all_updater(self.info_manager) PyFunceble.facility.ConfigLoader.start() self.fetch_file_to_test() self.run_test() else: logging.info( "Not authorized to run a test until %r (current time) > %r", datetime.now(), self.authorization_handler.next_authorization_time, ) sys.exit(0) elif save: self.run_autosave() else: self.run_end() def fetch_file_to_test(self) -> "Orchestration": """ Provides the latest version of the file to test. """ if self.authorization_handler.is_refresh_authorized(): logging.info( "We are authorized to refresh the lists! Let's do that.") logging.info("Raw Link: %r", self.info_manager.raw_link) if self.info_manager.raw_link: DownloadHelper(self.info_manager.raw_link).download_text( destination=self.origin_file.path) logging.info( "Could get the new version of the list. Updating the download time." ) self.info_manager["last_download_datetime"] = datetime.utcnow() self.info_manager[ "last_download_timestamp"] = self.info_manager[ "last_download_datetime"].timestamp() elif self.origin_file.exists(): logging.info( "Raw link not given or is empty. Let's work with %r.", self.origin_file.path, ) self.origin_file.read() logging.info("Emptying the download time.") self.info_manager[ "last_download_datetime"] = datetime.fromtimestamp(0) self.info_manager[ "last_download_timestamp"] = self.info_manager[ "last_download_datetime"].timestamp() else: logging.info(f"Could not find {self.origin_file.path}. " "Generating empty content to test.") self.origin_file.write("# No content yet.", overwrite=True) logging.info("Emptying the download time.") self.info_manager[ "last_download_datetime"] = datetime.fromtimestamp(0) self.info_manager[ "last_download_timestamp"] = self.info_manager[ "last_download_datetime"].timestamp() logging.info("Updated %r.", self.origin_file.path) return self def run_test(self): """ Run a test of the input list. """ if not self.info_manager.currently_under_test: self.info_manager["currently_under_test"] = True self.info_manager["start_datetime"] = datetime.utcnow() self.info_manager["start_timestamp"] = self.info_manager[ "start_datetime"].timestamp() self.info_manager["finish_datetime"] = datetime.fromtimestamp(0) self.info_manager["finish_timestamp"] = self.info_manager[ "finish_datetime"].timestamp() self.info_manager["latest_part_start_datetime"] = datetime.utcnow() self.info_manager["latest_part_start_timestamp"] = self.info_manager[ "latest_part_start_datetime"].timestamp() self.info_manager[ "latest_part_finish_datetime"] = datetime.fromtimestamp(0) self.info_manager["latest_part_finish_timestamp"] = self.info_manager[ "latest_part_finish_datetime"].timestamp() logging.info("Updated all timestamps.") logging.info("Starting PyFunceble %r ...", PyFunceble.__version__) Command(f"PyFunceble -f {self.origin_file.path}").run_to_stdout() if not dead_hosts.launcher.defaults.envs.GITHUB_TOKEN: self.run_end() def run_autosave(self): """ Run the autosave logic of the administration file. .. warning:: This is just about the administration file not PyFunceble. """ self.info_manager["latest_part_finish_datetime"] = datetime.utcnow() self.info_manager["latest_part_finish_timestamp"] = self.info_manager[ "latest_part_finish_datetime"].timestamp() logging.info("Updated all timestamps.") def run_end(self): """ Run the end logic. """ self.info_manager["currently_under_test"] = False self.info_manager["latest_part_finish_datetime"] = datetime.utcnow() self.info_manager["latest_part_finish_timestamp"] = self.info_manager[ "latest_part_finish_datetime"].timestamp() self.info_manager["finish_datetime"] = self.info_manager[ "latest_part_finish_datetime"] self.info_manager["finish_timestamp"] = self.info_manager[ "finish_datetime"].timestamp() logging.info( "Updated all timestamps and indexes that needed to be updated.") pyfunceble_active_list = FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, "output", dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME, "domains", "ACTIVE", "list", )) clean_list = [ "# File generated by the Dead-Hosts project with the help of PyFunceble.", "# Dead-Hosts: https://github.com/dead-hosts", "# PyFunceble: https://pyfunceble.github.io", f"# Generation Time: {datetime.utcnow().isoformat()}", ] logging.info( f"PyFunceble ACTIVE list output: {pyfunceble_active_list.path}") if pyfunceble_active_list.exists(): logging.info( f"{pyfunceble_active_list.path} exists, getting and formatting its content." ) self.output_file.write("\n".join(clean_list) + "\n\n", overwrite=True) with pyfunceble_active_list.open("r", encoding="utf-8") as file_stream: for line in file_stream: if line.startswith("#"): continue self.output_file.write(line) self.output_file.write("\n") logging.info("Updated of the content of %r", self.output_file.path)
def migrate(self) -> "WhoisJSON2CSVMigrator": """ Provides the migration logic. """ file_helper = FileHelper(self.source_file) if file_helper.exists(): self.dataset.set_authorized(True) dataset = { "subject": None, "idna_subject": None, "expiration_date": None, "epoch": None, } delete_file = True with file_helper.open("r", encoding="utf-8") as file_stream: for line in file_stream: if (self.continuous_integration and self.continuous_integration.is_time_exceeded()): delete_file = False break line = (line.strip().replace('"', "").replace(",", "").replace( "{", "", ).replace("}", "")) if ":" not in line: continue index, value = [x.strip() for x in line.split(":")] if not value: dataset["subject"], dataset["idna_subject"] = ( index, domain2idna.domain2idna(index), ) continue if index == "epoch": dataset["epoch"] = float(value) elif index == "expiration_date": dataset["expiration_date"] = value elif index == "state": PyFunceble.facility.Logger.debug( "Decoded dataset:\n%r.", dataset) self.dataset.update(dataset) if self.print_action_to_stdout: print_single_line() PyFunceble.facility.Logger.info( "Added %r into %r", dataset["idna_subject"], self.dataset) if delete_file: file_helper.delete() self.done = True return self
def restore_from_backup(self) -> "DirectoryStructureRestoration": """ Restores or reconstruct the output directory. """ # pylint: disable=too-many-locals PyFunceble.facility.Logger.info( "Started restoration of the directory structure") backup = self.get_backup_data() base_dir = self.get_output_basedir() dir_helper = DirectoryHelper() file_helper = FileHelper() if dir_helper.set_path(base_dir).exists(): for root, _, files in os.walk(dir_helper.path): reduced_path = self.get_path_without_base_dir(root) if reduced_path not in backup and root != reduced_path: dir_helper.set_path(root).delete() PyFunceble.facility.Logger.debug( "Added %r into the list of directories to delete. " "Reason: not found in own dataset.", root, ) continue for directory, files in backup.items(): dir_helper.set_path(os.path.join(base_dir, directory)).create() for file, dataset in files.items(): file_full_path = os.path.join(dir_helper.path, file) if (file == ".gitignore" and PyFunceble.cli.storage.STD_PARENT_DIRNAME not in file_full_path): to_delete = file_full_path file_helper.set_path(to_delete).delete() PyFunceble.facility.Logger.debug( "(If exists) Deleted: %r. Reason: We are going to " "replace it with .gitkeep", to_delete, ) file_full_path = file_full_path.replace( ".gitignore", ".gitkeep") file_helper.set_path(file_full_path) if not file_helper.exists(): file_helper.write(dataset["content"], overwrite=True) PyFunceble.facility.Logger.info( "Finished restoration of the directory structure") return self