def get_backup_data(self) -> dict: """ Provides the data which acts as a backup. """ result = {} base_dir = self.get_output_basedir() file_helper = FileHelper() hash_helper = HashHelper() for file in DirectoryHelper(base_dir).list_all_files(): file_helper.set_path(file) reduced_path = self.get_path_without_base_dir(file) directory, filename = reduced_path.rsplit(os.sep, 1) file_hash = hash_helper.hash_file(file_helper.path) dataset = { filename: { "content": file_helper.read(), "hash": file_hash } } if directory not in result: result[directory] = dataset else: result[directory].update(dataset) PyFunceble.facility.Logger.debug("Dir Structure to backup:\n%r", result) return result
def remove_removed(self) -> None: """ Removed the removed entries from all files to clean. """ file_helper = FileHelper() for file in self.files_to_clean: if not file_helper.set_path(file).exists(): continue logging.info( "Started to cleanup: %r", file, ) whitelist_core_tool( output_file=file, secondary_whitelist=self.whitelist_list.name, use_official=False, processes=os.cpu_count(), ).filter(file=file, already_formatted=True, standard_sort=True) logging.info( "Finished to cleanup: %r", file, )
def is_cloned( ) -> bool: # pragma: no cover ## Only used by 1 thing for dev. """ Checks if the local version is a cloned (from git) one. """ file_helper = FileHelper() directory_helper = DirectoryHelper() if not directory_helper.set_path(".git").exists(): return False list_of_files = [ ".coveragerc", ".gitignore", "CODE_OF_CONDUCT.rst", "CONTRIBUTING.rst", "MANIFEST.in", "README.rst", "requirements.txt", "setup.py", "version.yaml", ] list_of_dirs = ["docs", "PyFunceble", "tests", ".github"] if not all(file_helper.set_path(x).exists() for x in list_of_files): return False if not all( directory_helper.set_path(x).exists() for x in list_of_dirs): return False return True
def test_delete(self) -> None: """ Tests the method which let us delete a file. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() self.assertEqual(expected, actual) with open(file_helper.path, "w", encoding="utf-8") as file_stream: file_stream.write("") expected = True actual = file_helper.exists() self.assertEqual(expected, actual) file_helper.delete() expected = False actual = file_helper.exists() self.assertEqual(expected, actual)
def get_content(self) -> Optional[dict]: """ Provides the cached or the real contend of the dataset (after caching) :raise FileNotFoundError: When the declared file does not exists. """ if ( bool(self.STORAGE_INDEX) and hasattr(PyFunceble.storage, self.STORAGE_INDEX) and bool(getattr(PyFunceble.storage, self.STORAGE_INDEX)) ): return getattr(PyFunceble.storage, self.STORAGE_INDEX) file_helper = FileHelper(self.source_file) if not file_helper.exists() and bool( self.DOWNLOADER ): # pragma: no cover ## This is just a safety endpoint. self.DOWNLOADER.start() if not file_helper.exists(): raise FileNotFoundError(file_helper.path) content = DictHelper().from_json_file( self.source_file, return_dict_on_error=False ) setattr(PyFunceble.storage, self.STORAGE_INDEX, content) return content
def __init__(self, info_manager: InfoManager) -> None: self.destination_instance = FileHelper( os.path.join( info_manager.WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.README_FILENAME, )) super().__init__(info_manager)
def start(self) -> "UpdaterBase": """ Starts the update process. """ if self.administration.raw_link: logging.info("Started to download: %r", self.administration.raw_link) DownloadHelper(self.administration.raw_link).download_text( destination=self.download_temp_file.name) logging.info("Finished to download: %r", self.administration.raw_link) self.download_temp_file.seek(0) logging.info("Started comparison of: %r", self.final_destination) kept, removed, new = self.produce_diff() logging.info("Finished comparison of: %r", self.final_destination) to_write = kept.copy() to_write.update(new) try: # Safety. to_write.remove(None) except KeyError: pass try: # Safety. to_write.remove("") except KeyError: pass logging.info("Started to update: %r", self.final_destination) FileHelper(self.final_destination).write("\n".join(sorted(to_write)) + "\n", overwrite=True) logging.info("Finished to update: %r", self.final_destination) if removed: logging.info( "Started to write our temporary whitelist list into: %r", self.whitelist_list.name, ) FileHelper(self.whitelist_list.name).write("\n".join(removed) + "\n", overwrite=True) self.whitelist_list.seek(0) logging.info( "Finished to write our temporary whitelist list into: %r", self.whitelist_list.name, ) self.remove_removed()
def run_end(self): """ Run the end logic. """ self.info_manager["currently_under_test"] = False self.info_manager["latest_part_finish_datetime"] = datetime.utcnow() self.info_manager["latest_part_finish_timestamp"] = self.info_manager[ "latest_part_finish_datetime"].timestamp() self.info_manager["finish_datetime"] = self.info_manager[ "latest_part_finish_datetime"] self.info_manager["finish_timestamp"] = self.info_manager[ "finish_datetime"].timestamp() logging.info( "Updated all timestamps and indexes that needed to be updated.") pyfunceble_active_list = FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, "output", dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME, "domains", "ACTIVE", "list", )) clean_list = [ "# File generated by the Dead-Hosts project with the help of PyFunceble.", "# Dead-Hosts: https://github.com/dead-hosts", "# PyFunceble: https://pyfunceble.github.io", f"# Generation Time: {datetime.utcnow().isoformat()}", ] logging.info( f"PyFunceble ACTIVE list output: {pyfunceble_active_list.path}") if pyfunceble_active_list.exists(): logging.info( f"{pyfunceble_active_list.path} exists, getting and formatting its content." ) self.output_file.write("\n".join(clean_list) + "\n\n", overwrite=True) with pyfunceble_active_list.open("r", encoding="utf-8") as file_stream: for line in file_stream: if line.startswith("#"): continue self.output_file.write(line) self.output_file.write("\n") logging.info("Updated of the content of %r", self.output_file.path)
def test_open(self) -> None: """ Tests the method which let us open the given file as we want. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() self.assertEqual(expected, actual) with file_helper.open("w") as file_stream: file_stream.write("Hello, World!") expected = True actual = file_helper.exists() self.assertEqual(expected, actual) expected = "Hello, World!" actual = file_helper.read() self.assertEqual(expected, actual)
def __init__(self, save: bool = False, end: bool = False) -> None: self.info_manager = InfoManager() git_name = EnvironmentVariableHelper("GIT_NAME") git_email = EnvironmentVariableHelper("GIT_EMAIL") if git_email.exists() and "funilrys" in git_email.get_value(): git_name.set_value(dead_hosts.launcher.defaults.envs.GIT_NAME) git_email.set_value(dead_hosts.launcher.defaults.envs.GIT_EMAIL) EnvironmentVariableHelper("PYFUNCEBLE_OUTPUT_LOCATION").set_value( self.info_manager.WORKSPACE_DIR) EnvironmentVariableHelper("PYFUNCEBLE_CONFIG_DIR").set_value( self.info_manager.PYFUNCEBLE_CONFIG_DIR) self.authorization_handler = Authorization(self.info_manager) self.origin_file = FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME, )) self.output_file = FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.OUTPUT_FILENAME, )) logging.info("Origin file: %r", self.origin_file.path) logging.info("Output file: %r", self.output_file.path) if not end and not save: logging.info("Checking authorization to run.") if self.authorization_handler.is_test_authorized(): execute_all_updater(self.info_manager) PyFunceble.facility.ConfigLoader.start() self.fetch_file_to_test() self.run_test() else: logging.info( "Not authorized to run a test until %r (current time) > %r", datetime.now(), self.authorization_handler.next_authorization_time, ) sys.exit(0) elif save: self.run_autosave() else: self.run_end()
def test_set_path_return(self) -> None: """ Tests the response from the method which let us set the path to work with. """ given = tempfile.NamedTemporaryFile() file_helper = FileHelper() actual = file_helper.set_path(given.name) self.assertIsInstance(actual, FileHelper)
def test_set_path_not_str(self) -> None: """ Tests the method which let us set the path to work with for the case that it's not a string. """ given = ["Hello", "World"] file_helper = FileHelper() self.assertRaises(TypeError, lambda: file_helper.set_path(given))
def get_dot_env_file(self) -> str: """ Provides the dotenv file to work with. """ file_helper = FileHelper() for file in self.dotenv_locations: if file_helper.set_path(file).exists(): return file_helper.path return self.dotenv_locations[-1]
def fetch_dataset(self) -> "FilesystemCounter": """ Fetches the source file into the current instance. """ file_helper = FileHelper(self.source_file) if file_helper.exists(): self.dataset = DictHelper().from_json_file(file_helper.path) else: self.dataset = copy.deepcopy(self.STD_DATASET) return self
def test_is_empty(self) -> None: """ Tests the method which let us check if a file is empty. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() self.assertEqual(expected, actual) with open(file_helper.path, "w", encoding="utf-8") as file_stream: file_stream.write("") expected = True actual = file_helper.is_empty() self.assertEqual(expected, actual) with open(file_helper.path, "w", encoding="utf-8") as file_stream: file_stream.write("Hello, World!") expected = False actual = file_helper.is_empty() self.assertEqual(expected, actual) os.remove(file_helper.path)
def test_write(self) -> None: """ Tests the method which let us write a file. """ given = tempfile.NamedTemporaryFile(delete=False) file_helper = FileHelper(given.name) file_helper.write("Hello, World!") given.seek(0) expected = b"Hello, World!" actual = given.read() self.assertEqual(expected, actual) file_helper.write("Hello, this is Funilrys!") given.seek(0) expected = b"Hello, World!Hello, this is Funilrys!" actual = given.read() self.assertEqual(expected, actual) file_helper.write("Hello, World!", overwrite=True) given.seek(0) expected = b"Hello, World!" actual = given.read() self.assertEqual(expected, actual)
def mining_file_cleanup_target( continuous_integration: ContinuousIntegrationBase, ) -> None: """ Provides the target for the cleanup of the mining file. """ migrator = MiningFileCleanupMigrator(print_action_to_stdout=True) migrator.continuous_integration = continuous_integration if FileHelper(migrator.source_file).exists(): print( f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}" f"Started deletion of {migrator.source_file!r}." ) migrator.start() if migrator.done: print( f"{colorama.Fore.GREEN}{colorama.Style.BRIGHT}" f"Finished deletion of {migrator.source_file!r}." ) else: print( f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}" f"Unfinished deletion of {migrator.source_file!r}." ) else: PyFunceble.facility.Logger.info( "Stopped hashes_file_cleanup_target. File does not exist." )
def json2csv_inactive_target( continuous_integration: ContinuousIntegrationBase, ) -> None: """ Provides the target for the inactive database migrator. """ migrator = InactiveJSON2CSVMigrator(print_action_to_stdout=True) migrator.continuous_integration = continuous_integration if FileHelper(migrator.source_file).exists(): print( f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}" "Started migration (json2csv) of the inactive dataset." ) migrator.start() if migrator.done: print( f"\n{colorama.Fore.GREEN}{colorama.Style.BRIGHT}" "Finished migration (json2csv) of the inactive dataset." ) else: print( f"\n{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}" "Unfinished migration (json2csv) of the inactive dataset." ) else: PyFunceble.facility.Logger.info( "Stopped json2csv_inactive_target. File does not exist." )
def test_read_bytes(self) -> None: """ Tests the method which let us read (bytes) a file. """ given = tempfile.NamedTemporaryFile(delete=False) file_helper = FileHelper(given.name) file_helper.write("Hello, World!") given.seek(0) expected = b"Hello, World!" actual = file_helper.read_bytes() self.assertEqual(expected, actual)
def authorized(self) -> bool: """ Provides the authorization to launch. """ return not FileHelper( os.path.join(outputs.CURRENT_DIRECTORY, outputs.EXAMPLE_ADMINISTRATION_FILENAME)).exists()
def authorized(self) -> bool: return any( FileHelper(os.path.join(self.info_manager.WORKSPACE_DIR, x)).exists() for x in self.FILES_TO_REMOVE) or any( DirectoryHelper( os.path.join(self.info_manager.WORKSPACE_DIR, x)).exists() for x in self.DIRS_TO_REMOVE)
def config_file_exist( self, ) -> bool: # pragma: no cover ## Existance checker already tested. """ Checks if the config file exists. """ return FileHelper(self.path_to_config).exists()
def get_csv_writer(self) -> Tuple[csv.DictWriter, open]: """ Provides the standard and initiated CSV Dict writer along with the file that was open with it. """ file_helper = FileHelper(self.source_file) add_header = not file_helper.exists() file_handler = file_helper.open("a+", newline="") writer = csv.DictWriter(file_handler, fieldnames=self.FIELDS) if add_header: writer.writeheader() return writer, file_handler
def update_clean_list(self) -> "UHBPyFuncebleSystemLauncher": """ Updates the content of the :code:`clean.list` file. """ input_file = FileHelper(outputs.ACTIVE_SUBJECTS_DESTINATION) clean_file = FileHelper(outputs.CLEAN_DESTINATION) if input_file.exists(): logging.info("Started generation of %r.", clean_file.path) with input_file.open( "r", encoding="utf-8") as input_file_stream, clean_file.open( "w", encoding="utf-8") as clean_file_stream: for line in input_file_stream: line = line.strip() if not line or line.startswith("#") or "." not in line: continue if line.endswith("."): line = line[:-1] clean_file_stream.write("\n" + line) logging.info("Finished generation of %r.", clean_file.path) return self
def update_ip_list() -> "UHBPyFuncebleSystemLauncher": """ Updates the content of the :code:`ip.list` file. """ input_file = FileHelper(outputs.IP_SUBJECTS_DESTINATION) ip_file = FileHelper(outputs.IP_DESTINATION) if input_file.exists(): logging.info("Started generation of %r.", ip_file.path) with input_file.open( "r", encoding="utf-8") as input_file_stream, ip_file.open( "w", encoding="utf-8") as ip_file_stream: for line in input_file_stream: if not line.strip() or line.startswith("#"): continue ip_file_stream.write("\n".join(line.split()[1:]) + "\n") ip_file_stream.write("\n") whitelist_core_tool( output_file=ip_file.path, use_official=True, processes=os.cpu_count(), ).filter(file=ip_file.path, already_formatted=True, standard_sort=False) logging.info("Finished generation of %r.", ip_file.path)
def produce_diff(self) -> None: """ Produce the difference from teh downloaded file. """ file_helper = FileHelper(self.final_destination) new = set() kept = set() removed = set() if file_helper.exists(): with file_helper.open("r", encoding="utf-8") as file_stream: current_content = set(x.strip() for x in file_stream) else: current_content = set() downloaded_empty = True for line in self.download_temp_file: if downloaded_empty: downloaded_empty = False line = line.strip() if not line: continue kept_kept, new_new = self.__get_diff_data( current_content, get_subjects_from_line(line, "availability")) new.update(new_new) kept.update(kept_kept) if downloaded_empty: kept = current_content else: compare_base = kept.copy() compare_base.update(new) removed = current_content - compare_base self.download_temp_file.seek(0) return kept, removed, new
def start(self) -> "OurInfrastructureUpdater": file_helper = FileHelper() dir_helper = DirectoryHelper() for file in self.FILES_TO_REMOVE: file_helper.set_path( os.path.join(self.info_manager.WORKSPACE_DIR, file)) if file_helper.exists(): logging.info("Starting deletion of %r", file_helper.path) file_helper.delete() logging.info("Finished deletion of %r", file_helper.path) for directory in self.DIRS_TO_REMOVE: dir_helper.set_path( os.path.join(self.info_manager.WORKSPACE_DIR, directory)) if dir_helper.exists(): logging.info("Starting deletion of %r", dir_helper.path) dir_helper.delete() logging.info("Finished deletion of %r", dir_helper.path) return self
def __init__(self) -> None: self.info_file_instance = FileHelper(self.INFO_FILE) if self.info_file_instance.exists(): self.content = DictHelper().from_json_file(self.info_file_instance.path) else: self.content = {} logging.debug("Administration file path: %r", self.INFO_FILE) logging.debug( "Administration file exists: %r", self.info_file_instance.exists() ) logging.debug("Administration file content:\n%r", self.content) self.update() self.create_missing_index() self.clean() self.store()
def cleanup(self) -> "CSVContinueDataset": """ Deletes the source file (completely). """ if self.source_file: FileHelper(self.source_file).delete() PyFunceble.facility.Logger.debug("Deleted: %r", self.source_file) return self
def csv_file_delete_source_column_target( continuous_integration: ContinuousIntegrationBase, ) -> None: """ Provides the target for the deletion of the source column. """ migrator = InactiveDatasetDeleteSourceColumnMigrator( print_action_to_stdout=True ) migrator.continuous_integration = continuous_integration file_helper = FileHelper(migrator.source_file) if file_helper.exists(): with file_helper.open("r", encoding="utf-8") as file_stream: first_line = next(file_stream) if any(x in first_line for x in migrator.TO_DELETE): print( f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}" "Started deletion of the 'source' column into " f"{migrator.source_file!r}." ) migrator.start() if migrator.done: print( f"{colorama.Fore.GREEN}{colorama.Style.BRIGHT}" "Finished deletion of the 'source' column into " f"{migrator.source_file!r}." ) else: print( f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}" "unfinished deletion of the 'source' column into " f"{migrator.source_file!r}." ) else: PyFunceble.facility.Logger.info( "Stopped csv_file_delete_source_column_target. File does not exist." )