def cleanup_if_necessary(parent_dirname: str) -> None: """ Process the cleanup if it's necessary. """ cleanup_tool = FilesystemCleanup(parent_dirname) running_file_helper = FileHelper( os.path.join( cleanup_tool.get_output_basedir(), PyFunceble.cli.storage.TEST_RUNNING_FILE, )) if not running_file_helper.exists(): self.sessions_id[parent_dirname] = secrets.token_hex(12) cleanup_tool.clean_output_files() running_file_helper.write( f"{self.sessions_id[parent_dirname]} " f"{datetime.datetime.utcnow().isoformat()}", overwrite=True, ) else: possible_session_id = running_file_helper.read().split()[0] try: _ = datetime.datetime.fromisoformat(possible_session_id) self.sessions_id[parent_dirname] = None except (ValueError, TypeError): self.sessions_id[parent_dirname] = possible_session_id
def update_volatile_list(self) -> "UHBPyFuncebleSystemLauncher": """ Updates the content of the :code:`volatile.list` file. """ input_file = FileHelper(outputs.TEMP_VOLATIVE_DESTINATION) volatile_file = FileHelper(outputs.VOLATILE_DESTINATION) clean_file = FileHelper(outputs.CLEAN_DESTINATION) logging.info("Started generation of %r.", volatile_file.path) with volatile_file.open("w", encoding="utf-8") as volatile_file_stream: if clean_file.exists(): with clean_file.open("r", encoding="utf-8") as clean_file_stream: for line in clean_file_stream: line = line.strip() if not line or line.startswith("#") or "." not in line: continue if line.endswith("."): line = line[:-1] volatile_file_stream.write(line + "\n") if input_file.exists(): with input_file.open("r", encoding="utf-8") as input_file_stream: for line in input_file_stream: line = line.strip() if not line or line.startswith("#") or "." not in line: continue if line.endswith("."): line = line[:-1] volatile_file_stream.write(line + "\n") volatile_file.write("\n") whitelist_core_tool( output_file=volatile_file.path, use_official=True, processes=os.cpu_count(), ).filter(file=volatile_file.path, already_formatted=True, standard_sort=False) logging.info("Finished generation of %r.", volatile_file.path) return self
def test_move(self) -> None: """ Tests of the method which let us move a file to another location. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) destination_file_helper = FileHelper(tempfile.gettempdir()) destination_file_helper.set_path( destination_file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() actual_destination = destination_file_helper.exists() self.assertEqual(expected, actual) self.assertEqual(expected, actual_destination) file_helper.write("Hello, World!") expected = True actual = file_helper.exists() self.assertEqual(expected, actual) expected = False actual_destination = destination_file_helper.exists() self.assertEqual(expected, actual_destination) file_helper.move(destination_file_helper.path) expected = True actual_destination = destination_file_helper.exists() self.assertEqual(expected, actual_destination) expected = "Hello, World!" actual = destination_file_helper.read() self.assertEqual(expected, actual) expected = False actual = file_helper.exists() self.assertEqual(expected, actual) expected = True actual_destination = destination_file_helper.exists() self.assertEqual(expected, actual_destination)
def test_copy(self) -> None: """ Tests the method which let us copy a file to another place. """ file_helper = FileHelper(tempfile.gettempdir()) file_helper.set_path(file_helper.join_path(secrets.token_hex(8))) copy_file_helper = FileHelper(tempfile.gettempdir()) copy_file_helper.set_path( copy_file_helper.join_path(secrets.token_hex(8))) expected = False actual = file_helper.exists() actual_copy = copy_file_helper.exists() self.assertEqual(expected, actual) self.assertEqual(expected, actual_copy) file_helper.write("Hello, World!") expected = True actual = file_helper.exists() self.assertEqual(expected, actual) expected = False actual_copy = copy_file_helper.exists() self.assertEqual(expected, actual_copy) file_helper.copy(copy_file_helper.path) expected = True actual_copy = copy_file_helper.exists() self.assertEqual(expected, actual_copy) expected = "Hello, World!" actual = copy_file_helper.read() self.assertEqual(expected, actual) expected = True actual = file_helper.exists() actual_copy = copy_file_helper.exists() self.assertEqual(expected, actual) self.assertEqual(expected, actual_copy)
def test_read_bytes(self) -> None: """ Tests the method which let us read (bytes) a file. """ given = tempfile.NamedTemporaryFile(delete=False) file_helper = FileHelper(given.name) file_helper.write("Hello, World!") given.seek(0) expected = b"Hello, World!" actual = file_helper.read_bytes() self.assertEqual(expected, actual)
def test_write(self) -> None: """ Tests the method which let us write a file. """ given = tempfile.NamedTemporaryFile(delete=False) file_helper = FileHelper(given.name) file_helper.write("Hello, World!") given.seek(0) expected = b"Hello, World!" actual = given.read() self.assertEqual(expected, actual) file_helper.write("Hello, this is Funilrys!") given.seek(0) expected = b"Hello, World!Hello, this is Funilrys!" actual = given.read() self.assertEqual(expected, actual) file_helper.write("Hello, World!", overwrite=True) given.seek(0) expected = b"Hello, World!" actual = given.read() self.assertEqual(expected, actual)
def create_trigger_file_if_necessary(parent_dirname: str) -> None: """ Create the trigger file if necessary. The purpose of the trigger file is to have a file that is always updated until a test is completed done. """ if self.continuous_integration.authorized: cleanup_tool = FilesystemCleanup(parent_dirname) trigger_file_helper = FileHelper( os.path.join( cleanup_tool.get_output_basedir(), PyFunceble.cli.storage.CI_TRIGGER_FILE, )) # Ensures that we always have something to commit. trigger_file_helper.write( f"{self.sessions_id[parent_dirname]} " f"{secrets.token_hex(8)}", overwrite=True, )
def start(self) -> "UpdaterBase": """ Starts the update process. """ if self.administration.raw_link: with importlib.resources.path( "ultimate_hosts_blacklist.test_launcher.data.docs", "about_repository.md", ) as file_path: content = FileHelper(str(file_path)).read() else: with importlib.resources.path( "ultimate_hosts_blacklist.test_launcher.data.docs", "about_repository_self.md", ) as file_path: content = FileHelper(str(file_path)).read() content += "\n\n" with importlib.resources.path( "ultimate_hosts_blacklist.test_launcher.data.docs", "about_ultimate.md") as file_path: content += FileHelper(str(file_path)).read() content += "\n\n" with importlib.resources.path( "ultimate_hosts_blacklist.test_launcher.data.docs", "about_pyfunceble.md") as file_path: content += FileHelper(str(file_path)).read() content += "\n" readme_file = FileHelper( os.path.join(outputs.CURRENT_DIRECTORY, outputs.README_FILENAME)) readme_file.write(content, overwrite=True) logging.info("Updated: %r", readme_file.path)
CommandHelper(f"cd {clone_destination}").execute(raise_on_error=True) commit_count = int( CommandHelper("git rev-list --all --count").execute().split()[0] ) if commit_count > 1000: DirectoryHelper(os.path.join(clone_destination, ".git")).delete() CommandHelper("git init").execute() CommandHelper("git add --all && git commit -m {COMMIT_MESSAGE!r}") if commit_count > 1000: config_file_instance.write(config_file_content, overwrite=True) print( CommandHelper("git push -u --force origin master").execute( raise_on_error=True ) ) else: print(CommandHelper("git push").execute(raise_on_error=True)) CommandHelper("cd -").execute(raise_on_error=True) for line in response: print(line) DirectoryHelper(clone_destination).delete() print("Finished to handle:", repo.name)
def restore_from_backup(self) -> "DirectoryStructureRestoration": """ Restores or reconstruct the output directory. """ # pylint: disable=too-many-locals PyFunceble.facility.Logger.info( "Started restoration of the directory structure") backup = self.get_backup_data() base_dir = self.get_output_basedir() dir_helper = DirectoryHelper() file_helper = FileHelper() if dir_helper.set_path(base_dir).exists(): for root, _, files in os.walk(dir_helper.path): reduced_path = self.get_path_without_base_dir(root) if reduced_path not in backup and root != reduced_path: dir_helper.set_path(root).delete() PyFunceble.facility.Logger.debug( "Added %r into the list of directories to delete. " "Reason: not found in own dataset.", root, ) continue for directory, files in backup.items(): dir_helper.set_path(os.path.join(base_dir, directory)).create() for file, dataset in files.items(): file_full_path = os.path.join(dir_helper.path, file) if (file == ".gitignore" and PyFunceble.cli.storage.STD_PARENT_DIRNAME not in file_full_path): to_delete = file_full_path file_helper.set_path(to_delete).delete() PyFunceble.facility.Logger.debug( "(If exists) Deleted: %r. Reason: We are going to " "replace it with .gitkeep", to_delete, ) file_full_path = file_full_path.replace( ".gitignore", ".gitkeep") file_helper.set_path(file_full_path) if not file_helper.exists(): file_helper.write(dataset["content"], overwrite=True) PyFunceble.facility.Logger.info( "Finished restoration of the directory structure") return self
class Orchestration: """ Orchester the test launch. """ info_manager: Optional[InfoManager] = None authorization_handler: Optional[Authorization] = None origin_file: Optional[FileHelper] = None output_file: Optional[FileHelper] = None def __init__(self, save: bool = False, end: bool = False) -> None: self.info_manager = InfoManager() git_name = EnvironmentVariableHelper("GIT_NAME") git_email = EnvironmentVariableHelper("GIT_EMAIL") if git_email.exists() and "funilrys" in git_email.get_value(): git_name.set_value(dead_hosts.launcher.defaults.envs.GIT_NAME) git_email.set_value(dead_hosts.launcher.defaults.envs.GIT_EMAIL) EnvironmentVariableHelper("PYFUNCEBLE_OUTPUT_LOCATION").set_value( self.info_manager.WORKSPACE_DIR) EnvironmentVariableHelper("PYFUNCEBLE_CONFIG_DIR").set_value( self.info_manager.PYFUNCEBLE_CONFIG_DIR) self.authorization_handler = Authorization(self.info_manager) self.origin_file = FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME, )) self.output_file = FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.OUTPUT_FILENAME, )) logging.info("Origin file: %r", self.origin_file.path) logging.info("Output file: %r", self.output_file.path) if not end and not save: logging.info("Checking authorization to run.") if self.authorization_handler.is_test_authorized(): execute_all_updater(self.info_manager) PyFunceble.facility.ConfigLoader.start() self.fetch_file_to_test() self.run_test() else: logging.info( "Not authorized to run a test until %r (current time) > %r", datetime.now(), self.authorization_handler.next_authorization_time, ) sys.exit(0) elif save: self.run_autosave() else: self.run_end() def fetch_file_to_test(self) -> "Orchestration": """ Provides the latest version of the file to test. """ if self.authorization_handler.is_refresh_authorized(): logging.info( "We are authorized to refresh the lists! Let's do that.") logging.info("Raw Link: %r", self.info_manager.raw_link) if self.info_manager.raw_link: DownloadHelper(self.info_manager.raw_link).download_text( destination=self.origin_file.path) logging.info( "Could get the new version of the list. Updating the download time." ) self.info_manager["last_download_datetime"] = datetime.utcnow() self.info_manager[ "last_download_timestamp"] = self.info_manager[ "last_download_datetime"].timestamp() elif self.origin_file.exists(): logging.info( "Raw link not given or is empty. Let's work with %r.", self.origin_file.path, ) self.origin_file.read() logging.info("Emptying the download time.") self.info_manager[ "last_download_datetime"] = datetime.fromtimestamp(0) self.info_manager[ "last_download_timestamp"] = self.info_manager[ "last_download_datetime"].timestamp() else: logging.info(f"Could not find {self.origin_file.path}. " "Generating empty content to test.") self.origin_file.write("# No content yet.", overwrite=True) logging.info("Emptying the download time.") self.info_manager[ "last_download_datetime"] = datetime.fromtimestamp(0) self.info_manager[ "last_download_timestamp"] = self.info_manager[ "last_download_datetime"].timestamp() logging.info("Updated %r.", self.origin_file.path) return self def run_test(self): """ Run a test of the input list. """ if not self.info_manager.currently_under_test: self.info_manager["currently_under_test"] = True self.info_manager["start_datetime"] = datetime.utcnow() self.info_manager["start_timestamp"] = self.info_manager[ "start_datetime"].timestamp() self.info_manager["finish_datetime"] = datetime.fromtimestamp(0) self.info_manager["finish_timestamp"] = self.info_manager[ "finish_datetime"].timestamp() self.info_manager["latest_part_start_datetime"] = datetime.utcnow() self.info_manager["latest_part_start_timestamp"] = self.info_manager[ "latest_part_start_datetime"].timestamp() self.info_manager[ "latest_part_finish_datetime"] = datetime.fromtimestamp(0) self.info_manager["latest_part_finish_timestamp"] = self.info_manager[ "latest_part_finish_datetime"].timestamp() logging.info("Updated all timestamps.") logging.info("Starting PyFunceble %r ...", PyFunceble.__version__) Command(f"PyFunceble -f {self.origin_file.path}").run_to_stdout() if not dead_hosts.launcher.defaults.envs.GITHUB_TOKEN: self.run_end() def run_autosave(self): """ Run the autosave logic of the administration file. .. warning:: This is just about the administration file not PyFunceble. """ self.info_manager["latest_part_finish_datetime"] = datetime.utcnow() self.info_manager["latest_part_finish_timestamp"] = self.info_manager[ "latest_part_finish_datetime"].timestamp() logging.info("Updated all timestamps.") def run_end(self): """ Run the end logic. """ self.info_manager["currently_under_test"] = False self.info_manager["latest_part_finish_datetime"] = datetime.utcnow() self.info_manager["latest_part_finish_timestamp"] = self.info_manager[ "latest_part_finish_datetime"].timestamp() self.info_manager["finish_datetime"] = self.info_manager[ "latest_part_finish_datetime"] self.info_manager["finish_timestamp"] = self.info_manager[ "finish_datetime"].timestamp() logging.info( "Updated all timestamps and indexes that needed to be updated.") pyfunceble_active_list = FileHelper( os.path.join( self.info_manager.WORKSPACE_DIR, "output", dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME, "domains", "ACTIVE", "list", )) clean_list = [ "# File generated by the Dead-Hosts project with the help of PyFunceble.", "# Dead-Hosts: https://github.com/dead-hosts", "# PyFunceble: https://pyfunceble.github.io", f"# Generation Time: {datetime.utcnow().isoformat()}", ] logging.info( f"PyFunceble ACTIVE list output: {pyfunceble_active_list.path}") if pyfunceble_active_list.exists(): logging.info( f"{pyfunceble_active_list.path} exists, getting and formatting its content." ) self.output_file.write("\n".join(clean_list) + "\n\n", overwrite=True) with pyfunceble_active_list.open("r", encoding="utf-8") as file_stream: for line in file_stream: if line.startswith("#"): continue self.output_file.write(line) self.output_file.write("\n") logging.info("Updated of the content of %r", self.output_file.path)
file_helper = FileHelper() pyfunceble_config_loader = ConfigLoader() if file_helper.set_path( os.path.join( pyfunceble_webworker.storage.CONFIG_DIRECTORY, assets_defaults.OVERWRITE_CONFIG_FILE, )).exists(): local = DictHelper().from_yaml_file(file_helper.path) if local: pyfunceble_config_loader.custom_config = local else: pyfunceble_config_loader.custom_config = dict() else: file_helper.write("") pyfunceble_config_loader.custom_config = Merge( pyfunceble_defaults.PERSISTENT_CONFIG).into( pyfunceble_config_loader.custom_config) pyfunceble_config_loader.start() app = FastAPI( title=assets_defaults.PROJECT_NAME, description=assets_defaults.PROJECT_DESCRIPTION, version=__version__, docs_url=None, redoc_url=None, ) with importlib.resources.path("pyfunceble_webworker.data",
class ReadmeUpdater(UpdaterBase): """ Provides the updater of the README file. """ def __init__(self, info_manager: InfoManager) -> None: self.destination_instance = FileHelper( os.path.join( info_manager.WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.README_FILENAME, )) super().__init__(info_manager) @property def authorized(self) -> bool: return self.destination_instance.exists() def pre(self) -> "ReadmeUpdater": logging.info("Started to update the content of %r!", self.destination_instance.path) return self def post(self) -> "ReadmeUpdater": logging.info("Finished to update the content of %r!", self.destination_instance.path) return self def start(self) -> "ReadmeUpdater": logging.info( "Started to update the `About PyFunceble` section of %r", self.destination_instance.path, ) with importlib.resources.path("dead_hosts.launcher.data.docs", "about_pyfunceble.md") as file_path: updated_version = RegexHelper( dead_hosts.launcher.defaults.markers.ABOUT_FUNCEBLE_REGEX ).replace_match( self.destination_instance.read(), FileHelper(str(file_path)).read() + "\n\n", ) logging.info( "Finished to update the `About PyFunceble` section of %r", self.destination_instance.path, ) logging.info( "Started to update the `About Dead-Hosts` section of %r", self.destination_instance.path, ) with importlib.resources.path("dead_hosts.launcher.data.docs", "about_dead_hosts.md") as file_path: updated_version = RegexHelper( dead_hosts.launcher.defaults.markers.ABOUT_DEAD_HOSTS_REGEX ).replace_match( self.destination_instance.read(), FileHelper(str(file_path)).read() + "\n\n", ) logging.info( "Finished to update the `About Dead-Hosts` section of %s", self.destination_instance.path, ) self.destination_instance.write(updated_version, overwrite=True) return self