def cleanup_if_necessary(parent_dirname: str) -> None:
"""
Process the cleanup if it's necessary.
"""
cleanup_tool = FilesystemCleanup(parent_dirname)
running_file_helper = FileHelper(
os.path.join(
cleanup_tool.get_output_basedir(),
PyFunceble.cli.storage.TEST_RUNNING_FILE,
))
if not running_file_helper.exists():
self.sessions_id[parent_dirname] = secrets.token_hex(12)
cleanup_tool.clean_output_files()
running_file_helper.write(
f"{self.sessions_id[parent_dirname]} "
f"{datetime.datetime.utcnow().isoformat()}",
overwrite=True,
)
else:
possible_session_id = running_file_helper.read().split()[0]
try:
_ = datetime.datetime.fromisoformat(possible_session_id)
self.sessions_id[parent_dirname] = None
except (ValueError, TypeError):
self.sessions_id[parent_dirname] = possible_session_id
def update_volatile_list(self) -> "UHBPyFuncebleSystemLauncher":
"""
Updates the content of the :code:`volatile.list` file.
"""
input_file = FileHelper(outputs.TEMP_VOLATIVE_DESTINATION)
volatile_file = FileHelper(outputs.VOLATILE_DESTINATION)
clean_file = FileHelper(outputs.CLEAN_DESTINATION)
logging.info("Started generation of %r.", volatile_file.path)
with volatile_file.open("w", encoding="utf-8") as volatile_file_stream:
if clean_file.exists():
with clean_file.open("r",
encoding="utf-8") as clean_file_stream:
for line in clean_file_stream:
line = line.strip()
if not line or line.startswith("#") or "." not in line:
continue
if line.endswith("."):
line = line[:-1]
volatile_file_stream.write(line + "\n")
if input_file.exists():
with input_file.open("r",
encoding="utf-8") as input_file_stream:
for line in input_file_stream:
line = line.strip()
if not line or line.startswith("#") or "." not in line:
continue
if line.endswith("."):
line = line[:-1]
volatile_file_stream.write(line + "\n")
volatile_file.write("\n")
whitelist_core_tool(
output_file=volatile_file.path,
use_official=True,
processes=os.cpu_count(),
).filter(file=volatile_file.path,
already_formatted=True,
standard_sort=False)
logging.info("Finished generation of %r.", volatile_file.path)
return self
def test_move(self) -> None:
"""
Tests of the method which let us move a file to another location.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
destination_file_helper = FileHelper(tempfile.gettempdir())
destination_file_helper.set_path(
destination_file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
actual_destination = destination_file_helper.exists()
self.assertEqual(expected, actual)
self.assertEqual(expected, actual_destination)
file_helper.write("Hello, World!")
expected = True
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = False
actual_destination = destination_file_helper.exists()
self.assertEqual(expected, actual_destination)
file_helper.move(destination_file_helper.path)
expected = True
actual_destination = destination_file_helper.exists()
self.assertEqual(expected, actual_destination)
expected = "Hello, World!"
actual = destination_file_helper.read()
self.assertEqual(expected, actual)
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = True
actual_destination = destination_file_helper.exists()
self.assertEqual(expected, actual_destination)
def test_copy(self) -> None:
"""
Tests the method which let us copy a file to another place.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
copy_file_helper = FileHelper(tempfile.gettempdir())
copy_file_helper.set_path(
copy_file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
actual_copy = copy_file_helper.exists()
self.assertEqual(expected, actual)
self.assertEqual(expected, actual_copy)
file_helper.write("Hello, World!")
expected = True
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = False
actual_copy = copy_file_helper.exists()
self.assertEqual(expected, actual_copy)
file_helper.copy(copy_file_helper.path)
expected = True
actual_copy = copy_file_helper.exists()
self.assertEqual(expected, actual_copy)
expected = "Hello, World!"
actual = copy_file_helper.read()
self.assertEqual(expected, actual)
expected = True
actual = file_helper.exists()
actual_copy = copy_file_helper.exists()
self.assertEqual(expected, actual)
self.assertEqual(expected, actual_copy)
def test_read_bytes(self) -> None:
"""
Tests the method which let us read (bytes) a file.
"""
given = tempfile.NamedTemporaryFile(delete=False)
file_helper = FileHelper(given.name)
file_helper.write("Hello, World!")
given.seek(0)
expected = b"Hello, World!"
actual = file_helper.read_bytes()
self.assertEqual(expected, actual)
def test_write(self) -> None:
"""
Tests the method which let us write a file.
"""
given = tempfile.NamedTemporaryFile(delete=False)
file_helper = FileHelper(given.name)
file_helper.write("Hello, World!")
given.seek(0)
expected = b"Hello, World!"
actual = given.read()
self.assertEqual(expected, actual)
file_helper.write("Hello, this is Funilrys!")
given.seek(0)
expected = b"Hello, World!Hello, this is Funilrys!"
actual = given.read()
self.assertEqual(expected, actual)
file_helper.write("Hello, World!", overwrite=True)
given.seek(0)
expected = b"Hello, World!"
actual = given.read()
self.assertEqual(expected, actual)
def create_trigger_file_if_necessary(parent_dirname: str) -> None:
"""
Create the trigger file if necessary. The purpose of the trigger
file is to have a file that is always updated until a test is
completed done.
"""
if self.continuous_integration.authorized:
cleanup_tool = FilesystemCleanup(parent_dirname)
trigger_file_helper = FileHelper(
os.path.join(
cleanup_tool.get_output_basedir(),
PyFunceble.cli.storage.CI_TRIGGER_FILE,
))
# Ensures that we always have something to commit.
trigger_file_helper.write(
f"{self.sessions_id[parent_dirname]} "
f"{secrets.token_hex(8)}",
overwrite=True,
)
def start(self) -> "UpdaterBase":
"""
Starts the update process.
"""
if self.administration.raw_link:
with importlib.resources.path(
"ultimate_hosts_blacklist.test_launcher.data.docs",
"about_repository.md",
) as file_path:
content = FileHelper(str(file_path)).read()
else:
with importlib.resources.path(
"ultimate_hosts_blacklist.test_launcher.data.docs",
"about_repository_self.md",
) as file_path:
content = FileHelper(str(file_path)).read()
content += "\n\n"
with importlib.resources.path(
"ultimate_hosts_blacklist.test_launcher.data.docs",
"about_ultimate.md") as file_path:
content += FileHelper(str(file_path)).read()
content += "\n\n"
with importlib.resources.path(
"ultimate_hosts_blacklist.test_launcher.data.docs",
"about_pyfunceble.md") as file_path:
content += FileHelper(str(file_path)).read()
content += "\n"
readme_file = FileHelper(
os.path.join(outputs.CURRENT_DIRECTORY, outputs.README_FILENAME))
readme_file.write(content, overwrite=True)
logging.info("Updated: %r", readme_file.path)
CommandHelper(f"cd {clone_destination}").execute(raise_on_error=True)
commit_count = int(
CommandHelper("git rev-list --all --count").execute().split()[0]
)
if commit_count > 1000:
DirectoryHelper(os.path.join(clone_destination, ".git")).delete()
CommandHelper("git init").execute()
CommandHelper("git add --all && git commit -m {COMMIT_MESSAGE!r}")
if commit_count > 1000:
config_file_instance.write(config_file_content, overwrite=True)
print(
CommandHelper("git push -u --force origin master").execute(
raise_on_error=True
)
)
else:
print(CommandHelper("git push").execute(raise_on_error=True))
CommandHelper("cd -").execute(raise_on_error=True)
for line in response:
print(line)
DirectoryHelper(clone_destination).delete()
print("Finished to handle:", repo.name)
def restore_from_backup(self) -> "DirectoryStructureRestoration":
"""
Restores or reconstruct the output directory.
"""
# pylint: disable=too-many-locals
PyFunceble.facility.Logger.info(
"Started restoration of the directory structure")
backup = self.get_backup_data()
base_dir = self.get_output_basedir()
dir_helper = DirectoryHelper()
file_helper = FileHelper()
if dir_helper.set_path(base_dir).exists():
for root, _, files in os.walk(dir_helper.path):
reduced_path = self.get_path_without_base_dir(root)
if reduced_path not in backup and root != reduced_path:
dir_helper.set_path(root).delete()
PyFunceble.facility.Logger.debug(
"Added %r into the list of directories to delete. "
"Reason: not found in own dataset.",
root,
)
continue
for directory, files in backup.items():
dir_helper.set_path(os.path.join(base_dir, directory)).create()
for file, dataset in files.items():
file_full_path = os.path.join(dir_helper.path, file)
if (file == ".gitignore"
and PyFunceble.cli.storage.STD_PARENT_DIRNAME
not in file_full_path):
to_delete = file_full_path
file_helper.set_path(to_delete).delete()
PyFunceble.facility.Logger.debug(
"(If exists) Deleted: %r. Reason: We are going to "
"replace it with .gitkeep",
to_delete,
)
file_full_path = file_full_path.replace(
".gitignore", ".gitkeep")
file_helper.set_path(file_full_path)
if not file_helper.exists():
file_helper.write(dataset["content"], overwrite=True)
PyFunceble.facility.Logger.info(
"Finished restoration of the directory structure")
return self
class Orchestration:
"""
Orchester the test launch.
"""
info_manager: Optional[InfoManager] = None
authorization_handler: Optional[Authorization] = None
origin_file: Optional[FileHelper] = None
output_file: Optional[FileHelper] = None
def __init__(self, save: bool = False, end: bool = False) -> None:
self.info_manager = InfoManager()
git_name = EnvironmentVariableHelper("GIT_NAME")
git_email = EnvironmentVariableHelper("GIT_EMAIL")
if git_email.exists() and "funilrys" in git_email.get_value():
git_name.set_value(dead_hosts.launcher.defaults.envs.GIT_NAME)
git_email.set_value(dead_hosts.launcher.defaults.envs.GIT_EMAIL)
EnvironmentVariableHelper("PYFUNCEBLE_OUTPUT_LOCATION").set_value(
self.info_manager.WORKSPACE_DIR)
EnvironmentVariableHelper("PYFUNCEBLE_CONFIG_DIR").set_value(
self.info_manager.PYFUNCEBLE_CONFIG_DIR)
self.authorization_handler = Authorization(self.info_manager)
self.origin_file = FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME,
))
self.output_file = FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
dead_hosts.launcher.defaults.paths.OUTPUT_FILENAME,
))
logging.info("Origin file: %r", self.origin_file.path)
logging.info("Output file: %r", self.output_file.path)
if not end and not save:
logging.info("Checking authorization to run.")
if self.authorization_handler.is_test_authorized():
execute_all_updater(self.info_manager)
PyFunceble.facility.ConfigLoader.start()
self.fetch_file_to_test()
self.run_test()
else:
logging.info(
"Not authorized to run a test until %r (current time) > %r",
datetime.now(),
self.authorization_handler.next_authorization_time,
)
sys.exit(0)
elif save:
self.run_autosave()
else:
self.run_end()
def fetch_file_to_test(self) -> "Orchestration":
"""
Provides the latest version of the file to test.
"""
if self.authorization_handler.is_refresh_authorized():
logging.info(
"We are authorized to refresh the lists! Let's do that.")
logging.info("Raw Link: %r", self.info_manager.raw_link)
if self.info_manager.raw_link:
DownloadHelper(self.info_manager.raw_link).download_text(
destination=self.origin_file.path)
logging.info(
"Could get the new version of the list. Updating the download time."
)
self.info_manager["last_download_datetime"] = datetime.utcnow()
self.info_manager[
"last_download_timestamp"] = self.info_manager[
"last_download_datetime"].timestamp()
elif self.origin_file.exists():
logging.info(
"Raw link not given or is empty. Let's work with %r.",
self.origin_file.path,
)
self.origin_file.read()
logging.info("Emptying the download time.")
self.info_manager[
"last_download_datetime"] = datetime.fromtimestamp(0)
self.info_manager[
"last_download_timestamp"] = self.info_manager[
"last_download_datetime"].timestamp()
else:
logging.info(f"Could not find {self.origin_file.path}. "
"Generating empty content to test.")
self.origin_file.write("# No content yet.", overwrite=True)
logging.info("Emptying the download time.")
self.info_manager[
"last_download_datetime"] = datetime.fromtimestamp(0)
self.info_manager[
"last_download_timestamp"] = self.info_manager[
"last_download_datetime"].timestamp()
logging.info("Updated %r.", self.origin_file.path)
return self
def run_test(self):
"""
Run a test of the input list.
"""
if not self.info_manager.currently_under_test:
self.info_manager["currently_under_test"] = True
self.info_manager["start_datetime"] = datetime.utcnow()
self.info_manager["start_timestamp"] = self.info_manager[
"start_datetime"].timestamp()
self.info_manager["finish_datetime"] = datetime.fromtimestamp(0)
self.info_manager["finish_timestamp"] = self.info_manager[
"finish_datetime"].timestamp()
self.info_manager["latest_part_start_datetime"] = datetime.utcnow()
self.info_manager["latest_part_start_timestamp"] = self.info_manager[
"latest_part_start_datetime"].timestamp()
self.info_manager[
"latest_part_finish_datetime"] = datetime.fromtimestamp(0)
self.info_manager["latest_part_finish_timestamp"] = self.info_manager[
"latest_part_finish_datetime"].timestamp()
logging.info("Updated all timestamps.")
logging.info("Starting PyFunceble %r ...", PyFunceble.__version__)
Command(f"PyFunceble -f {self.origin_file.path}").run_to_stdout()
if not dead_hosts.launcher.defaults.envs.GITHUB_TOKEN:
self.run_end()
def run_autosave(self):
"""
Run the autosave logic of the administration file.
.. warning::
This is just about the administration file not PyFunceble.
"""
self.info_manager["latest_part_finish_datetime"] = datetime.utcnow()
self.info_manager["latest_part_finish_timestamp"] = self.info_manager[
"latest_part_finish_datetime"].timestamp()
logging.info("Updated all timestamps.")
def run_end(self):
"""
Run the end logic.
"""
self.info_manager["currently_under_test"] = False
self.info_manager["latest_part_finish_datetime"] = datetime.utcnow()
self.info_manager["latest_part_finish_timestamp"] = self.info_manager[
"latest_part_finish_datetime"].timestamp()
self.info_manager["finish_datetime"] = self.info_manager[
"latest_part_finish_datetime"]
self.info_manager["finish_timestamp"] = self.info_manager[
"finish_datetime"].timestamp()
logging.info(
"Updated all timestamps and indexes that needed to be updated.")
pyfunceble_active_list = FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
"output",
dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME,
"domains",
"ACTIVE",
"list",
))
clean_list = [
"# File generated by the Dead-Hosts project with the help of PyFunceble.",
"# Dead-Hosts: https://github.com/dead-hosts",
"# PyFunceble: https://pyfunceble.github.io",
f"# Generation Time: {datetime.utcnow().isoformat()}",
]
logging.info(
f"PyFunceble ACTIVE list output: {pyfunceble_active_list.path}")
if pyfunceble_active_list.exists():
logging.info(
f"{pyfunceble_active_list.path} exists, getting and formatting its content."
)
self.output_file.write("\n".join(clean_list) + "\n\n",
overwrite=True)
with pyfunceble_active_list.open("r",
encoding="utf-8") as file_stream:
for line in file_stream:
if line.startswith("#"):
continue
self.output_file.write(line)
self.output_file.write("\n")
logging.info("Updated of the content of %r", self.output_file.path)
file_helper = FileHelper()
pyfunceble_config_loader = ConfigLoader()
if file_helper.set_path(
os.path.join(
pyfunceble_webworker.storage.CONFIG_DIRECTORY,
assets_defaults.OVERWRITE_CONFIG_FILE,
)).exists():
local = DictHelper().from_yaml_file(file_helper.path)
if local:
pyfunceble_config_loader.custom_config = local
else:
pyfunceble_config_loader.custom_config = dict()
else:
file_helper.write("")
pyfunceble_config_loader.custom_config = Merge(
pyfunceble_defaults.PERSISTENT_CONFIG).into(
pyfunceble_config_loader.custom_config)
pyfunceble_config_loader.start()
app = FastAPI(
title=assets_defaults.PROJECT_NAME,
description=assets_defaults.PROJECT_DESCRIPTION,
version=__version__,
docs_url=None,
redoc_url=None,
)
with importlib.resources.path("pyfunceble_webworker.data",
class ReadmeUpdater(UpdaterBase):
"""
Provides the updater of the README file.
"""
def __init__(self, info_manager: InfoManager) -> None:
self.destination_instance = FileHelper(
os.path.join(
info_manager.WORKSPACE_DIR,
dead_hosts.launcher.defaults.paths.README_FILENAME,
))
super().__init__(info_manager)
@property
def authorized(self) -> bool:
return self.destination_instance.exists()
def pre(self) -> "ReadmeUpdater":
logging.info("Started to update the content of %r!",
self.destination_instance.path)
return self
def post(self) -> "ReadmeUpdater":
logging.info("Finished to update the content of %r!",
self.destination_instance.path)
return self
def start(self) -> "ReadmeUpdater":
logging.info(
"Started to update the `About PyFunceble` section of %r",
self.destination_instance.path,
)
with importlib.resources.path("dead_hosts.launcher.data.docs",
"about_pyfunceble.md") as file_path:
updated_version = RegexHelper(
dead_hosts.launcher.defaults.markers.ABOUT_FUNCEBLE_REGEX
).replace_match(
self.destination_instance.read(),
FileHelper(str(file_path)).read() + "\n\n",
)
logging.info(
"Finished to update the `About PyFunceble` section of %r",
self.destination_instance.path,
)
logging.info(
"Started to update the `About Dead-Hosts` section of %r",
self.destination_instance.path,
)
with importlib.resources.path("dead_hosts.launcher.data.docs",
"about_dead_hosts.md") as file_path:
updated_version = RegexHelper(
dead_hosts.launcher.defaults.markers.ABOUT_DEAD_HOSTS_REGEX
).replace_match(
self.destination_instance.read(),
FileHelper(str(file_path)).read() + "\n\n",
)
logging.info(
"Finished to update the `About Dead-Hosts` section of %s",
self.destination_instance.path,
)
self.destination_instance.write(updated_version, overwrite=True)
return self
