def cleanup_if_necessary(parent_dirname: str) -> None:
"""
Process the cleanup if it's necessary.
"""
cleanup_tool = FilesystemCleanup(parent_dirname)
running_file_helper = FileHelper(
os.path.join(
cleanup_tool.get_output_basedir(),
PyFunceble.cli.storage.TEST_RUNNING_FILE,
))
if not running_file_helper.exists():
self.sessions_id[parent_dirname] = secrets.token_hex(12)
cleanup_tool.clean_output_files()
running_file_helper.write(
f"{self.sessions_id[parent_dirname]} "
f"{datetime.datetime.utcnow().isoformat()}",
overwrite=True,
)
else:
possible_session_id = running_file_helper.read().split()[0]
try:
_ = datetime.datetime.fromisoformat(possible_session_id)
self.sessions_id[parent_dirname] = None
except (ValueError, TypeError):
self.sessions_id[parent_dirname] = possible_session_id
def get_backup_data(self) -> dict:
"""
Provides the data which acts as a backup.
"""
result = {}
base_dir = self.get_output_basedir()
file_helper = FileHelper()
hash_helper = HashHelper()
for file in DirectoryHelper(base_dir).list_all_files():
file_helper.set_path(file)
reduced_path = self.get_path_without_base_dir(file)
directory, filename = reduced_path.rsplit(os.sep, 1)
file_hash = hash_helper.hash_file(file_helper.path)
dataset = {
filename: {
"content": file_helper.read(),
"hash": file_hash
}
}
if directory not in result:
result[directory] = dataset
else:
result[directory].update(dataset)
PyFunceble.facility.Logger.debug("Dir Structure to backup:\n%r",
result)
return result
def test_open(self) -> None:
"""
Tests the method which let us open the given file as we want.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
with file_helper.open("w") as file_stream:
file_stream.write("Hello, World!")
expected = True
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = "Hello, World!"
actual = file_helper.read()
self.assertEqual(expected, actual)
def test_move(self) -> None:
"""
Tests of the method which let us move a file to another location.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
destination_file_helper = FileHelper(tempfile.gettempdir())
destination_file_helper.set_path(
destination_file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
actual_destination = destination_file_helper.exists()
self.assertEqual(expected, actual)
self.assertEqual(expected, actual_destination)
file_helper.write("Hello, World!")
expected = True
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = False
actual_destination = destination_file_helper.exists()
self.assertEqual(expected, actual_destination)
file_helper.move(destination_file_helper.path)
expected = True
actual_destination = destination_file_helper.exists()
self.assertEqual(expected, actual_destination)
expected = "Hello, World!"
actual = destination_file_helper.read()
self.assertEqual(expected, actual)
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = True
actual_destination = destination_file_helper.exists()
self.assertEqual(expected, actual_destination)
def test_copy(self) -> None:
"""
Tests the method which let us copy a file to another place.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
copy_file_helper = FileHelper(tempfile.gettempdir())
copy_file_helper.set_path(
copy_file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
actual_copy = copy_file_helper.exists()
self.assertEqual(expected, actual)
self.assertEqual(expected, actual_copy)
file_helper.write("Hello, World!")
expected = True
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = False
actual_copy = copy_file_helper.exists()
self.assertEqual(expected, actual_copy)
file_helper.copy(copy_file_helper.path)
expected = True
actual_copy = copy_file_helper.exists()
self.assertEqual(expected, actual_copy)
expected = "Hello, World!"
actual = copy_file_helper.read()
self.assertEqual(expected, actual)
expected = True
actual = file_helper.exists()
actual_copy = copy_file_helper.exists()
self.assertEqual(expected, actual)
self.assertEqual(expected, actual_copy)
def test_read(self) -> None:
"""
Tests the method which let us read a file.
"""
given = tempfile.NamedTemporaryFile(delete=False)
file_helper = FileHelper(given.name)
file_helper.write("Hello, World!")
given.seek(0)
expected = "Hello, World!"
actual = file_helper.read()
self.assertEqual(expected, actual)
def test_read_file_does_not_exists(self) -> None:
"""
Tests the method which let us read a file for the case that the given
file does not exists.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = None
actual = file_helper.read()
self.assertEqual(expected, actual)
class Administration:
"""
Provides the administration interface.
"""
info_file_location: Optional[str] = None
info_file_helper: Optional[FileHelper] = None
__our_info: dict = dict()
def __init__(self) -> None:
self.info_file_location = outputs.ADMINISTRATION_DESTINATION
self.info_file_helper = FileHelper(self.info_file_location)
self.__our_info.update(self.load())
def __contains__(self, index: str) -> bool:
return index in self.__our_info
def __getitem__(self, index: str) -> Any:
if index in self.__our_info:
return self.__our_info[index]
raise KeyError(index)
def __getattr__(self, index: str) -> Any:
if index in self.__our_info:
return self.__our_info[index]
raise AttributeError(index)
def __setattr__(self, index: str, value: Any) -> None:
if self.__our_info:
self.__our_info[index] = value
else:
super().__setattr__(index, value)
def __delattr__(self, index: str) -> None:
if index in self.info:
del self.info[index]
def __del__(self) -> None:
self.save()
@property
def exists(self) -> bool:
"""
Checks if the administration file exists.
"""
return self.info_file_helper.exists()
@staticmethod
def convert_data_for_system(data: dict) -> dict:
"""
Given the content of the info file, we convert it into something our
system may understand.
.. warning::
This method also delete the keys that are scheduled or declared
for deletion
:param data:
The data to work with.
"""
result = dict()
for key, value in dict(data).items():
if key in infrastructure.ADMINISTRATION_INDEXES["delete"]:
continue
if key == "name":
result[key] = (
CommandHelper("basename $(git rev-parse --show-toplevel)"
).execute().strip())
elif key in infrastructure.ADMINISTRATION_INDEXES["bool"]:
result[key] = bool(int(value))
elif key in infrastructure.ADMINISTRATION_INDEXES["int"]:
result[key] = int(value)
elif key in infrastructure.ADMINISTRATION_INDEXES["dict"]:
result[key] = dict(value)
elif key in infrastructure.ADMINISTRATION_INDEXES["datetime"]:
try:
result[key] = datetime.fromisoformat(value)
except ValueError:
result[key] = datetime.utcnow()
elif key in infrastructure.ADMINISTRATION_INDEXES["epoch"]:
result[key] = datetime.fromtimestamp(value)
else:
result[key] = value
if "pyfunceble" not in result:
result["pyfunceble"] = {
"config": {},
}
for sanitize_type, keys in infrastructure.ADMINISTRATION_INDEXES.items(
):
if sanitize_type == "delete":
continue
for key in keys:
if key not in result:
if sanitize_type == "bool":
local_result = bool(None)
elif sanitize_type == "int":
local_result = 0
elif sanitize_type == "dict":
local_result = dict()
elif sanitize_type == "datetime":
local_result = datetime.utcnow() - timedelta(
days=365.25)
elif sanitize_type == "epoch":
local_result = datetime.utcnow() - timedelta(
days=365.25)
else:
local_result = None
result[key] = local_result
return result
@staticmethod
def convert_data_for_file(data: dict) -> dict:
"""
Given the content of the info file, we convert it into something that
can be saved into a JSON file.
.. warning::
This method also delete the keys that are scheduled or declared
for deletion
:param data:
The data to work with.
"""
result = dict()
for key, value in dict(data).items():
if key in infrastructure.ADMINISTRATION_INDEXES["delete"]:
continue
if key in infrastructure.ADMINISTRATION_INDEXES["bool"]:
result[key] = bool(value)
elif key in infrastructure.ADMINISTRATION_INDEXES["int"]:
result[key] = int(value)
elif key in infrastructure.ADMINISTRATION_INDEXES["datetime"]:
result[key] = value.isoformat()
elif key in infrastructure.ADMINISTRATION_INDEXES["epoch"]:
result[key] = value.timestamp()
else:
result[key] = value
return result
def load(self) -> dict:
"""
Loads and return the content of the administration file.
"""
if self.info_file_helper.read():
content = self.info_file_helper.read()
logging.debug("Administration file content:\n%s", content)
return self.convert_data_for_system(DictHelper().from_json(
content, return_dict_on_error=False))
return dict()
def save(self) -> None:
"""
Saves the loaded content of the administration file.
"""
if self.info_file_helper.exists():
DictHelper(self.convert_data_for_file(
self.__our_info)).to_json_file(self.info_file_location)
repo_slug = f"{ORG_NAME}/{repo.name}"
clone_url = CLONE_TEMPLATE.replace("%slug%", repo_slug)
clone_destination = f"{CLONE_DIR}/{repo.name}"
if not DirectoryHelper(clone_destination).exists():
response = CommandHelper(f"git clone {clone_url} {clone_destination}").run()
else:
response = CommandHelper(f"cd {clone_destination} && git pull").run()
for line in response:
print(line)
config_file_instance = FileHelper(
os.path.join(clone_destination, ".git", "config")
)
config_file_content = config_file_instance.read()
for root, _, files in os.walk(dir_helper.path):
unformatted_root = root
root = root.replace(dir_helper.path, "")
if root.startswith("/"):
root = root[1:]
local_dir_helper = DirectoryHelper(os.path.join(clone_destination, root))
if not local_dir_helper.exists():
local_dir_helper.create()
for file in files:
file_helper.set_path(os.path.join(dir_helper.path, root, file)).copy(
class Orchestration:
"""
Orchester the test launch.
"""
info_manager: Optional[InfoManager] = None
authorization_handler: Optional[Authorization] = None
origin_file: Optional[FileHelper] = None
output_file: Optional[FileHelper] = None
def __init__(self, save: bool = False, end: bool = False) -> None:
self.info_manager = InfoManager()
git_name = EnvironmentVariableHelper("GIT_NAME")
git_email = EnvironmentVariableHelper("GIT_EMAIL")
if git_email.exists() and "funilrys" in git_email.get_value():
git_name.set_value(dead_hosts.launcher.defaults.envs.GIT_NAME)
git_email.set_value(dead_hosts.launcher.defaults.envs.GIT_EMAIL)
EnvironmentVariableHelper("PYFUNCEBLE_OUTPUT_LOCATION").set_value(
self.info_manager.WORKSPACE_DIR)
EnvironmentVariableHelper("PYFUNCEBLE_CONFIG_DIR").set_value(
self.info_manager.PYFUNCEBLE_CONFIG_DIR)
self.authorization_handler = Authorization(self.info_manager)
self.origin_file = FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME,
))
self.output_file = FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
dead_hosts.launcher.defaults.paths.OUTPUT_FILENAME,
))
logging.info("Origin file: %r", self.origin_file.path)
logging.info("Output file: %r", self.output_file.path)
if not end and not save:
logging.info("Checking authorization to run.")
if self.authorization_handler.is_test_authorized():
execute_all_updater(self.info_manager)
PyFunceble.facility.ConfigLoader.start()
self.fetch_file_to_test()
self.run_test()
else:
logging.info(
"Not authorized to run a test until %r (current time) > %r",
datetime.now(),
self.authorization_handler.next_authorization_time,
)
sys.exit(0)
elif save:
self.run_autosave()
else:
self.run_end()
def fetch_file_to_test(self) -> "Orchestration":
"""
Provides the latest version of the file to test.
"""
if self.authorization_handler.is_refresh_authorized():
logging.info(
"We are authorized to refresh the lists! Let's do that.")
logging.info("Raw Link: %r", self.info_manager.raw_link)
if self.info_manager.raw_link:
DownloadHelper(self.info_manager.raw_link).download_text(
destination=self.origin_file.path)
logging.info(
"Could get the new version of the list. Updating the download time."
)
self.info_manager["last_download_datetime"] = datetime.utcnow()
self.info_manager[
"last_download_timestamp"] = self.info_manager[
"last_download_datetime"].timestamp()
elif self.origin_file.exists():
logging.info(
"Raw link not given or is empty. Let's work with %r.",
self.origin_file.path,
)
self.origin_file.read()
logging.info("Emptying the download time.")
self.info_manager[
"last_download_datetime"] = datetime.fromtimestamp(0)
self.info_manager[
"last_download_timestamp"] = self.info_manager[
"last_download_datetime"].timestamp()
else:
logging.info(f"Could not find {self.origin_file.path}. "
"Generating empty content to test.")
self.origin_file.write("# No content yet.", overwrite=True)
logging.info("Emptying the download time.")
self.info_manager[
"last_download_datetime"] = datetime.fromtimestamp(0)
self.info_manager[
"last_download_timestamp"] = self.info_manager[
"last_download_datetime"].timestamp()
logging.info("Updated %r.", self.origin_file.path)
return self
def run_test(self):
"""
Run a test of the input list.
"""
if not self.info_manager.currently_under_test:
self.info_manager["currently_under_test"] = True
self.info_manager["start_datetime"] = datetime.utcnow()
self.info_manager["start_timestamp"] = self.info_manager[
"start_datetime"].timestamp()
self.info_manager["finish_datetime"] = datetime.fromtimestamp(0)
self.info_manager["finish_timestamp"] = self.info_manager[
"finish_datetime"].timestamp()
self.info_manager["latest_part_start_datetime"] = datetime.utcnow()
self.info_manager["latest_part_start_timestamp"] = self.info_manager[
"latest_part_start_datetime"].timestamp()
self.info_manager[
"latest_part_finish_datetime"] = datetime.fromtimestamp(0)
self.info_manager["latest_part_finish_timestamp"] = self.info_manager[
"latest_part_finish_datetime"].timestamp()
logging.info("Updated all timestamps.")
logging.info("Starting PyFunceble %r ...", PyFunceble.__version__)
Command(f"PyFunceble -f {self.origin_file.path}").run_to_stdout()
if not dead_hosts.launcher.defaults.envs.GITHUB_TOKEN:
self.run_end()
def run_autosave(self):
"""
Run the autosave logic of the administration file.
.. warning::
This is just about the administration file not PyFunceble.
"""
self.info_manager["latest_part_finish_datetime"] = datetime.utcnow()
self.info_manager["latest_part_finish_timestamp"] = self.info_manager[
"latest_part_finish_datetime"].timestamp()
logging.info("Updated all timestamps.")
def run_end(self):
"""
Run the end logic.
"""
self.info_manager["currently_under_test"] = False
self.info_manager["latest_part_finish_datetime"] = datetime.utcnow()
self.info_manager["latest_part_finish_timestamp"] = self.info_manager[
"latest_part_finish_datetime"].timestamp()
self.info_manager["finish_datetime"] = self.info_manager[
"latest_part_finish_datetime"]
self.info_manager["finish_timestamp"] = self.info_manager[
"finish_datetime"].timestamp()
logging.info(
"Updated all timestamps and indexes that needed to be updated.")
pyfunceble_active_list = FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
"output",
dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME,
"domains",
"ACTIVE",
"list",
))
clean_list = [
"# File generated by the Dead-Hosts project with the help of PyFunceble.",
"# Dead-Hosts: https://github.com/dead-hosts",
"# PyFunceble: https://pyfunceble.github.io",
f"# Generation Time: {datetime.utcnow().isoformat()}",
]
logging.info(
f"PyFunceble ACTIVE list output: {pyfunceble_active_list.path}")
if pyfunceble_active_list.exists():
logging.info(
f"{pyfunceble_active_list.path} exists, getting and formatting its content."
)
self.output_file.write("\n".join(clean_list) + "\n\n",
overwrite=True)
with pyfunceble_active_list.open("r",
encoding="utf-8") as file_stream:
for line in file_stream:
if line.startswith("#"):
continue
self.output_file.write(line)
self.output_file.write("\n")
logging.info("Updated of the content of %r", self.output_file.path)
class PyFuncebleConfigUpdater(UpdaterBase):
"""
Provides the updated of the PyFunceble configuration.
"""
def __init__(self, info_manager: InfoManager) -> None:
self.pyfunceble_config_file_instance = FileHelper(
os.path.join(info_manager.PYFUNCEBLE_CONFIG_DIR,
".PyFunceble.yaml"))
super().__init__(info_manager)
@property
def authorized(self) -> bool:
return not self.info_manager.own_management
@staticmethod
def get_commit_message(message: str, ping: Optional[str] = None) -> str:
"""
Provides the commit message to use.
"""
if ping:
return f"{message} | cc {ping} | "
return message
def pre(self) -> "PyFuncebleConfigUpdater":
logging.info(
"Started to update %r.",
self.pyfunceble_config_file_instance.path,
)
return self
def post(self) -> "PyFuncebleConfigUpdater":
logging.info(
"Finished to update %r",
self.pyfunceble_config_file_instance.path,
)
return self
def start(self) -> "PyFuncebleConfigUpdater":
with importlib.resources.path(
"PyFunceble.data.infrastructure",
".PyFunceble_production.yaml") as file_path:
local_version = DictHelper(DictHelper().from_yaml_file(
str(file_path))).flatten()
local_version = Merge(
dead_hosts.launcher.defaults.pyfunceble.CONFIGURATION).into(
local_version, strict=True)
if self.info_manager.custom_pyfunceble_config and isinstance(
self.info_manager.custom_pyfunceble_config, dict):
logging.info("Custom PyFunceble configuration given, "
"appending them to the local configuration file.")
local_version = Merge(
self.info_manager.custom_pyfunceble_config).into(local_version,
strict=True)
if self.info_manager.ping:
logging.info(
"Ping names given, appending them to the commit message.")
local_version[
"cli_testing.ci.end_commit_message"] = self.get_commit_message(
local_version["cli_testing.ci.end_commit_message"],
ping=self.info_manager.get_ping_for_commit(),
)
local_version = Merge(
dead_hosts.launcher.defaults.pyfunceble.PERSISTENT_CONFIG).into(
local_version, strict=True)
if FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
dead_hosts.launcher.defaults.paths.EXAMPLE_INFO_FILENAME,
)).exists():
local_version["cli_testing.ci.active"] = False
# Default behavior of PyFunceble since 4.0.0b12.
local_version["cli_testing.autocontinue"] = False
local_version = DictHelper(local_version).unflatten()
DictHelper(local_version).to_yaml_file(
self.pyfunceble_config_file_instance.path)
logging.debug("Configuration:\n%s",
self.pyfunceble_config_file_instance.read())
return self
class ReadmeUpdater(UpdaterBase):
"""
Provides the updater of the README file.
"""
def __init__(self, info_manager: InfoManager) -> None:
self.destination_instance = FileHelper(
os.path.join(
info_manager.WORKSPACE_DIR,
dead_hosts.launcher.defaults.paths.README_FILENAME,
))
super().__init__(info_manager)
@property
def authorized(self) -> bool:
return self.destination_instance.exists()
def pre(self) -> "ReadmeUpdater":
logging.info("Started to update the content of %r!",
self.destination_instance.path)
return self
def post(self) -> "ReadmeUpdater":
logging.info("Finished to update the content of %r!",
self.destination_instance.path)
return self
def start(self) -> "ReadmeUpdater":
logging.info(
"Started to update the `About PyFunceble` section of %r",
self.destination_instance.path,
)
with importlib.resources.path("dead_hosts.launcher.data.docs",
"about_pyfunceble.md") as file_path:
updated_version = RegexHelper(
dead_hosts.launcher.defaults.markers.ABOUT_FUNCEBLE_REGEX
).replace_match(
self.destination_instance.read(),
FileHelper(str(file_path)).read() + "\n\n",
)
logging.info(
"Finished to update the `About PyFunceble` section of %r",
self.destination_instance.path,
)
logging.info(
"Started to update the `About Dead-Hosts` section of %r",
self.destination_instance.path,
)
with importlib.resources.path("dead_hosts.launcher.data.docs",
"about_dead_hosts.md") as file_path:
updated_version = RegexHelper(
dead_hosts.launcher.defaults.markers.ABOUT_DEAD_HOSTS_REGEX
).replace_match(
self.destination_instance.read(),
FileHelper(str(file_path)).read() + "\n\n",
)
logging.info(
"Finished to update the `About Dead-Hosts` section of %s",
self.destination_instance.path,
)
self.destination_instance.write(updated_version, overwrite=True)
return self
