def test_open(self) -> None:
"""
Tests the method which let us open the given file as we want.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
with file_helper.open("w") as file_stream:
file_stream.write("Hello, World!")
expected = True
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = "Hello, World!"
actual = file_helper.read()
self.assertEqual(expected, actual)
def test_delete(self) -> None:
"""
Tests the method which let us delete a file.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
with open(file_helper.path, "w", encoding="utf-8") as file_stream:
file_stream.write("")
expected = True
actual = file_helper.exists()
self.assertEqual(expected, actual)
file_helper.delete()
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
def get_content(self) -> Optional[dict]:
"""
Provides the cached or the real contend of the dataset (after caching)
:raise FileNotFoundError:
When the declared file does not exists.
"""
if (
bool(self.STORAGE_INDEX)
and hasattr(PyFunceble.storage, self.STORAGE_INDEX)
and bool(getattr(PyFunceble.storage, self.STORAGE_INDEX))
):
return getattr(PyFunceble.storage, self.STORAGE_INDEX)
file_helper = FileHelper(self.source_file)
if not file_helper.exists() and bool(
self.DOWNLOADER
): # pragma: no cover ## This is just a safety endpoint.
self.DOWNLOADER.start()
if not file_helper.exists():
raise FileNotFoundError(file_helper.path)
content = DictHelper().from_json_file(
self.source_file, return_dict_on_error=False
)
setattr(PyFunceble.storage, self.STORAGE_INDEX, content)
return content
def test_get_size(self) -> None:
"""
Tests the method which let us get the size of a file.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
with open(file_helper.path, "w", encoding="utf-8") as file_stream:
file_stream.write("Hello, World!")
expected = True
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = 13
actual = file_helper.get_size()
self.assertEqual(expected, actual)
os.remove(file_helper.path)
def update_volatile_list(self) -> "UHBPyFuncebleSystemLauncher":
"""
Updates the content of the :code:`volatile.list` file.
"""
input_file = FileHelper(outputs.TEMP_VOLATIVE_DESTINATION)
volatile_file = FileHelper(outputs.VOLATILE_DESTINATION)
clean_file = FileHelper(outputs.CLEAN_DESTINATION)
logging.info("Started generation of %r.", volatile_file.path)
with volatile_file.open("w", encoding="utf-8") as volatile_file_stream:
if clean_file.exists():
with clean_file.open("r",
encoding="utf-8") as clean_file_stream:
for line in clean_file_stream:
line = line.strip()
if not line or line.startswith("#") or "." not in line:
continue
if line.endswith("."):
line = line[:-1]
volatile_file_stream.write(line + "\n")
if input_file.exists():
with input_file.open("r",
encoding="utf-8") as input_file_stream:
for line in input_file_stream:
line = line.strip()
if not line or line.startswith("#") or "." not in line:
continue
if line.endswith("."):
line = line[:-1]
volatile_file_stream.write(line + "\n")
volatile_file.write("\n")
whitelist_core_tool(
output_file=volatile_file.path,
use_official=True,
processes=os.cpu_count(),
).filter(file=volatile_file.path,
already_formatted=True,
standard_sort=False)
logging.info("Finished generation of %r.", volatile_file.path)
return self
def start(self) -> "OurInfrastructureUpdater":
file_helper = FileHelper()
dir_helper = DirectoryHelper()
for file in self.FILES_TO_REMOVE:
file_helper.set_path(
os.path.join(self.info_manager.WORKSPACE_DIR, file))
if file_helper.exists():
logging.info("Starting deletion of %r", file_helper.path)
file_helper.delete()
logging.info("Finished deletion of %r", file_helper.path)
for directory in self.DIRS_TO_REMOVE:
dir_helper.set_path(
os.path.join(self.info_manager.WORKSPACE_DIR, directory))
if dir_helper.exists():
logging.info("Starting deletion of %r", dir_helper.path)
dir_helper.delete()
logging.info("Finished deletion of %r", dir_helper.path)
return self
def cleanup_if_necessary(parent_dirname: str) -> None:
"""
Process the cleanup if it's necessary.
"""
cleanup_tool = FilesystemCleanup(parent_dirname)
running_file_helper = FileHelper(
os.path.join(
cleanup_tool.get_output_basedir(),
PyFunceble.cli.storage.TEST_RUNNING_FILE,
))
if not running_file_helper.exists():
self.sessions_id[parent_dirname] = secrets.token_hex(12)
cleanup_tool.clean_output_files()
running_file_helper.write(
f"{self.sessions_id[parent_dirname]} "
f"{datetime.datetime.utcnow().isoformat()}",
overwrite=True,
)
else:
possible_session_id = running_file_helper.read().split()[0]
try:
_ = datetime.datetime.fromisoformat(possible_session_id)
self.sessions_id[parent_dirname] = None
except (ValueError, TypeError):
self.sessions_id[parent_dirname] = possible_session_id
def update_ip_list() -> "UHBPyFuncebleSystemLauncher":
"""
Updates the content of the :code:`ip.list` file.
"""
input_file = FileHelper(outputs.IP_SUBJECTS_DESTINATION)
ip_file = FileHelper(outputs.IP_DESTINATION)
if input_file.exists():
logging.info("Started generation of %r.", ip_file.path)
with input_file.open(
"r", encoding="utf-8") as input_file_stream, ip_file.open(
"w", encoding="utf-8") as ip_file_stream:
for line in input_file_stream:
if not line.strip() or line.startswith("#"):
continue
ip_file_stream.write("\n".join(line.split()[1:]) + "\n")
ip_file_stream.write("\n")
whitelist_core_tool(
output_file=ip_file.path,
use_official=True,
processes=os.cpu_count(),
).filter(file=ip_file.path,
already_formatted=True,
standard_sort=False)
logging.info("Finished generation of %r.", ip_file.path)
def update_clean_list(self) -> "UHBPyFuncebleSystemLauncher":
"""
Updates the content of the :code:`clean.list` file.
"""
input_file = FileHelper(outputs.ACTIVE_SUBJECTS_DESTINATION)
clean_file = FileHelper(outputs.CLEAN_DESTINATION)
if input_file.exists():
logging.info("Started generation of %r.", clean_file.path)
with input_file.open(
"r",
encoding="utf-8") as input_file_stream, clean_file.open(
"w", encoding="utf-8") as clean_file_stream:
for line in input_file_stream:
line = line.strip()
if not line or line.startswith("#") or "." not in line:
continue
if line.endswith("."):
line = line[:-1]
clean_file_stream.write("\n" + line)
logging.info("Finished generation of %r.", clean_file.path)
return self
def run_end(self):
"""
Run the end logic.
"""
self.info_manager["currently_under_test"] = False
self.info_manager["latest_part_finish_datetime"] = datetime.utcnow()
self.info_manager["latest_part_finish_timestamp"] = self.info_manager[
"latest_part_finish_datetime"].timestamp()
self.info_manager["finish_datetime"] = self.info_manager[
"latest_part_finish_datetime"]
self.info_manager["finish_timestamp"] = self.info_manager[
"finish_datetime"].timestamp()
logging.info(
"Updated all timestamps and indexes that needed to be updated.")
pyfunceble_active_list = FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
"output",
dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME,
"domains",
"ACTIVE",
"list",
))
clean_list = [
"# File generated by the Dead-Hosts project with the help of PyFunceble.",
"# Dead-Hosts: https://github.com/dead-hosts",
"# PyFunceble: https://pyfunceble.github.io",
f"# Generation Time: {datetime.utcnow().isoformat()}",
]
logging.info(
f"PyFunceble ACTIVE list output: {pyfunceble_active_list.path}")
if pyfunceble_active_list.exists():
logging.info(
f"{pyfunceble_active_list.path} exists, getting and formatting its content."
)
self.output_file.write("\n".join(clean_list) + "\n\n",
overwrite=True)
with pyfunceble_active_list.open("r",
encoding="utf-8") as file_stream:
for line in file_stream:
if line.startswith("#"):
continue
self.output_file.write(line)
self.output_file.write("\n")
logging.info("Updated of the content of %r", self.output_file.path)
def start(self) -> "InfrastructureCleaner":
analytic_directory = DirectoryHelper(
os.path.join(outputs.OUTPUT_ROOT_DIRECTORY, "json"))
if analytic_directory.exists():
for element in os.listdir(outputs.OUTPUT_ROOT_DIRECTORY):
if any(x in element for x in self.STD_FILES_TO_IGNORE):
continue
dir_helper = DirectoryHelper(
os.path.join(outputs.OUTPUT_ROOT_DIRECTORY, element))
if dir_helper.exists():
dir_helper.delete()
logging.info("Deleted: %r", dir_helper.path)
del analytic_directory
for file in self.FILES_TO_REMOVE:
if not isinstance(file, list):
file_helper = FileHelper(
os.path.join(outputs.CURRENT_DIRECTORY, file))
else:
file_helper = FileHelper(
os.path.join(outputs.CURRENT_DIRECTORY, *file))
if file_helper.exists():
file_helper.delete()
logging.info("Deleted: %r", file_helper.path)
for file in self.FILES_TO_MOVE_TO_PYFUNCEBLE_CONFIG:
file_helper = FileHelper(
os.path.join(outputs.CURRENT_DIRECTORY, file))
if file_helper.exists():
file_helper.move(
os.path.join(outputs.PYFUNCEBLE_CONFIG_DIRECTORY, file))
logging.info("Moved: %r", file_helper.path)
return self
def get_content(self) -> open:
"""
Provides a file handler which does let you read the content line by
line.
:raise FileNotFoundError:
When the declared file does not exists.
"""
file_helper = FileHelper(self.source_file)
if not file_helper.exists() and bool(
self.DOWNLOADER): # pragma: no cover
## pragma reason: Safety.
self.DOWNLOADER.start()
if not file_helper.exists():
raise FileNotFoundError(file_helper.path)
return file_helper.open("r", encoding="utf-8")
def fetch_dataset(self) -> "FilesystemCounter":
"""
Fetches the source file into the current instance.
"""
file_helper = FileHelper(self.source_file)
if file_helper.exists():
self.dataset = DictHelper().from_json_file(file_helper.path)
else:
self.dataset = copy.deepcopy(self.STD_DATASET)
return self
def test_move(self) -> None:
"""
Tests of the method which let us move a file to another location.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
destination_file_helper = FileHelper(tempfile.gettempdir())
destination_file_helper.set_path(
destination_file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
actual_destination = destination_file_helper.exists()
self.assertEqual(expected, actual)
self.assertEqual(expected, actual_destination)
file_helper.write("Hello, World!")
expected = True
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = False
actual_destination = destination_file_helper.exists()
self.assertEqual(expected, actual_destination)
file_helper.move(destination_file_helper.path)
expected = True
actual_destination = destination_file_helper.exists()
self.assertEqual(expected, actual_destination)
expected = "Hello, World!"
actual = destination_file_helper.read()
self.assertEqual(expected, actual)
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = True
actual_destination = destination_file_helper.exists()
self.assertEqual(expected, actual_destination)
def start(self) -> "PyFuncebleConfigLocationUpdater":
for file in self.FILES_TO_MOVE:
source_file = FileHelper(
os.path.join(self.info_manager.WORKSPACE_DIR, file))
destination_file = FileHelper(
os.path.join(self.info_manager.PYFUNCEBLE_CONFIG_DIR, file))
if source_file.exists():
logging.info(
"Starting to move %r into %r.",
source_file.path,
destination_file.path,
)
destination_file.delete()
source_file.move(destination_file.path)
logging.info(
"Finished to move %r into %r",
source_file.path,
destination_file.path,
)
else:
logging.info(
"Did not moved move %r into %r: It does not exists.",
source_file.path,
destination_file.path,
)
if (all(
FileHelper(
os.path.join(self.info_manager.PYFUNCEBLE_CONFIG_DIR,
x)).exists()
for x in self.INACTIVE_FILES_TO_DELETE) or FileHelper(
os.path.join(
self.info_manager.PYFUNCEBLE_CONFIG_DIR,
self.INACTIVE_FILES_TO_DELETE[0],
)).exists()):
logging.info("Starting to delete inactive files.", )
for inactive_file in self.INACTIVE_FILES_TO_DELETE:
FileHelper(
os.path.join(self.info_manager.PYFUNCEBLE_CONFIG_DIR,
inactive_file)).delete()
logging.info("Finished to delete inactive files.", )
return self
def test_copy(self) -> None:
"""
Tests the method which let us copy a file to another place.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
copy_file_helper = FileHelper(tempfile.gettempdir())
copy_file_helper.set_path(
copy_file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
actual_copy = copy_file_helper.exists()
self.assertEqual(expected, actual)
self.assertEqual(expected, actual_copy)
file_helper.write("Hello, World!")
expected = True
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = False
actual_copy = copy_file_helper.exists()
self.assertEqual(expected, actual_copy)
file_helper.copy(copy_file_helper.path)
expected = True
actual_copy = copy_file_helper.exists()
self.assertEqual(expected, actual_copy)
expected = "Hello, World!"
actual = copy_file_helper.read()
self.assertEqual(expected, actual)
expected = True
actual = file_helper.exists()
actual_copy = copy_file_helper.exists()
self.assertEqual(expected, actual)
self.assertEqual(expected, actual_copy)
def get_csv_writer(self) -> Tuple[csv.DictWriter, open]:
"""
Provides the standard and initiated CSV Dict writer along with the
file that was open with it.
"""
file_helper = FileHelper(self.source_file)
add_header = not file_helper.exists()
file_handler = file_helper.open("a+", newline="")
writer = csv.DictWriter(file_handler, fieldnames=self.FIELDS)
if add_header:
writer.writeheader()
return writer, file_handler
def produce_diff(self) -> None:
"""
Produce the difference from teh downloaded file.
"""
file_helper = FileHelper(self.final_destination)
new = set()
kept = set()
removed = set()
if file_helper.exists():
with file_helper.open("r", encoding="utf-8") as file_stream:
current_content = set(x.strip() for x in file_stream)
else:
current_content = set()
downloaded_empty = True
for line in self.download_temp_file:
if downloaded_empty:
downloaded_empty = False
line = line.strip()
if not line:
continue
kept_kept, new_new = self.__get_diff_data(
current_content, get_subjects_from_line(line, "availability"))
new.update(new_new)
kept.update(kept_kept)
if downloaded_empty:
kept = current_content
else:
compare_base = kept.copy()
compare_base.update(new)
removed = current_content - compare_base
self.download_temp_file.seek(0)
return kept, removed, new
def migrate(self) -> "MigratorBase":
"""
Provides the migrator (itself).
"""
file_helper = FileHelper(self.source_file)
if file_helper.exists():
with file_helper.open("r", encoding="utf-8") as file_stream:
first_line = next(file_stream)
if any(x in first_line for x in self.TO_DELETE):
temp_destination = tempfile.NamedTemporaryFile(
"a+", newline="", encoding="utf-8", delete=False
)
file_handler = file_helper.open(newline="")
reader = csv.DictReader(file_handler)
writer = csv.DictWriter(
temp_destination,
fieldnames=[x for x in self.FIELDS if x not in self.TO_DELETE],
)
writer.writeheader()
keys_found = False
for row in reader:
row = dict(row)
for key in self.TO_DELETE:
if key in row:
del row[key]
keys_found = True
if not keys_found:
break
writer.writerow(row)
if self.print_action_to_stdout:
print_single_line()
temp_destination.seek(0)
FileHelper(temp_destination.name).move(self.source_file)
self.done = True
def test_read_bytes_file_does_not_exists(self) -> None:
"""
Tests the method which let us read (bytes) a file for the case that
the given file does not exists.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = None
actual = file_helper.read_bytes()
self.assertEqual(expected, actual)
def csv_file_delete_source_column_target(
continuous_integration: ContinuousIntegrationBase,
) -> None:
"""
Provides the target for the deletion of the source column.
"""
migrator = InactiveDatasetDeleteSourceColumnMigrator(
print_action_to_stdout=True
)
migrator.continuous_integration = continuous_integration
file_helper = FileHelper(migrator.source_file)
if file_helper.exists():
with file_helper.open("r", encoding="utf-8") as file_stream:
first_line = next(file_stream)
if any(x in first_line for x in migrator.TO_DELETE):
print(
f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}"
"Started deletion of the 'source' column into "
f"{migrator.source_file!r}."
)
migrator.start()
if migrator.done:
print(
f"{colorama.Fore.GREEN}{colorama.Style.BRIGHT}"
"Finished deletion of the 'source' column into "
f"{migrator.source_file!r}."
)
else:
print(
f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}"
"unfinished deletion of the 'source' column into "
f"{migrator.source_file!r}."
)
else:
PyFunceble.facility.Logger.info(
"Stopped csv_file_delete_source_column_target. File does not exist."
)
def update_whitelisted_list(self) -> "UHBPyFuncebleSystemLauncher":
"""
Updates the content of the :code:`whitelist.list` file.
"""
input_file = FileHelper(outputs.CLEAN_DESTINATION)
whitelist_file = FileHelper(outputs.WHITELISTED_DESTINATION)
if input_file.exists():
logging.info("Started generation of %r.", whitelist_file.path)
whitelist_core_tool(
output_file=whitelist_file.path,
use_official=True,
processes=os.cpu_count(),
).filter(file=input_file.path,
already_formatted=True,
standard_sort=False)
logging.info("Finished generation of %r.", whitelist_file.path)
return self
def get_content(self) -> Generator[Optional[dict], None, None]:
"""
Provides a generator which provides the next line to read.
"""
file_helper = FileHelper(self.source_file)
if file_helper.exists():
file_handler = file_helper.open(newline="")
reader = csv.DictReader(file_handler)
for row in reader:
if "tested_at" in row:
try:
row["tested_at"] = datetime.fromisoformat(
row["tested_at"])
except (TypeError, ValueError):
row["tested_at"] = datetime.utcnow() - timedelta(
days=365)
yield row
file_handler.close()
def migrate(self) -> "InactiveJSON2CSVMigrator":
"""
Starts the migration.
"""
file_helper = FileHelper(self.source_file)
if file_helper.exists():
self.dataset.set_authorized(True)
dataset = {
"idna_subject": None,
"status": None,
"status_source": None,
"checker_type": "AVAILABILITY",
"destination": None,
"source": None,
"tested_at": None,
"session_id": None,
}
delete_file = True
with file_helper.open("r", encoding="utf-8") as file_stream:
for line in file_stream:
if (self.continuous_integration and
self.continuous_integration.is_time_exceeded()):
delete_file = False
break
line = (line.strip().replace('"',
"").replace(",", "").replace(
"{",
"",
).replace("}", ""))
if ":" not in line:
continue
index, value = [x.strip() for x in line.rsplit(":", 1)]
if not value:
if index.isdigit():
dataset[
"tested_at"] = datetime.datetime.fromtimestamp(
float(index)).isoformat()
else:
dataset["source"] = os.path.abspath(index)
dataset[
"destination"] = get_destination_from_origin(
dataset["source"])
continue
dataset["idna_subject"] = domain2idna.domain2idna(index)
dataset["status"] = value
if not dataset["tested_at"]:
dataset["tested_at"] = datetime.datetime.utcnow(
).isoformat()
PyFunceble.facility.Logger.debug("Decoded dataset:\n%r.",
dataset)
self.dataset.update(dataset)
if self.print_action_to_stdout:
print_single_line()
PyFunceble.facility.Logger.info("Added %r into %r",
dataset["idna_subject"],
self.dataset)
if delete_file:
file_helper.delete()
self.done = True
return self
class InfoManager:
"""
Provides an interface for the management of the :code:`info.json` file.
.. warning::
Keep in mind that this interface provides everything that may be
needed by other interfaces.
"""
WORKSPACE_DIR: str = dead_hosts.launcher.defaults.envs.WORKSPACE_DIR
PYFUNCEBLE_CONFIG_DIR: str = (
dead_hosts.launcher.defaults.paths.PYFUNCEBLE_CONFIG_DIRECTORY
)
GHA_WORKFLOWS_DIR: str = os.path.join(
WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.GHA_WORKFLOW_DIR
)
INFO_FILE = os.path.join(
WORKSPACE_DIR, dead_hosts.launcher.defaults.paths.INFO_FILENAME
)
def __init__(self) -> None:
self.info_file_instance = FileHelper(self.INFO_FILE)
if self.info_file_instance.exists():
self.content = DictHelper().from_json_file(self.info_file_instance.path)
else:
self.content = {}
logging.debug("Administration file path: %r", self.INFO_FILE)
logging.debug(
"Administration file exists: %r", self.info_file_instance.exists()
)
logging.debug("Administration file content:\n%r", self.content)
self.update()
self.create_missing_index()
self.clean()
self.store()
def __getattr__(self, index: str) -> Any:
if index in self.content:
return self.content[index]
raise AttributeError(index)
def __getitem__(self, index: str) -> Any:
if index in self.content:
return self.content[index]
raise AttributeError(index)
def __setitem__(self, index: str, value: Any):
self.content[index] = value
def __del__(self):
self.store()
def store(self) -> "InfoManager":
"""
Stores the current state.
"""
local_copy = {}
for index, value in self.content.items():
if index.endswith("_timestamp") and isinstance(value, datetime):
local_copy[index] = value.timestamp()
elif index.endswith("_datetime") and isinstance(value, datetime):
local_copy[index] = value.isoformat()
else:
local_copy[index] = copy.deepcopy(value)
DictHelper(local_copy).to_json_file(self.info_file_instance.path)
return self
def clean(self) -> "InfoManager":
"""
Cleans the unneeded indexes.
"""
for index in [
"arguments",
"clean_list_file",
"clean_original",
"commit_autosave_message",
"last_test",
"list_name",
"stable",
]:
if index in self.content:
del self.content[index]
logging.debug(
"Deleted the %r index of the administration file, "
"it is not needed anymore.",
index,
)
return self
def create_missing_index(self) -> "InfoManager":
"""
Creates the missing indexes.
"""
default_datetime = datetime.utcnow() - timedelta(days=15)
indexes = {
"currently_under_test": False,
"custom_pyfunceble_config": {},
"days_until_next_test": 2,
"finish_datetime": default_datetime,
"finish_timestamp": default_datetime.timestamp(),
"last_download_datetime": default_datetime,
"last_download_timestamp": default_datetime.timestamp(),
"latest_part_finish_timestamp": default_datetime.timestamp(),
"latest_part_start_timestamp": default_datetime.timestamp(),
"latest_part_finish_datetime": default_datetime,
"latest_part_start_datetime": default_datetime,
"name": dead_hosts.launcher.defaults.paths.GIT_BASE_NAME,
"own_management": False,
"ping": [],
"raw_link": None,
"start_datetime": default_datetime,
"start_timestamp": default_datetime.timestamp(),
"live_update": True,
}
for index, value in indexes.items():
if index not in self.content:
self.content[index] = value
logging.debug(
"Created the %r index of the administration file, it was not found.",
index,
)
def update(self) -> "InfoManager":
"""
Updates and filters the new content.
"""
# pylint: disable=too-many-branches
self.content["name"] = dead_hosts.launcher.defaults.paths.GIT_BASE_NAME
logging.debug("Updated the `name` index of the administration file.")
to_delete = [
FileHelper(os.path.join(self.WORKSPACE_DIR, ".administrators")),
FileHelper(os.path.join(self.WORKSPACE_DIR, "update_me.py")),
FileHelper(os.path.join(self.WORKSPACE_DIR, "admin.py")),
]
if "list_name" in self.content:
to_delete.append(
FileHelper(os.path.join(self.WORKSPACE_DIR, self.content["list_name"]))
)
if "ping" in self.content:
local_ping_result = []
for username in self.content["ping"]:
if username.startswith("@"):
local_ping_result.append(username)
else:
local_ping_result.append(f"@{username}")
self.content["ping"] = local_ping_result
logging.debug(
"Updated the `ping` index of the administration file, "
"the format has to stay the same everywhere."
)
if (
"raw_link" in self.content
and isinstance(self.content["raw_link"], str)
and not self.content["raw_link"]
):
self.content["raw_link"] = None
logging.debug(
"Updated the `raw_link` index of the administration file, "
"empty string not accepted."
)
if "custom_pyfunceble_config" in self.content:
if self.content["custom_pyfunceble_config"]:
if not isinstance(self.content["custom_pyfunceble_config"], dict):
self.content["custom_pyfunceble_config"] = {}
else:
self.content["custom_pyfunceble_config"] = DictHelper(
self.content["custom_pyfunceble_config"]
).flatten()
else:
self.content["custom_pyfunceble_config"] = {}
logging.debug(
"Updated the `custom_pyfunceble_config` index of the "
"administration file, it should be a %r.",
dict,
)
if (
"custom_pyfunceble_config" in self.content
and self.content["custom_pyfunceble_config"]
and not isinstance(self.content["custom_pyfunceble_config"], dict)
):
self.content["custom_pyfunceble_config"] = {}
logging.debug(
"Updated the `custom_pyfunceble_config` index of the "
"administration file, it should be a %r.",
dict,
)
for index in ["currently_under_test"]:
if index in self.content and not isinstance(self.content[index], bool):
self.content[index] = bool(int(self.content[index]))
logging.debug(
"Updated the %r index of the administration file, "
"it should be a %r.",
index,
bool,
)
for index in [
"days_until_next_test",
"finish_timestamp",
"last_download_datetime"
"last_download_timestamp"
"lastest_part_finish_timestamp",
"lastest_part_start_timestamp",
"start_timestamp",
]:
if index in self.content and not isinstance(self.content[index], float):
self.content[index] = float(self.content[index])
logging.debug(
"Updated the %r index of the administration file, "
"it should be a %r.",
index,
float,
)
for index in [
"finish_timestamp",
"last_download_timestamp",
"lastest_part_finish_timestamp",
"lastest_part_start_timestamp",
"start_timestamp",
]:
if index in self.content and not isinstance(self.content[index], datetime):
self.content[index] = datetime.fromtimestamp(self.content[index])
logging.debug(
"Updated the %r index of the administration file, "
"the system understands %r only."
" (JSON => %s).",
index,
datetime,
dict,
)
for index in [
"finish_datetime",
"last_download_datetime",
"lastest_part_finish_datetime",
"lastest_part_start_datetime",
"start_datetime",
]:
if index in self.content:
if self.content[index] and not isinstance(
self.content[index], datetime
):
self.content[index] = datetime.fromisoformat(self.content[index])
logging.debug(
"Updated the %r index of the administration file, "
"the system understands %r only."
" (JSON => %r.",
index,
datetime,
dict,
)
else:
self.content[index] = datetime.fromtimestamp(0)
logging.debug(
"Set the %r index of the administration file, "
"it was not previously set.",
repr(index),
)
for file in to_delete:
if file.exists():
file.delete()
logging.debug(
"Deleted the %r file, it is not needed anymore.",
file.path,
)
def get_ping_for_commit(self) -> str:
"""
Provides the string to append in order to mention the users to ping.
"""
if "ping" in self.content:
return " ".join(self.content["ping"])
return ""
class ReadmeUpdater(UpdaterBase):
"""
Provides the updater of the README file.
"""
def __init__(self, info_manager: InfoManager) -> None:
self.destination_instance = FileHelper(
os.path.join(
info_manager.WORKSPACE_DIR,
dead_hosts.launcher.defaults.paths.README_FILENAME,
))
super().__init__(info_manager)
@property
def authorized(self) -> bool:
return self.destination_instance.exists()
def pre(self) -> "ReadmeUpdater":
logging.info("Started to update the content of %r!",
self.destination_instance.path)
return self
def post(self) -> "ReadmeUpdater":
logging.info("Finished to update the content of %r!",
self.destination_instance.path)
return self
def start(self) -> "ReadmeUpdater":
logging.info(
"Started to update the `About PyFunceble` section of %r",
self.destination_instance.path,
)
with importlib.resources.path("dead_hosts.launcher.data.docs",
"about_pyfunceble.md") as file_path:
updated_version = RegexHelper(
dead_hosts.launcher.defaults.markers.ABOUT_FUNCEBLE_REGEX
).replace_match(
self.destination_instance.read(),
FileHelper(str(file_path)).read() + "\n\n",
)
logging.info(
"Finished to update the `About PyFunceble` section of %r",
self.destination_instance.path,
)
logging.info(
"Started to update the `About Dead-Hosts` section of %r",
self.destination_instance.path,
)
with importlib.resources.path("dead_hosts.launcher.data.docs",
"about_dead_hosts.md") as file_path:
updated_version = RegexHelper(
dead_hosts.launcher.defaults.markers.ABOUT_DEAD_HOSTS_REGEX
).replace_match(
self.destination_instance.read(),
FileHelper(str(file_path)).read() + "\n\n",
)
logging.info(
"Finished to update the `About Dead-Hosts` section of %s",
self.destination_instance.path,
)
self.destination_instance.write(updated_version, overwrite=True)
return self
class Administration:
"""
Provides the administration interface.
"""
info_file_location: Optional[str] = None
info_file_helper: Optional[FileHelper] = None
__our_info: dict = dict()
def __init__(self) -> None:
self.info_file_location = outputs.ADMINISTRATION_DESTINATION
self.info_file_helper = FileHelper(self.info_file_location)
self.__our_info.update(self.load())
def __contains__(self, index: str) -> bool:
return index in self.__our_info
def __getitem__(self, index: str) -> Any:
if index in self.__our_info:
return self.__our_info[index]
raise KeyError(index)
def __getattr__(self, index: str) -> Any:
if index in self.__our_info:
return self.__our_info[index]
raise AttributeError(index)
def __setattr__(self, index: str, value: Any) -> None:
if self.__our_info:
self.__our_info[index] = value
else:
super().__setattr__(index, value)
def __delattr__(self, index: str) -> None:
if index in self.info:
del self.info[index]
def __del__(self) -> None:
self.save()
@property
def exists(self) -> bool:
"""
Checks if the administration file exists.
"""
return self.info_file_helper.exists()
@staticmethod
def convert_data_for_system(data: dict) -> dict:
"""
Given the content of the info file, we convert it into something our
system may understand.
.. warning::
This method also delete the keys that are scheduled or declared
for deletion
:param data:
The data to work with.
"""
result = dict()
for key, value in dict(data).items():
if key in infrastructure.ADMINISTRATION_INDEXES["delete"]:
continue
if key == "name":
result[key] = (
CommandHelper("basename $(git rev-parse --show-toplevel)"
).execute().strip())
elif key in infrastructure.ADMINISTRATION_INDEXES["bool"]:
result[key] = bool(int(value))
elif key in infrastructure.ADMINISTRATION_INDEXES["int"]:
result[key] = int(value)
elif key in infrastructure.ADMINISTRATION_INDEXES["dict"]:
result[key] = dict(value)
elif key in infrastructure.ADMINISTRATION_INDEXES["datetime"]:
try:
result[key] = datetime.fromisoformat(value)
except ValueError:
result[key] = datetime.utcnow()
elif key in infrastructure.ADMINISTRATION_INDEXES["epoch"]:
result[key] = datetime.fromtimestamp(value)
else:
result[key] = value
if "pyfunceble" not in result:
result["pyfunceble"] = {
"config": {},
}
for sanitize_type, keys in infrastructure.ADMINISTRATION_INDEXES.items(
):
if sanitize_type == "delete":
continue
for key in keys:
if key not in result:
if sanitize_type == "bool":
local_result = bool(None)
elif sanitize_type == "int":
local_result = 0
elif sanitize_type == "dict":
local_result = dict()
elif sanitize_type == "datetime":
local_result = datetime.utcnow() - timedelta(
days=365.25)
elif sanitize_type == "epoch":
local_result = datetime.utcnow() - timedelta(
days=365.25)
else:
local_result = None
result[key] = local_result
return result
@staticmethod
def convert_data_for_file(data: dict) -> dict:
"""
Given the content of the info file, we convert it into something that
can be saved into a JSON file.
.. warning::
This method also delete the keys that are scheduled or declared
for deletion
:param data:
The data to work with.
"""
result = dict()
for key, value in dict(data).items():
if key in infrastructure.ADMINISTRATION_INDEXES["delete"]:
continue
if key in infrastructure.ADMINISTRATION_INDEXES["bool"]:
result[key] = bool(value)
elif key in infrastructure.ADMINISTRATION_INDEXES["int"]:
result[key] = int(value)
elif key in infrastructure.ADMINISTRATION_INDEXES["datetime"]:
result[key] = value.isoformat()
elif key in infrastructure.ADMINISTRATION_INDEXES["epoch"]:
result[key] = value.timestamp()
else:
result[key] = value
return result
def load(self) -> dict:
"""
Loads and return the content of the administration file.
"""
if self.info_file_helper.read():
content = self.info_file_helper.read()
logging.debug("Administration file content:\n%s", content)
return self.convert_data_for_system(DictHelper().from_json(
content, return_dict_on_error=False))
return dict()
def save(self) -> None:
"""
Saves the loaded content of the administration file.
"""
if self.info_file_helper.exists():
DictHelper(self.convert_data_for_file(
self.__our_info)).to_json_file(self.info_file_location)
class Orchestration:
"""
Orchester the test launch.
"""
info_manager: Optional[InfoManager] = None
authorization_handler: Optional[Authorization] = None
origin_file: Optional[FileHelper] = None
output_file: Optional[FileHelper] = None
def __init__(self, save: bool = False, end: bool = False) -> None:
self.info_manager = InfoManager()
git_name = EnvironmentVariableHelper("GIT_NAME")
git_email = EnvironmentVariableHelper("GIT_EMAIL")
if git_email.exists() and "funilrys" in git_email.get_value():
git_name.set_value(dead_hosts.launcher.defaults.envs.GIT_NAME)
git_email.set_value(dead_hosts.launcher.defaults.envs.GIT_EMAIL)
EnvironmentVariableHelper("PYFUNCEBLE_OUTPUT_LOCATION").set_value(
self.info_manager.WORKSPACE_DIR)
EnvironmentVariableHelper("PYFUNCEBLE_CONFIG_DIR").set_value(
self.info_manager.PYFUNCEBLE_CONFIG_DIR)
self.authorization_handler = Authorization(self.info_manager)
self.origin_file = FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME,
))
self.output_file = FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
dead_hosts.launcher.defaults.paths.OUTPUT_FILENAME,
))
logging.info("Origin file: %r", self.origin_file.path)
logging.info("Output file: %r", self.output_file.path)
if not end and not save:
logging.info("Checking authorization to run.")
if self.authorization_handler.is_test_authorized():
execute_all_updater(self.info_manager)
PyFunceble.facility.ConfigLoader.start()
self.fetch_file_to_test()
self.run_test()
else:
logging.info(
"Not authorized to run a test until %r (current time) > %r",
datetime.now(),
self.authorization_handler.next_authorization_time,
)
sys.exit(0)
elif save:
self.run_autosave()
else:
self.run_end()
def fetch_file_to_test(self) -> "Orchestration":
"""
Provides the latest version of the file to test.
"""
if self.authorization_handler.is_refresh_authorized():
logging.info(
"We are authorized to refresh the lists! Let's do that.")
logging.info("Raw Link: %r", self.info_manager.raw_link)
if self.info_manager.raw_link:
DownloadHelper(self.info_manager.raw_link).download_text(
destination=self.origin_file.path)
logging.info(
"Could get the new version of the list. Updating the download time."
)
self.info_manager["last_download_datetime"] = datetime.utcnow()
self.info_manager[
"last_download_timestamp"] = self.info_manager[
"last_download_datetime"].timestamp()
elif self.origin_file.exists():
logging.info(
"Raw link not given or is empty. Let's work with %r.",
self.origin_file.path,
)
self.origin_file.read()
logging.info("Emptying the download time.")
self.info_manager[
"last_download_datetime"] = datetime.fromtimestamp(0)
self.info_manager[
"last_download_timestamp"] = self.info_manager[
"last_download_datetime"].timestamp()
else:
logging.info(f"Could not find {self.origin_file.path}. "
"Generating empty content to test.")
self.origin_file.write("# No content yet.", overwrite=True)
logging.info("Emptying the download time.")
self.info_manager[
"last_download_datetime"] = datetime.fromtimestamp(0)
self.info_manager[
"last_download_timestamp"] = self.info_manager[
"last_download_datetime"].timestamp()
logging.info("Updated %r.", self.origin_file.path)
return self
def run_test(self):
"""
Run a test of the input list.
"""
if not self.info_manager.currently_under_test:
self.info_manager["currently_under_test"] = True
self.info_manager["start_datetime"] = datetime.utcnow()
self.info_manager["start_timestamp"] = self.info_manager[
"start_datetime"].timestamp()
self.info_manager["finish_datetime"] = datetime.fromtimestamp(0)
self.info_manager["finish_timestamp"] = self.info_manager[
"finish_datetime"].timestamp()
self.info_manager["latest_part_start_datetime"] = datetime.utcnow()
self.info_manager["latest_part_start_timestamp"] = self.info_manager[
"latest_part_start_datetime"].timestamp()
self.info_manager[
"latest_part_finish_datetime"] = datetime.fromtimestamp(0)
self.info_manager["latest_part_finish_timestamp"] = self.info_manager[
"latest_part_finish_datetime"].timestamp()
logging.info("Updated all timestamps.")
logging.info("Starting PyFunceble %r ...", PyFunceble.__version__)
Command(f"PyFunceble -f {self.origin_file.path}").run_to_stdout()
if not dead_hosts.launcher.defaults.envs.GITHUB_TOKEN:
self.run_end()
def run_autosave(self):
"""
Run the autosave logic of the administration file.
.. warning::
This is just about the administration file not PyFunceble.
"""
self.info_manager["latest_part_finish_datetime"] = datetime.utcnow()
self.info_manager["latest_part_finish_timestamp"] = self.info_manager[
"latest_part_finish_datetime"].timestamp()
logging.info("Updated all timestamps.")
def run_end(self):
"""
Run the end logic.
"""
self.info_manager["currently_under_test"] = False
self.info_manager["latest_part_finish_datetime"] = datetime.utcnow()
self.info_manager["latest_part_finish_timestamp"] = self.info_manager[
"latest_part_finish_datetime"].timestamp()
self.info_manager["finish_datetime"] = self.info_manager[
"latest_part_finish_datetime"]
self.info_manager["finish_timestamp"] = self.info_manager[
"finish_datetime"].timestamp()
logging.info(
"Updated all timestamps and indexes that needed to be updated.")
pyfunceble_active_list = FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
"output",
dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME,
"domains",
"ACTIVE",
"list",
))
clean_list = [
"# File generated by the Dead-Hosts project with the help of PyFunceble.",
"# Dead-Hosts: https://github.com/dead-hosts",
"# PyFunceble: https://pyfunceble.github.io",
f"# Generation Time: {datetime.utcnow().isoformat()}",
]
logging.info(
f"PyFunceble ACTIVE list output: {pyfunceble_active_list.path}")
if pyfunceble_active_list.exists():
logging.info(
f"{pyfunceble_active_list.path} exists, getting and formatting its content."
)
self.output_file.write("\n".join(clean_list) + "\n\n",
overwrite=True)
with pyfunceble_active_list.open("r",
encoding="utf-8") as file_stream:
for line in file_stream:
if line.startswith("#"):
continue
self.output_file.write(line)
self.output_file.write("\n")
logging.info("Updated of the content of %r", self.output_file.path)
def migrate(self) -> "WhoisJSON2CSVMigrator":
"""
Provides the migration logic.
"""
file_helper = FileHelper(self.source_file)
if file_helper.exists():
self.dataset.set_authorized(True)
dataset = {
"subject": None,
"idna_subject": None,
"expiration_date": None,
"epoch": None,
}
delete_file = True
with file_helper.open("r", encoding="utf-8") as file_stream:
for line in file_stream:
if (self.continuous_integration and
self.continuous_integration.is_time_exceeded()):
delete_file = False
break
line = (line.strip().replace('"',
"").replace(",", "").replace(
"{",
"",
).replace("}", ""))
if ":" not in line:
continue
index, value = [x.strip() for x in line.split(":")]
if not value:
dataset["subject"], dataset["idna_subject"] = (
index,
domain2idna.domain2idna(index),
)
continue
if index == "epoch":
dataset["epoch"] = float(value)
elif index == "expiration_date":
dataset["expiration_date"] = value
elif index == "state":
PyFunceble.facility.Logger.debug(
"Decoded dataset:\n%r.", dataset)
self.dataset.update(dataset)
if self.print_action_to_stdout:
print_single_line()
PyFunceble.facility.Logger.info(
"Added %r into %r", dataset["idna_subject"],
self.dataset)
if delete_file:
file_helper.delete()
self.done = True
return self
def restore_from_backup(self) -> "DirectoryStructureRestoration":
"""
Restores or reconstruct the output directory.
"""
# pylint: disable=too-many-locals
PyFunceble.facility.Logger.info(
"Started restoration of the directory structure")
backup = self.get_backup_data()
base_dir = self.get_output_basedir()
dir_helper = DirectoryHelper()
file_helper = FileHelper()
if dir_helper.set_path(base_dir).exists():
for root, _, files in os.walk(dir_helper.path):
reduced_path = self.get_path_without_base_dir(root)
if reduced_path not in backup and root != reduced_path:
dir_helper.set_path(root).delete()
PyFunceble.facility.Logger.debug(
"Added %r into the list of directories to delete. "
"Reason: not found in own dataset.",
root,
)
continue
for directory, files in backup.items():
dir_helper.set_path(os.path.join(base_dir, directory)).create()
for file, dataset in files.items():
file_full_path = os.path.join(dir_helper.path, file)
if (file == ".gitignore"
and PyFunceble.cli.storage.STD_PARENT_DIRNAME
not in file_full_path):
to_delete = file_full_path
file_helper.set_path(to_delete).delete()
PyFunceble.facility.Logger.debug(
"(If exists) Deleted: %r. Reason: We are going to "
"replace it with .gitkeep",
to_delete,
)
file_full_path = file_full_path.replace(
".gitignore", ".gitkeep")
file_helper.set_path(file_full_path)
if not file_helper.exists():
file_helper.write(dataset["content"], overwrite=True)
PyFunceble.facility.Logger.info(
"Finished restoration of the directory structure")
return self
