def get_backup_data(self) -> dict:
"""
Provides the data which acts as a backup.
"""
result = {}
base_dir = self.get_output_basedir()
file_helper = FileHelper()
hash_helper = HashHelper()
for file in DirectoryHelper(base_dir).list_all_files():
file_helper.set_path(file)
reduced_path = self.get_path_without_base_dir(file)
directory, filename = reduced_path.rsplit(os.sep, 1)
file_hash = hash_helper.hash_file(file_helper.path)
dataset = {
filename: {
"content": file_helper.read(),
"hash": file_hash
}
}
if directory not in result:
result[directory] = dataset
else:
result[directory].update(dataset)
PyFunceble.facility.Logger.debug("Dir Structure to backup:\n%r",
result)
return result
def remove_removed(self) -> None:
"""
Removed the removed entries from all files to clean.
"""
file_helper = FileHelper()
for file in self.files_to_clean:
if not file_helper.set_path(file).exists():
continue
logging.info(
"Started to cleanup: %r",
file,
)
whitelist_core_tool(
output_file=file,
secondary_whitelist=self.whitelist_list.name,
use_official=False,
processes=os.cpu_count(),
).filter(file=file, already_formatted=True, standard_sort=True)
logging.info(
"Finished to cleanup: %r",
file,
)
def is_cloned(
) -> bool: # pragma: no cover ## Only used by 1 thing for dev.
"""
Checks if the local version is a cloned (from git) one.
"""
file_helper = FileHelper()
directory_helper = DirectoryHelper()
if not directory_helper.set_path(".git").exists():
return False
list_of_files = [
".coveragerc",
".gitignore",
"CODE_OF_CONDUCT.rst",
"CONTRIBUTING.rst",
"MANIFEST.in",
"README.rst",
"requirements.txt",
"setup.py",
"version.yaml",
]
list_of_dirs = ["docs", "PyFunceble", "tests", ".github"]
if not all(file_helper.set_path(x).exists() for x in list_of_files):
return False
if not all(
directory_helper.set_path(x).exists() for x in list_of_dirs):
return False
return True
def test_delete(self) -> None:
"""
Tests the method which let us delete a file.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
with open(file_helper.path, "w", encoding="utf-8") as file_stream:
file_stream.write("")
expected = True
actual = file_helper.exists()
self.assertEqual(expected, actual)
file_helper.delete()
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
def get_content(self) -> Optional[dict]:
"""
Provides the cached or the real contend of the dataset (after caching)
:raise FileNotFoundError:
When the declared file does not exists.
"""
if (
bool(self.STORAGE_INDEX)
and hasattr(PyFunceble.storage, self.STORAGE_INDEX)
and bool(getattr(PyFunceble.storage, self.STORAGE_INDEX))
):
return getattr(PyFunceble.storage, self.STORAGE_INDEX)
file_helper = FileHelper(self.source_file)
if not file_helper.exists() and bool(
self.DOWNLOADER
): # pragma: no cover ## This is just a safety endpoint.
self.DOWNLOADER.start()
if not file_helper.exists():
raise FileNotFoundError(file_helper.path)
content = DictHelper().from_json_file(
self.source_file, return_dict_on_error=False
)
setattr(PyFunceble.storage, self.STORAGE_INDEX, content)
return content
def __init__(self, info_manager: InfoManager) -> None:
self.destination_instance = FileHelper(
os.path.join(
info_manager.WORKSPACE_DIR,
dead_hosts.launcher.defaults.paths.README_FILENAME,
))
super().__init__(info_manager)
def start(self) -> "UpdaterBase":
"""
Starts the update process.
"""
if self.administration.raw_link:
logging.info("Started to download: %r",
self.administration.raw_link)
DownloadHelper(self.administration.raw_link).download_text(
destination=self.download_temp_file.name)
logging.info("Finished to download: %r",
self.administration.raw_link)
self.download_temp_file.seek(0)
logging.info("Started comparison of: %r", self.final_destination)
kept, removed, new = self.produce_diff()
logging.info("Finished comparison of: %r", self.final_destination)
to_write = kept.copy()
to_write.update(new)
try:
# Safety.
to_write.remove(None)
except KeyError:
pass
try:
# Safety.
to_write.remove("")
except KeyError:
pass
logging.info("Started to update: %r", self.final_destination)
FileHelper(self.final_destination).write("\n".join(sorted(to_write)) +
"\n",
overwrite=True)
logging.info("Finished to update: %r", self.final_destination)
if removed:
logging.info(
"Started to write our temporary whitelist list into: %r",
self.whitelist_list.name,
)
FileHelper(self.whitelist_list.name).write("\n".join(removed) +
"\n",
overwrite=True)
self.whitelist_list.seek(0)
logging.info(
"Finished to write our temporary whitelist list into: %r",
self.whitelist_list.name,
)
self.remove_removed()
def run_end(self):
"""
Run the end logic.
"""
self.info_manager["currently_under_test"] = False
self.info_manager["latest_part_finish_datetime"] = datetime.utcnow()
self.info_manager["latest_part_finish_timestamp"] = self.info_manager[
"latest_part_finish_datetime"].timestamp()
self.info_manager["finish_datetime"] = self.info_manager[
"latest_part_finish_datetime"]
self.info_manager["finish_timestamp"] = self.info_manager[
"finish_datetime"].timestamp()
logging.info(
"Updated all timestamps and indexes that needed to be updated.")
pyfunceble_active_list = FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
"output",
dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME,
"domains",
"ACTIVE",
"list",
))
clean_list = [
"# File generated by the Dead-Hosts project with the help of PyFunceble.",
"# Dead-Hosts: https://github.com/dead-hosts",
"# PyFunceble: https://pyfunceble.github.io",
f"# Generation Time: {datetime.utcnow().isoformat()}",
]
logging.info(
f"PyFunceble ACTIVE list output: {pyfunceble_active_list.path}")
if pyfunceble_active_list.exists():
logging.info(
f"{pyfunceble_active_list.path} exists, getting and formatting its content."
)
self.output_file.write("\n".join(clean_list) + "\n\n",
overwrite=True)
with pyfunceble_active_list.open("r",
encoding="utf-8") as file_stream:
for line in file_stream:
if line.startswith("#"):
continue
self.output_file.write(line)
self.output_file.write("\n")
logging.info("Updated of the content of %r", self.output_file.path)
def test_open(self) -> None:
"""
Tests the method which let us open the given file as we want.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
with file_helper.open("w") as file_stream:
file_stream.write("Hello, World!")
expected = True
actual = file_helper.exists()
self.assertEqual(expected, actual)
expected = "Hello, World!"
actual = file_helper.read()
self.assertEqual(expected, actual)
def __init__(self, save: bool = False, end: bool = False) -> None:
self.info_manager = InfoManager()
git_name = EnvironmentVariableHelper("GIT_NAME")
git_email = EnvironmentVariableHelper("GIT_EMAIL")
if git_email.exists() and "funilrys" in git_email.get_value():
git_name.set_value(dead_hosts.launcher.defaults.envs.GIT_NAME)
git_email.set_value(dead_hosts.launcher.defaults.envs.GIT_EMAIL)
EnvironmentVariableHelper("PYFUNCEBLE_OUTPUT_LOCATION").set_value(
self.info_manager.WORKSPACE_DIR)
EnvironmentVariableHelper("PYFUNCEBLE_CONFIG_DIR").set_value(
self.info_manager.PYFUNCEBLE_CONFIG_DIR)
self.authorization_handler = Authorization(self.info_manager)
self.origin_file = FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
dead_hosts.launcher.defaults.paths.ORIGIN_FILENAME,
))
self.output_file = FileHelper(
os.path.join(
self.info_manager.WORKSPACE_DIR,
dead_hosts.launcher.defaults.paths.OUTPUT_FILENAME,
))
logging.info("Origin file: %r", self.origin_file.path)
logging.info("Output file: %r", self.output_file.path)
if not end and not save:
logging.info("Checking authorization to run.")
if self.authorization_handler.is_test_authorized():
execute_all_updater(self.info_manager)
PyFunceble.facility.ConfigLoader.start()
self.fetch_file_to_test()
self.run_test()
else:
logging.info(
"Not authorized to run a test until %r (current time) > %r",
datetime.now(),
self.authorization_handler.next_authorization_time,
)
sys.exit(0)
elif save:
self.run_autosave()
else:
self.run_end()
def test_set_path_return(self) -> None:
"""
Tests the response from the method which let us set the path to work with.
"""
given = tempfile.NamedTemporaryFile()
file_helper = FileHelper()
actual = file_helper.set_path(given.name)
self.assertIsInstance(actual, FileHelper)
def test_set_path_not_str(self) -> None:
"""
Tests the method which let us set the path to work with for the case
that it's not a string.
"""
given = ["Hello", "World"]
file_helper = FileHelper()
self.assertRaises(TypeError, lambda: file_helper.set_path(given))
def get_dot_env_file(self) -> str:
"""
Provides the dotenv file to work with.
"""
file_helper = FileHelper()
for file in self.dotenv_locations:
if file_helper.set_path(file).exists():
return file_helper.path
return self.dotenv_locations[-1]
def fetch_dataset(self) -> "FilesystemCounter":
"""
Fetches the source file into the current instance.
"""
file_helper = FileHelper(self.source_file)
if file_helper.exists():
self.dataset = DictHelper().from_json_file(file_helper.path)
else:
self.dataset = copy.deepcopy(self.STD_DATASET)
return self
def test_is_empty(self) -> None:
"""
Tests the method which let us check if a file is empty.
"""
file_helper = FileHelper(tempfile.gettempdir())
file_helper.set_path(file_helper.join_path(secrets.token_hex(8)))
expected = False
actual = file_helper.exists()
self.assertEqual(expected, actual)
with open(file_helper.path, "w", encoding="utf-8") as file_stream:
file_stream.write("")
expected = True
actual = file_helper.is_empty()
self.assertEqual(expected, actual)
with open(file_helper.path, "w", encoding="utf-8") as file_stream:
file_stream.write("Hello, World!")
expected = False
actual = file_helper.is_empty()
self.assertEqual(expected, actual)
os.remove(file_helper.path)
def test_write(self) -> None:
"""
Tests the method which let us write a file.
"""
given = tempfile.NamedTemporaryFile(delete=False)
file_helper = FileHelper(given.name)
file_helper.write("Hello, World!")
given.seek(0)
expected = b"Hello, World!"
actual = given.read()
self.assertEqual(expected, actual)
file_helper.write("Hello, this is Funilrys!")
given.seek(0)
expected = b"Hello, World!Hello, this is Funilrys!"
actual = given.read()
self.assertEqual(expected, actual)
file_helper.write("Hello, World!", overwrite=True)
given.seek(0)
expected = b"Hello, World!"
actual = given.read()
self.assertEqual(expected, actual)
def mining_file_cleanup_target(
continuous_integration: ContinuousIntegrationBase,
) -> None:
"""
Provides the target for the cleanup of the mining file.
"""
migrator = MiningFileCleanupMigrator(print_action_to_stdout=True)
migrator.continuous_integration = continuous_integration
if FileHelper(migrator.source_file).exists():
print(
f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}"
f"Started deletion of {migrator.source_file!r}."
)
migrator.start()
if migrator.done:
print(
f"{colorama.Fore.GREEN}{colorama.Style.BRIGHT}"
f"Finished deletion of {migrator.source_file!r}."
)
else:
print(
f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}"
f"Unfinished deletion of {migrator.source_file!r}."
)
else:
PyFunceble.facility.Logger.info(
"Stopped hashes_file_cleanup_target. File does not exist."
)
def json2csv_inactive_target(
continuous_integration: ContinuousIntegrationBase,
) -> None:
"""
Provides the target for the inactive database migrator.
"""
migrator = InactiveJSON2CSVMigrator(print_action_to_stdout=True)
migrator.continuous_integration = continuous_integration
if FileHelper(migrator.source_file).exists():
print(
f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}"
"Started migration (json2csv) of the inactive dataset."
)
migrator.start()
if migrator.done:
print(
f"\n{colorama.Fore.GREEN}{colorama.Style.BRIGHT}"
"Finished migration (json2csv) of the inactive dataset."
)
else:
print(
f"\n{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}"
"Unfinished migration (json2csv) of the inactive dataset."
)
else:
PyFunceble.facility.Logger.info(
"Stopped json2csv_inactive_target. File does not exist."
)
def test_read_bytes(self) -> None:
"""
Tests the method which let us read (bytes) a file.
"""
given = tempfile.NamedTemporaryFile(delete=False)
file_helper = FileHelper(given.name)
file_helper.write("Hello, World!")
given.seek(0)
expected = b"Hello, World!"
actual = file_helper.read_bytes()
self.assertEqual(expected, actual)
def authorized(self) -> bool:
"""
Provides the authorization to launch.
"""
return not FileHelper(
os.path.join(outputs.CURRENT_DIRECTORY,
outputs.EXAMPLE_ADMINISTRATION_FILENAME)).exists()
def authorized(self) -> bool:
return any(
FileHelper(os.path.join(self.info_manager.WORKSPACE_DIR,
x)).exists()
for x in self.FILES_TO_REMOVE) or any(
DirectoryHelper(
os.path.join(self.info_manager.WORKSPACE_DIR, x)).exists()
for x in self.DIRS_TO_REMOVE)
def config_file_exist(
self,
) -> bool: # pragma: no cover ## Existance checker already tested.
"""
Checks if the config file exists.
"""
return FileHelper(self.path_to_config).exists()
def get_csv_writer(self) -> Tuple[csv.DictWriter, open]:
"""
Provides the standard and initiated CSV Dict writer along with the
file that was open with it.
"""
file_helper = FileHelper(self.source_file)
add_header = not file_helper.exists()
file_handler = file_helper.open("a+", newline="")
writer = csv.DictWriter(file_handler, fieldnames=self.FIELDS)
if add_header:
writer.writeheader()
return writer, file_handler
def update_clean_list(self) -> "UHBPyFuncebleSystemLauncher":
"""
Updates the content of the :code:`clean.list` file.
"""
input_file = FileHelper(outputs.ACTIVE_SUBJECTS_DESTINATION)
clean_file = FileHelper(outputs.CLEAN_DESTINATION)
if input_file.exists():
logging.info("Started generation of %r.", clean_file.path)
with input_file.open(
"r",
encoding="utf-8") as input_file_stream, clean_file.open(
"w", encoding="utf-8") as clean_file_stream:
for line in input_file_stream:
line = line.strip()
if not line or line.startswith("#") or "." not in line:
continue
if line.endswith("."):
line = line[:-1]
clean_file_stream.write("\n" + line)
logging.info("Finished generation of %r.", clean_file.path)
return self
def update_ip_list() -> "UHBPyFuncebleSystemLauncher":
"""
Updates the content of the :code:`ip.list` file.
"""
input_file = FileHelper(outputs.IP_SUBJECTS_DESTINATION)
ip_file = FileHelper(outputs.IP_DESTINATION)
if input_file.exists():
logging.info("Started generation of %r.", ip_file.path)
with input_file.open(
"r", encoding="utf-8") as input_file_stream, ip_file.open(
"w", encoding="utf-8") as ip_file_stream:
for line in input_file_stream:
if not line.strip() or line.startswith("#"):
continue
ip_file_stream.write("\n".join(line.split()[1:]) + "\n")
ip_file_stream.write("\n")
whitelist_core_tool(
output_file=ip_file.path,
use_official=True,
processes=os.cpu_count(),
).filter(file=ip_file.path,
already_formatted=True,
standard_sort=False)
logging.info("Finished generation of %r.", ip_file.path)
def produce_diff(self) -> None:
"""
Produce the difference from teh downloaded file.
"""
file_helper = FileHelper(self.final_destination)
new = set()
kept = set()
removed = set()
if file_helper.exists():
with file_helper.open("r", encoding="utf-8") as file_stream:
current_content = set(x.strip() for x in file_stream)
else:
current_content = set()
downloaded_empty = True
for line in self.download_temp_file:
if downloaded_empty:
downloaded_empty = False
line = line.strip()
if not line:
continue
kept_kept, new_new = self.__get_diff_data(
current_content, get_subjects_from_line(line, "availability"))
new.update(new_new)
kept.update(kept_kept)
if downloaded_empty:
kept = current_content
else:
compare_base = kept.copy()
compare_base.update(new)
removed = current_content - compare_base
self.download_temp_file.seek(0)
return kept, removed, new
def start(self) -> "OurInfrastructureUpdater":
file_helper = FileHelper()
dir_helper = DirectoryHelper()
for file in self.FILES_TO_REMOVE:
file_helper.set_path(
os.path.join(self.info_manager.WORKSPACE_DIR, file))
if file_helper.exists():
logging.info("Starting deletion of %r", file_helper.path)
file_helper.delete()
logging.info("Finished deletion of %r", file_helper.path)
for directory in self.DIRS_TO_REMOVE:
dir_helper.set_path(
os.path.join(self.info_manager.WORKSPACE_DIR, directory))
if dir_helper.exists():
logging.info("Starting deletion of %r", dir_helper.path)
dir_helper.delete()
logging.info("Finished deletion of %r", dir_helper.path)
return self
def __init__(self) -> None:
self.info_file_instance = FileHelper(self.INFO_FILE)
if self.info_file_instance.exists():
self.content = DictHelper().from_json_file(self.info_file_instance.path)
else:
self.content = {}
logging.debug("Administration file path: %r", self.INFO_FILE)
logging.debug(
"Administration file exists: %r", self.info_file_instance.exists()
)
logging.debug("Administration file content:\n%r", self.content)
self.update()
self.create_missing_index()
self.clean()
self.store()
def cleanup(self) -> "CSVContinueDataset":
"""
Deletes the source file (completely).
"""
if self.source_file:
FileHelper(self.source_file).delete()
PyFunceble.facility.Logger.debug("Deleted: %r", self.source_file)
return self
def csv_file_delete_source_column_target(
continuous_integration: ContinuousIntegrationBase,
) -> None:
"""
Provides the target for the deletion of the source column.
"""
migrator = InactiveDatasetDeleteSourceColumnMigrator(
print_action_to_stdout=True
)
migrator.continuous_integration = continuous_integration
file_helper = FileHelper(migrator.source_file)
if file_helper.exists():
with file_helper.open("r", encoding="utf-8") as file_stream:
first_line = next(file_stream)
if any(x in first_line for x in migrator.TO_DELETE):
print(
f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}"
"Started deletion of the 'source' column into "
f"{migrator.source_file!r}."
)
migrator.start()
if migrator.done:
print(
f"{colorama.Fore.GREEN}{colorama.Style.BRIGHT}"
"Finished deletion of the 'source' column into "
f"{migrator.source_file!r}."
)
else:
print(
f"{colorama.Fore.MAGENTA}{colorama.Style.BRIGHT}"
"unfinished deletion of the 'source' column into "
f"{migrator.source_file!r}."
)
else:
PyFunceble.facility.Logger.info(
"Stopped csv_file_delete_source_column_target. File does not exist."
)
