def delete_uneeded(self):
"""
Delete the directory which are not registered into our structure.
"""
# We get the structure we have to apply.
structure = self._get_structure()
# We get the list of key which is implicitly the list of directory we do not bave to delete.
list_of_key = list(structure.keys())
# We move to the content of the parent as we know that we are creating only one directory.
# Note: if one day we will have to create multiple directory, we will have to change
# the following.
structure = structure[list_of_key[0]]
# We also set the parent directory as we are going to construct its childen.
parent_path = list_of_key[0]
if not parent_path.endswith(PyFunceble.directory_separator):
parent_path += PyFunceble.directory_separator
for root, _, _ in PyFunceble.walk(parent_path):
# We loop through each directories of the parent path.
# We fix the path in order to avoid issues.
root = Directory(root).fix_path()
if root.replace(parent_path, "") not in structure:
# The currently read directory is not in our structure.
# We delete it.
PyFunceble.rmtree(root)
def __init__(self, path_to_config):
self.path_to_config = path_to_config
if path_to_config.endswith(directory_separator):
self.path_to_config += directory_separator
self.path_to_config += PyFunceble.CONFIGURATION_FILENAME
try:
self.load_config_file()
self.install_iana_config()
except FileNotFoundError:
if "PYFUNCEBLE_AUTO_CONFIGURATION" not in environ:
while True:
response = input(
"%s was not found.\n\
Install the default configuration in the current directory ? [y/n] " %
(Style.BRIGHT + path_to_config + Style.RESET_ALL))
if isinstance(response, str):
if response.lower() == "y":
self.install_production_config()
self.load_config_file()
self.install_iana_config()
break
elif response.lower() == "n":
raise Exception(
"Unable to find the configuration file.")
else:
self.install_production_config()
self.load_config_file()
self.install_iana_config()
for main_key in ["domains", "hosts", "splited"]:
PyFunceble.CONFIGURATION["outputs"][main_key][
"directory"] = Directory(PyFunceble.CONFIGURATION["outputs"]
[main_key]["directory"]).fix_path()
for main_key in ["http_analytic", "logs"]:
for key, value in PyFunceble.CONFIGURATION["outputs"][main_key][
"directories"].items():
PyFunceble.CONFIGURATION["outputs"][main_key]["directories"][
key] = Directory(value).fix_path()
PyFunceble.CONFIGURATION["outputs"]["main"] = Directory(
PyFunceble.CONFIGURATION["outputs"]["main"]).fix_path()
PyFunceble.STATUS.update(PyFunceble.CONFIGURATION["status"])
PyFunceble.OUTPUTS.update(PyFunceble.CONFIGURATION["outputs"])
PyFunceble.HTTP_CODE.update(PyFunceble.CONFIGURATION["http_codes"])
PyFunceble.LINKS.update(PyFunceble.CONFIGURATION["links"])
PyFunceble.CONFIGURATION.update({
"done": Fore.GREEN + "✔",
"error": Fore.RED + "✘"
})
def test_get_current(self):
"""
Tests the method which let us get the current directory.
"""
expected = getcwd()
actual = Directory.get_current()
self.assertEqual(expected, actual)
def test_get_current_with_end_sep(self):
"""
Tests the method which let us get the current directory for the
case that we want to have the directory separator at the end.
"""
expected = getcwd() + directory_separator
actual = Directory.get_current(with_end_sep=True)
self.assertEqual(expected, actual)
def test_fix_path(self):
"""
This method will test Directory.fix_path().
"""
expected = "hello" + PyFunceble.directory_separator + "world" + PyFunceble.directory_separator # pylint: disable=line-too-long
actual = Directory("/hello/world").fix_path()
self.assertEqual(expected, actual)
actual = Directory("\\hello\\world").fix_path()
self.assertEqual(expected, actual)
actual = Directory("hello\\world").fix_path()
self.assertEqual(expected, actual)
actual = Directory(r"hello\world").fix_path()
self.assertEqual(expected, actual)
actual = Directory(r"hello/world/").fix_path()
self.assertEqual(expected, actual)
def test_fix_path(self):
"""
Test Directory.fix_path().
"""
expected = (
"hello"
+ PyFunceble.directory_separator
+ "world"
+ PyFunceble.directory_separator
) # pylint: disable=line-too-long
actual = Directory("/hello/world").fix_path()
self.assertEqual(expected, actual)
actual = Directory("\\hello\\world").fix_path()
self.assertEqual(expected, actual)
actual = Directory("hello\\world").fix_path()
self.assertEqual(expected, actual)
actual = Directory(r"hello\world").fix_path()
self.assertEqual(expected, actual)
actual = Directory(r"hello/world/").fix_path()
self.assertEqual(expected, actual)
to_test = ["", None, []]
for element in to_test:
actual = Directory(element).fix_path()
self.assertEqual(element, actual)
def test_fix_path(self):
"""
Tests the method which allows us to fix
directory paths.
"""
expected = "hello" + directory_separator + "world" + directory_separator
actual = Directory("/hello/world").fix_path()
self.assertEqual(expected, actual)
actual = Directory("\\hello\\world").fix_path()
self.assertEqual(expected, actual)
actual = Directory("hello\\world").fix_path()
self.assertEqual(expected, actual)
actual = Directory(r"hello\world").fix_path()
self.assertEqual(expected, actual)
actual = Directory(r"hello/world/").fix_path()
self.assertEqual(expected, actual)
to_test = ["", None, []]
for element in to_test:
actual = Directory(element).fix_path()
self.assertEqual(element, actual)
def backup(self):
"""
Backup the developer state of `output/` in order to make it restorable
and portable for user.
"""
# We set the current output directory path.
output_path = self.base + PyFunceble.OUTPUTS["parent_directory"]
# We initiate the structure base.
result = {PyFunceble.OUTPUTS["parent_directory"]: {}}
for root, _, files in PyFunceble.walk(output_path):
# We loop through the current output directory structure.
# We get the currently read directory name.
directories = Directory(root.split(output_path)[1]).fix_path()
# We initiate a local variable which will get the structure of the subdirectory.
local_result = result[PyFunceble.OUTPUTS["parent_directory"]]
for file in files:
# We loop through the list of files.
# We construct the file path.
file_path = root + PyFunceble.directory_separator + file
# We get the hash of the file.
file_hash = Hash(file_path, "sha512", True).get()
# We convert the file content to a list.
lines_in_list = [line.rstrip("\n") for line in open(file_path)]
# We convert the file content into a more flat format.
# We use `@@@` as glue and implicitly replacement for `\n`.
formatted_content = "@@@".join(lines_in_list)
# We update the local result (and implicitly the global result)
# with the files and directory informations/structure.
local_result = local_result.setdefault(
directories,
{
file: {
"sha512": file_hash,
"content": formatted_content
}
},
)
# We finally save the directory structure into the production file.
Dict(result).to_json(self.base + "dir_structure_production.json")
def tests_create_and_delete(self):
"""
Tests the methods which let us create and delete
a directory.
"""
directory_test = "this_directory_is_a_ghost"
dir_instance = Directory(directory_test)
dir_instance.delete()
expected = False
actual = path.isdir(directory_test)
self.assertEqual(expected, actual)
dir_instance.create()
expected = True
actual = path.isdir(directory_test)
self.assertEqual(expected, actual)
dir_instance.delete()
def _update_structure_from_config(self, structure):
"""
Update the paths according to configs.
:param dict structure: The read structure.
"""
# We initiate a variable which will map what we have to replace `ouput` to.
# Indeed, as we allow the user to change directory names directly from the
# configuration, here we initiate what we have to replace `output/` with.
to_replace_base_map = {
"output/": PyFunceble.OUTPUTS["parent_directory"]
}
# We map the replacement of other directories.
to_replace_map = {
#########################################################################
# The following part is there for historical reason. #
#########################################################################
# We get the replacement of the HTTP_Analytic directory from the
# configuration file.
"HTTP_Analytic/":
PyFunceble.OUTPUTS["analytic"]["directories"]["parent"],
# We get the replacement of the HTTP_Analytic/ACTIVE directory from the
# configuration file.
"HTTP_Analytic/ACTIVE/":
PyFunceble.OUTPUTS["analytic"]["directories"]["parent"] +
PyFunceble.OUTPUTS["analytic"]["directories"]["up"],
"HTTP_Analytic/POTENTIALLY_ACTIVE/":
PyFunceble.OUTPUTS["analytic"]["directories"]["parent"] +
PyFunceble.OUTPUTS["analytic"]["directories"]["potentially_up"],
# We get the replacement of the HTTP_Analytic/POTENTIALLY_INACTIVE directory
# from the configuration file.
"HTTP_Analytic/POTENTIALLY_INACTIVE/":
PyFunceble.OUTPUTS["analytic"]["directories"]["parent"] +
PyFunceble.OUTPUTS["analytic"]["directories"]["potentially_down"],
#########################################################################
# The previous part is there for historical reason. #
#########################################################################
# We get the replacement of the Analytic directory from the
# configuration file.
"Analytic/":
PyFunceble.OUTPUTS["analytic"]["directories"]["parent"],
# We get the replacement of the Analytic/ACTIVE directory from the
# configuration file.
"Analytic/ACTIVE/":
PyFunceble.OUTPUTS["analytic"]["directories"]["parent"] +
PyFunceble.OUTPUTS["analytic"]["directories"]["up"],
"Analytic/POTENTIALLY_ACTIVE/":
PyFunceble.OUTPUTS["analytic"]["directories"]["parent"] +
PyFunceble.OUTPUTS["analytic"]["directories"]["potentially_up"],
# We get the replacement of the Analytic/POTENTIALLY_INACTIVE directory
# from the configuration file.
"Analytic/POTENTIALLY_INACTIVE/":
PyFunceble.OUTPUTS["analytic"]["directories"]["parent"] +
PyFunceble.OUTPUTS["analytic"]["directories"]["potentially_down"],
# We get the replacement of the Analytic/SUSPICIOUS directory
# from the configuration file.
"Analytic/SUSPICIOUS/":
PyFunceble.OUTPUTS["analytic"]["directories"]["parent"] +
PyFunceble.OUTPUTS["analytic"]["directories"]["suspicious"],
# We get the replacement of the complements directory from the
# configuration file.
"complements/":
PyFunceble.OUTPUTS["complements"]["directory"],
# We get the replacement of the complements/ACTIVE directory from the
# configuration file.
"complements/ACTIVE/":
PyFunceble.OUTPUTS["complements"]["directory"] +
PyFunceble.STATUS["official"]["up"],
# We get the replacement of the complements/INACTIVE directory from the
# configuration file.
"complements/INACTIVE/":
PyFunceble.OUTPUTS["complements"]["directory"] +
PyFunceble.STATUS["official"]["down"],
# We get the replacement of the complements/INVALID directory from the
# configuration file.
"complements/INVALID/":
PyFunceble.OUTPUTS["complements"]["directory"] +
PyFunceble.STATUS["official"]["invalid"],
# We get the replacement of the complements/VALID directory from the
# configuration file.
"complements/VALID/":
PyFunceble.OUTPUTS["complements"]["directory"] +
PyFunceble.STATUS["official"]["valid"],
# We get the replacement of the domains directory from the
# configuration file.
"domains/":
PyFunceble.OUTPUTS["domains"]["directory"],
# We get the replacement of the domains/ACTIVE directory from the
# configuration file.
"domains/ACTIVE/":
PyFunceble.OUTPUTS["domains"]["directory"] +
PyFunceble.STATUS["official"]["up"],
# We get the replacement of the domains/INACTIVE directory from the
# configuration file.
"domains/INACTIVE/":
PyFunceble.OUTPUTS["domains"]["directory"] +
PyFunceble.STATUS["official"]["down"],
# We get the replacement of the domains/INVALID directory from the
# configuration file.
"domains/INVALID/":
PyFunceble.OUTPUTS["domains"]["directory"] +
PyFunceble.STATUS["official"]["invalid"],
# We get the replacement of the domains/VALID directory from the
# configuration file.
"domains/VALID/":
PyFunceble.OUTPUTS["domains"]["directory"] +
PyFunceble.STATUS["official"]["valid"],
# We get the replacement of the hosts directory from the
# configuration file.
"hosts/":
PyFunceble.OUTPUTS["hosts"]["directory"],
# We get the replacement of the hosts/ACTIVE directory from the
# configuration file.
"hosts/ACTIVE/":
PyFunceble.OUTPUTS["hosts"]["directory"] +
PyFunceble.STATUS["official"]["up"],
# We get the replacement of the hosts/INACTIVE directory from the
# configuration file.
"hosts/INACTIVE/":
PyFunceble.OUTPUTS["hosts"]["directory"] +
PyFunceble.STATUS["official"]["down"],
# We get the replacement of the hosts/INVALID directory from the
# configuration file.
"hosts/INVALID/":
PyFunceble.OUTPUTS["hosts"]["directory"] +
PyFunceble.STATUS["official"]["invalid"],
# We get the replacement of the hosts/VALID directory from the
# configuration file.
"hosts/VALID/":
PyFunceble.OUTPUTS["hosts"]["directory"] +
PyFunceble.STATUS["official"]["valid"],
# We get the replacement of the json directory from the
# configuration file.
"json/":
PyFunceble.OUTPUTS["json"]["directory"],
# We get the replacement of the json/ACTIVE directory from the
# configuration file.
"json/ACTIVE/":
PyFunceble.OUTPUTS["json"]["directory"] +
PyFunceble.STATUS["official"]["up"],
# We get the replacement of the json/INACTIVE directory from the
# configuration file.
"json/INACTIVE/":
PyFunceble.OUTPUTS["json"]["directory"] +
PyFunceble.STATUS["official"]["down"],
# We get the replacement of the json/INVALID directory from the
# configuration file.
"json/INVALID/":
PyFunceble.OUTPUTS["json"]["directory"] +
PyFunceble.STATUS["official"]["invalid"],
# We get the replacement of the json/VALID directory from the
# configuration file.
"json/VALID/":
PyFunceble.OUTPUTS["json"]["directory"] +
PyFunceble.STATUS["official"]["valid"],
# We get the replacement of the logs directory from the
# configuration file.
"logs/":
PyFunceble.OUTPUTS["logs"]["directories"]["parent"],
# We get the replacement of the logs/percentage directory from the
# configuration file.
"logs/percentage/":
PyFunceble.OUTPUTS["logs"]["directories"]["parent"] +
PyFunceble.OUTPUTS["logs"]["directories"]["percentage"],
# We get the replacement of the splited directory from the
# configuration file.
"splited/":
PyFunceble.OUTPUTS["splited"]["directory"],
}
# We initiate the variable which will be used for the structure
# update.
to_replace = {}
for mapped, declared in to_replace_map.items():
# We loop through the declared mad.
# We fix the path of the declared.
declared = Directory(declared).fix_path()
# print('dec', declared, 'map', mapped)
# And we update our data.
to_replace.update({mapped: declared})
to_replace_base = {}
for mapped, declared in to_replace_base_map.items():
# We loop through the declared mad.
# We fix the path of the declared.
declared = Directory(declared).fix_path()
# And we update our data.
to_replace_base.update({mapped: declared})
# We perform the replacement of the base directory.
structure = Dict(structure).rename_key(to_replace_base)
# We perform the replacement of every subdirectories.
structure[PyFunceble.OUTPUTS["parent_directory"]] = Dict(structure[
PyFunceble.OUTPUTS["parent_directory"]]).rename_key(to_replace)
try:
# We try to save the structure into the right path.
Dict(structure).to_json(self.structure)
except FileNotFoundError:
# But if we get a FileNotFoundError exception,
# We create the directory where the directory structure should be saved.
PyFunceble.mkdir(
PyFunceble.directory_separator.join(
self.structure.split(PyFunceble.directory_separator)[:-1]))
# And we retry to save the structure into the right path.
Dict(structure).to_json(self.structure)
# We finaly return the new structure in case it's needed for other logic.
return structure
def __init__(self, path_to_config):
# We initiate 2 variables:
# * One with the path to the config file
# * The second one is the path to the default configuration file which is
# used only if the first one is not found.
self.path_to_config, self.path_to_default_config = self._set_path_to_configs(
path_to_config)
try:
# We try to load the configuration.
self._load_config_file()
except FileNotFoundError:
# We got a FileNotFoundError
if "PYFUNCEBLE_AUTO_CONFIGURATION" not in PyFunceble.environ:
# `PYFUNCEBLE_AUTO_CONFIGURATION` is not into the environnements variables.
while True:
# We infinitly loop until we get a reponse which is `y|Y` or `n|N`.
# We ask the user if we should install and load the default configuration.
response = input(
"%s was not found.\n\
Install and load the default configuration at the mentioned location? [y/n] " %
(PyFunceble.Style.BRIGHT + self.path_to_config +
PyFunceble.Style.RESET_ALL))
if isinstance(response, str):
# The response is a string
if response.lower() == "y":
# The response is a `y` or `Y`.
# We install the production configuration.
self._install_production_config()
# We load the installed configuration.
self._load_config_file()
# And we break the loop as we got a satisfied response.
break
elif response.lower() == "n":
# The response is a `n` or `N`.
# We inform the user that something went wrong.
raise Exception(
"Unable to find the configuration file.")
else:
# `PYFUNCEBLE_AUTO_CONFIGURATION` is not into the environnements variables.
# We install the production configuration.
self._install_production_config()
# We load the installed configuration.
self._load_config_file()
for main_key in ["domains", "hosts", "splited", "json"]:
# We loop through the key which contain paths under the `outputs` index.
# And we fix the path.
# Which means: If they do not end with the directory separator, we append
# it to the end.
PyFunceble.CONFIGURATION["outputs"][main_key][
"directory"] = Directory(PyFunceble.CONFIGURATION["outputs"]
[main_key]["directory"]).fix_path()
for main_key in ["analytic", "logs"]:
# We loop through the key which are more deeper under the `outputs` index.
for key, value in PyFunceble.CONFIGURATION["outputs"][main_key][
"directories"].items():
# We loop through the more deeper indexes.
# And we fix the path.
# Which means: If they do not end with the directory separator, we append
# it to the end.
PyFunceble.CONFIGURATION["outputs"][main_key]["directories"][
key] = Directory(value).fix_path()
# We fix the path.
# Which means: If they do not end with the directory separator, we append
# it to the end.
PyFunceble.CONFIGURATION["outputs"]["parent_directory"] = Directory(
PyFunceble.CONFIGURATION["outputs"]
["parent_directory"]).fix_path()
# We update the STATUS variable with the status from the configuration.
PyFunceble.STATUS.update(PyFunceble.CONFIGURATION["status"])
# We update the OUTPUTS variable with the outputs from the configuration.
PyFunceble.OUTPUTS.update(PyFunceble.CONFIGURATION["outputs"])
# We update the HTTP_CODE variable with the http_codes from the configuration.
PyFunceble.HTTP_CODE.update(PyFunceble.CONFIGURATION["http_codes"])
# We update the LINKS variable with the links from the configuration.
PyFunceble.LINKS.update(PyFunceble.CONFIGURATION["links"])
# Those 2 strings are used to say if something like the cleaning went right (done)
# or wrong (error).
PyFunceble.INTERN.update({
"done": PyFunceble.Fore.GREEN + "✔",
"error": PyFunceble.Fore.RED + "✘"
})
# We load the PSL database.
PublicSuffix().load()
# We load the IANA database.
IANA().load()