def _load_config_file(self):
"""
Load .PyFunceble.yaml into the system.
"""
try:
# We try to load the configuration file.
PyFunceble.CONFIGURATION.update(
Dict.from_yaml(File(self.path_to_config).read()))
# We install the latest iana configuration file.
self._install_iana_config()
# We install the latest public suffix configuration file.
self._install_psl_config()
# We install the latest directory structure file.
self._install_directory_structure_file()
except FileNotFoundError as exception:
# But if the configuration file is not found.
if PyFunceble.path.isfile(self.path_to_default_config):
# The `DEFAULT_CONFIGURATION_FILENAME` file exists.
# We copy it as the configuration file.
File(self.path_to_default_config).copy(self.path_to_config)
# And we load the configuration file as it does exist (yet).
self._load_config_file()
else:
# The `DEFAULT_CONFIGURATION_FILENAME` file does not exists.
# We raise the exception we were handling.
raise exception
def test_remove(self):
"""
Test the deletion subsystem.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
PyFunceble.INTERN["mined"] = self.excepted_content
PyFunceble.INTERN["to_test_type"] = "domain"
PyFunceble.INTERN["to_test"] = "www.google.com"
expected = {
PyFunceble.INTERN["file_to_test"]: {
"myètherwället.com": ["www.facebook.com"]
}
}
Mining().remove()
self.assertEqual(expected, PyFunceble.INTERN["mined"])
del PyFunceble.INTERN["mined"]
del PyFunceble.INTERN["to_test"]
del PyFunceble.INTERN["to_test_type"]
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
def test_list_of_mined(self):
"""
Test Mining.list_of_mined
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
self.mining.database = self.excepted_content
expected = [
("myètherwället.com", "www.google.com"),
("myètherwället.com", "www.facebook.com"),
]
self.assertEqual(expected, self.mining.list_of_mined())
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
def test_save(self):
"""
Test the saving system.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
self.mining.database = self.excepted_content
self.mining.save()
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(
self.excepted_content, Dict().from_json(File(self.file).read())
)
File(self.file).delete()
actual = PyFunceble.path.isfile(self.file)
expected = False
self.assertEqual(expected, actual)
def test_add(self):
"""
Test the addition subsystem.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
expected = {
self.file_to_test: {"www.google.com": ["facebook.com", "www.facebook.com"]}
}
self.mining["www.google.com"] = ["facebook.com", "www.facebook.com"]
self.assertEqual(expected, self.mining.database)
self.mining["www.google.com"] = ["github.com"]
expected[self.file_to_test]["www.google.com"].append("github.com")
self.assertEqual(expected, self.mining.database)
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_remove(self):
"""
Test the deletion subsystem.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
expected = {
self.file_to_test: {
"myètherwället.com": ["www.facebook.com", "facebook.com"],
"example.org": ["facebook.com"],
}
}
self.mining.database = {self.file_to_test: {}}
self.mining["myètherwället.com"] = ["www.facebook.com", "facebook.com"]
self.mining["example.org"] = ["www.facebook.com", "facebook.com"]
self.mining.remove("example.org", "www.facebook.com")
self.assertEqual(expected, self.mining.database)
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
def test_retrieve_file_not_exist(self):
"""
Test the case that we want to retrieve a file that does not exist.
"""
File(self.file).delete()
actual = PyFunceble.path.isfile(self.file)
expected = False
self.assertEqual(expected, actual)
Mining()._retrieve()
excepted = {}
self.assertEqual(excepted, PyFunceble.INTERN["mined"])
PyFunceble.INTERN["mined"] = {}
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def before_header(self):
"""
Print informations about PyFunceble and the date of generation of a file
into a given path, if doesn't exist.
"""
if not PyFunceble.CONFIGURATION[
"no_files"] and self.output and not path.isfile(self.output):
link = ("# File generated with %s\n" % PyFunceble.LINKS["repo"])
date_of_generation = ("# Date of generation: %s \n\n" %
PyFunceble.CURRENT_TIME)
if self.template in [
"Generic_File",
PyFunceble.STATUS["official"]["up"],
PyFunceble.STATUS["official"]["down"],
PyFunceble.STATUS["official"]["invalid"],
"Less",
]:
header = self._header_constructor(self.currently_used_header,
None)[0] + "\n"
try:
File(self.output).write(link + date_of_generation + header)
except UnboundLocalError:
File(self.output).write(link + date_of_generation)
def test_retrieve_file_exist(self):
"""
Test the case that we want to retrieve a file that exist.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
PyFunceble.INTERN["to_test_type"] = "domain"
Dict(self.excepted_content).to_json(self.file)
Mining()._retrieve()
self.assertEqual(self.excepted_content, PyFunceble.INTERN["mined"])
del PyFunceble.INTERN["mined"]
del PyFunceble.INTERN["to_test_type"]
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def _get_structure(self):
"""
This method return the structure we are goinng to work with.
"""
structure_file = ""
req = ""
if path.isfile(self.structure):
structure_file = self.structure
elif path.isfile(self.base + "dir_structure_production.json"):
structure_file = self.base + "dir_structure_production.json"
else:
if "dev" not in PyFunceble.VERSION:
req = requests.get(PyFunceble.LINKS["dir_structure"])
else:
req = requests.get(PyFunceble.LINKS["dir_structure"].replace(
"master", "dev"))
if structure_file.endswith("_production.json"):
structure = Dict().from_json(File(structure_file).read())
return self._update_structure_from_config(structure)
elif structure_file.endswith(".json"):
return Dict().from_json(File(structure_file).read())
return self._update_structure_from_config(Dict().from_json(req.text))
def test_write_overwrite_delete(self):
"""
This metthod test File.write() along with File.write() for the case that
we want to overwrite the content of a file.
"""
expected = "Hello, World! I'm domain2idna"
File("hi").write(expected)
with open("hi") as file:
actual = file.read()
self.assertEqual(expected, actual)
expected = "Hello, World! Python is great, you should consider learning it!"
File("hi").write(expected, overwrite=True)
with open("hi") as file:
actual = file.read()
self.assertEqual(expected, actual)
expected = False
File("hi").delete()
actual = PyFunceble.path.isfile("hi")
self.assertEqual(expected, actual)
def test_to_yaml(self):
"""
This method will test Dict().to_yaml.
"""
file_to_read = "this_yaml_is_a_ghost.yaml"
expected = False
actual = PyFunceble.path.isfile(file_to_read)
self.assertEqual(expected, actual)
to_write = {
"hello": ["This is PyFunceble!", "Uhh!"],
"world": "Fun Ilrys"
}
expected = """hello: [This is PyFunceble!, Uhh!]
world: Fun Ilrys
"""
Dict(to_write).to_yaml(file_to_read)
actual = File(file_to_read).read()
self.assertEqual(expected, actual)
File(file_to_read).delete()
expected = False
actual = PyFunceble.path.isfile(file_to_read)
self.assertEqual(expected, actual)
def test_get_specific_algo(self):
"""
This method will test Hash.get() for the case that we want a specifig
algorithm.
"""
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
File(self.file).write("\n".join(self.data_to_write))
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
expected = self.expected_hashed["sha512"]
actual = Hash(self.file, algorithm="sha512", only_hash=True).get()
self.assertEqual(expected, actual)
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_hash_data(self):
"""
This method will test Hash.hash_data().
"""
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
File(self.file).write("\n".join(self.data_to_write))
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
for algo, result in self.expected_hashed.items():
self.assertEqual(
result,
Hash(self.file).hash_data(algo),
msg="%s did not passed the test" % repr(algo),
)
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_backup(self):
"""
Test the backup system.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
PyFunceble.INTERN["mined"] = self.excepted_content
Mining()._backup()
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(self.excepted_content,
Dict().from_json(File(self.file).read()))
del PyFunceble.INTERN["mined"]
File(self.file).delete()
actual = PyFunceble.path.isfile(self.file)
expected = False
self.assertEqual(expected, actual)
def _before_header(self):
"""
Print informations about PyFunceble and the date of generation of a file
into a given path, if doesn't exist.
"""
if (
not PyFunceble.CONFIGURATION["no_files"]
and self.output
and not PyFunceble.path.isfile(self.output)
):
# * We are allowed to generate files.
# and
# * And output is given.
# and
# * The given output does not exist.
# We initiate the information about what generated the file.
link = "# File generated by %s\n" % PyFunceble.LINKS["repo"]
# We initiate the information about the generation date of this file.
date_of_generation = (
"# Date of generation: %s \n\n" % PyFunceble.CURRENT_TIME
)
# We initiate a variable which will save the list of
# templates which have to meet in order to write the before
# header informations.
authorized_templates = [
"Generic_File",
PyFunceble.STATUS["official"]["up"],
PyFunceble.STATUS["official"]["down"],
PyFunceble.STATUS["official"]["invalid"],
PyFunceble.STATUS["official"]["valid"],
"Less",
]
if self.template in authorized_templates:
# The current header is in our list of authorized templated.
# We get the header.
header = (
self._header_constructor(self.currently_used_header, None)[0] + "\n"
)
try:
# We try to print the link, the date of generation and the header in the
# given file.
File(self.output).write(link + date_of_generation + header)
except UnboundLocalError:
# We don't have any header.
# We print the link and the date in the given file.
File(self.output).write(link + date_of_generation)
def setUp(self):
"""
Setups everything needed for the tests.
"""
self.file = "this_file_is_a_ghost"
self.file_instance = File(self.file)
self.file_instance_2 = File(self.file + "_2")
self.file_instance.delete()
self.file_instance_2.delete()
def test_before_header(self, header_constructor_patch):
"""
Test the functionability of Prints().before_header()
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
expected = """# File generated by {0} (v{1}) / {2}
# Date of generation: {3}
""".format(
PyFunceble.NAME,
PyFunceble.VERSION.split()[0],
PyFunceble.LINKS["repo"],
PyFunceble.CURRENT_TIME + " ",
)
Prints(None, None, output_file=self.file, only_on_file=False)._before_header()
self.assertEqual(expected, File(self.file).read())
# Test of the case that we have a Generic_File template
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
expected = """# File generated by {0} (v{1}) / {2}
# Date of generation: {3}
Hello World!
""".format(
PyFunceble.NAME,
PyFunceble.VERSION.split()[0],
PyFunceble.LINKS["repo"],
PyFunceble.CURRENT_TIME + " ",
)
header_constructor_patch.return_value = ["Hello World!"]
Prints(
None, "Generic_File", output_file=self.file, only_on_file=False
)._before_header()
self.assertEqual(expected, File(self.file).read())
def __init__(self):
if PyFunceble.CONFIGURATION["auto_continue"]:
self.autocontinue_log_file = PyFunceble.CURRENT_DIRECTORY + PyFunceble.OUTPUTS[
"parent_directory"] + PyFunceble.OUTPUTS["logs"]["filenames"][
"auto_continue"]
if path.isfile(self.autocontinue_log_file):
self.backup_content = Dict().from_json(
File(self.autocontinue_log_file).read())
else:
self.backup_content = {}
File(self.autocontinue_log_file).write(str(
self.backup_content))
def restore(self):
"""
Restore the 'output/' directory structure based on the `dir_structure.json` file.
"""
structure = self._get_structure()
list_of_key = list(structure.keys())
structure = structure[list_of_key[0]]
parent_path = list_of_key[0] + directory_separator
for directory in structure:
base = self.base + parent_path + directory + directory_separator
self._create_directory(base)
for file in structure[directory]:
file_path = base + file
content_to_write = structure[directory][file]["content"]
online_sha = structure[directory][file]["sha512"]
content_to_write = Regex(content_to_write,
"@@@",
escape=True,
replace_with="\\n").replace()
git_to_keep = file_path.replace("gitignore", "keep")
keep_to_git = file_path.replace("keep", "gitignore")
if self._restore_replace():
if path.isfile(file_path) and Hash(
file_path, "sha512", True).get() == online_sha:
rename(file_path, git_to_keep)
write = False
else:
File(file_path).delete()
file_path = git_to_keep
write = True
else:
if path.isfile(keep_to_git) and Hash(
file_path, "sha512", True).get() == online_sha:
rename(file_path, keep_to_git)
write = False
else:
File(keep_to_git).delete()
file_path = keep_to_git
write = True
if write:
File(file_path).write(content_to_write + "\n", True)
def test_copy(self):
"""
Test File.copy().
"""
file_to_write = "hello_world"
copy_destination = "world_hello"
expected = False
actual = PyFunceble.path.isfile(file_to_write)
self.assertEqual(expected, actual)
expected = "Hello, World! Python is great, you should consider learning it!"
File(file_to_write).write(expected)
with open(file_to_write) as file:
actual = file.read()
self.assertEqual(expected, actual)
expected = False
actual = PyFunceble.path.isfile(copy_destination)
self.assertEqual(expected, actual)
File(file_to_write).copy(copy_destination)
expected = True
actual = PyFunceble.path.isfile(copy_destination)
self.assertEqual(expected, actual)
expected = "Hello, World! Python is great, you should consider learning it!"
with open(copy_destination) as file:
actual = file.read()
self.assertEqual(expected, actual)
File(copy_destination).delete()
File(file_to_write).delete()
expected = False
actual = PyFunceble.path.isfile(copy_destination)
self.assertEqual(expected, actual)
actual = PyFunceble.path.isfile(file_to_write)
self.assertEqual(expected, actual)
def _update_docs(self, file_to_update):
"""
Update the given documentation file or :code:`README.rst` so that
it always gives branch related URL and informations.
.. note::
This only apply to :code:`dev` and :code:`master` branch.
:param file_to_update: The file to update.
:type file_to_update: str
"""
if self.is_dev_version():
# The current version is the dev version.
# We map what we have to replace.
# Format: {match:replacement}
regexes = {
"/%s/" % "dev": r"\/%s\/" % "master",
"=%s" % "dev": "=%s" % "master",
}
elif self.is_master_version():
# The current version is the master version.
# We map what we have to replace.
regexes = {
"/%s/" % "master": r"\/%s\/" % "dev",
"=%s" % "master": "=%s" % "dev",
}
else:
# The current version is not the master nor the dev version.
# We raise an exception as the branch we are currently is not meaned
# for production.
raise Exception("Please switch to `dev` or `master` branch.")
# We get the content of the file to fix.
to_update = File(file_to_update).read()
for replacement, regex in regexes.items():
# We loop through reach element of the map.
# We process the replacement.
to_update = Regex(to_update, regex,
replace_with=replacement).replace()
# We finally overwrite the file to fix with the filtered.
# content.
File(file_to_update).write(to_update, overwrite=True)
def _update_setup_py(self):
"""
Update :code:`setup.py` so that it always have the right name.
"""
# We initiate the path to the file we have to filter.
setup_py_path = PyFunceble.CURRENT_DIRECTORY + "setup.py"
if self.is_dev_version():
# The current version is the `dev` version.
# We map what we have to replace.
# Format: {match:replacement}
regexes = {
'name="PyFunceble-dev"': r'name=".*"',
'"Development Status :: 4 - Beta"':
r'"Development\sStatus\s::.*"',
}
elif self.is_master_version():
# The current version is the `dev` version.
# We map what we have to replace.
regexes = {
'name="PyFunceble"':
r'name=".*"',
'"Development Status :: 5 - Production/Stable"':
r'"Development\sStatus\s::.*"',
}
else:
# The current version is not the `dev` nor the `master` version.
# We raise an exception to the user, the current branch is not meant for
# production.
raise Exception("Please switch to `dev` or `master` branch.")
# We get the file content.
to_update = File(setup_py_path).read()
for replacement, regex in regexes.items():
# We loop through our map.
# And we process the replacement.
to_update = Regex(to_update, regex,
replace_with=replacement).replace()
# We finally replace the content of the file with the filtered
# version.
File(setup_py_path).write(to_update, overwrite=True)
def test_read_delete(self):
"""
Test File.read() along with helpers.File.delete.
"""
expected = "Hello, World! This has been written by Fun Ilrys."
File("hi").write(expected)
actual = File("hi").read()
self.assertEqual(expected, actual)
expected = False
File("hi").delete()
actual = PyFunceble.path.isfile("hi")
self.assertEqual(expected, actual)
def test_add_file_path_not_present(self): # pylint: disable=invalid-name
"""
Test Inactive.add() for the case that the path is not
present into the Inactive.
"""
self.test_file_not_exist()
timestamp = str(Inactive()._timestamp())
PyFunceble.INTERN["to_test"] = "hello.world"
expected = {
PyFunceble.INTERN["file_to_test"]: {
timestamp: ["hello.world"]
}
}
Inactive().add()
actual = Dict().from_json(File(self.file).read())
self.assertEqual(expected, actual)
del PyFunceble.INTERN["to_test"]
del PyFunceble.INTERN["inactive_db"]
self.test_file_not_exist()
def test_colorify(self):
"""
Test Prints().colorify(). In other word, it test the coloration
of the line we have to print depending of the status.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
# Test with a template that is not designed for colorify
expected = self.to_print["basic_string"]
actual = Prints(None, "Hehehe", output_file=None, only_on_file=False)._colorify(
self.to_print["basic_string"]
)
self.assertEqual(expected, actual)
# Test with a template that is designed for colorify + Status is UP
expected = (
PyFunceble.Fore.BLACK
+ PyFunceble.Back.GREEN
+ self.to_print["basic_string"]
)
actual = Prints(
["This is a test", PyFunceble.STATUS["official"]["up"]],
"Generic",
output_file=None,
only_on_file=False,
)._colorify(self.to_print["basic_string"])
self.assertEqual(expected, actual)
# Test with a template that is designed for colorify + Status is DOWN
expected = (
PyFunceble.Fore.BLACK + PyFunceble.Back.RED + self.to_print["basic_string"]
)
actual = Prints(
["This is a test", PyFunceble.STATUS["official"]["down"]],
"Generic",
output_file=None,
only_on_file=False,
)._colorify(self.to_print["basic_string"])
self.assertEqual(expected, actual)
# Test with a template that is designed for colorify + Status is
# UNKNOWN or INVALID
expected = (
PyFunceble.Fore.BLACK + PyFunceble.Back.CYAN + self.to_print["basic_string"]
)
actual = Prints(
["This is a test", PyFunceble.STATUS["official"]["invalid"]],
"Generic",
output_file=None,
only_on_file=False,
)._colorify(self.to_print["basic_string"])
self.assertEqual(expected, actual)
def test_header_constructor_without_separator(self): # pylint: disable=invalid-name
"""
Test Prints()._header_constructor() for the case that we
want to print the result of the test.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
expected = ["hello world here is PyFunceble"]
actual = Prints(
None, None, output_file=None, only_on_file=False
)._header_constructor(self.to_print["basic"], None)
self.assertEqual(expected, actual)
# Test of the case that we want to print the hosts file format.
expected = [" ".join(self.to_print["hosts"].keys())]
actual = Prints(
None, None, output_file=None, only_on_file=False
)._header_constructor(self.to_print["hosts"], None)
self.assertEqual(expected, actual)
def _retrieve(self):
"""
Retrieve the mining informations.
"""
if PyFunceble.CONFIGURATION["mining"]:
# The mining is activated.
if "mined" not in PyFunceble.INTERN:
PyFunceble.INTERN["mined"] = {}
if PyFunceble.path.isfile(self.file):
# Our backup file exist.
# We return the information from our backup.
data = Dict().from_json(File(self.file).read())
# We clean the empty elements.
for file_path in data:
PyFunceble.INTERN["mined"][file_path] = {}
for element in data[file_path]:
if data[file_path][element]:
PyFunceble.INTERN["mined"][file_path][
element] = data[file_path][element]
return
# * The mining is not activated.
# or
# * Our backup file does not exist.
# We return nothing.
PyFunceble.INTERN["mined"] = {}
return
def test_tested_same_last(self):
"""
Test the cleaning process in the case that the number of tested
is = len(list_to_test).
"""
File(self.file).write("Hello, World!")
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
self.set_counter(3)
expected = {"up": 3, "down": 3, "invalid": 3, "tested": 3}
actual = PyFunceble.CONFIGURATION["counter"]["number"]
self.assertEqual(expected, actual)
Clean(["hello.world", "world.hello", "hello-world.com"])
expected = {"up": 0, "down": 0, "invalid": 0, "tested": 0}
actual = PyFunceble.CONFIGURATION["counter"]["number"]
self.assertEqual(expected, actual)
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_tested_out_of_index(self):
"""
TTest the cleaning process in the case that the number of tested
is > len(list_to_test).
"""
File(self.file).write("Hello, World!")
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
self.set_counter()
expected = {"up": 15, "down": 15, "invalid": 15, "tested": 15}
actual = PyFunceble.CONFIGURATION["counter"]["number"]
self.assertEqual(expected, actual)
Clean(["hello.world"])
expected = {"up": 0, "down": 0, "invalid": 0, "tested": 0}
actual = PyFunceble.CONFIGURATION["counter"]["number"]
self.assertEqual(expected, actual)
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
