def save_to_env_file(cls, envs, env_file_location):
"""
Save the given dict of environment variable into our environment file.
:param dict envs: A dict of environment variables to save.
:param str env_file_location: The location of the file we have to update.
"""
file_instance = File(env_file_location)
try:
content = file_instance.read()
except FileNotFoundError:
content = ""
if content:
for environment_variable, value in envs.items():
to_write = "{0}={1}".format(environment_variable, value)
regex = r"{0}=.*".format(environment_variable)
if Regex(content, regex, return_data=False).match():
content = Regex(content, regex,
replace_with=to_write).replace()
else:
if not content.endswith("\n"):
content += "\n{0}\n".format(to_write)
else:
content += "{0}\n".format(to_write)
else:
for environment_variable, value in envs.items():
to_write = "{0}={1}".format(environment_variable, value)
content += "{0}\n".format(to_write)
file_instance.write(content, overwrite=True)
def _get_structure(self):
"""
This method return the structure we are goinng to work with.
"""
structure_file = ""
req = ""
if path.isfile(self.structure):
structure_file = self.structure
elif path.isfile(self.base + "dir_structure_production.json"):
structure_file = self.base + "dir_structure_production.json"
else:
if "dev" not in PyFunceble.VERSION:
req = requests.get(PyFunceble.LINKS["dir_structure"])
else:
req = requests.get(PyFunceble.LINKS["dir_structure"].replace(
"master", "dev"))
if structure_file.endswith("_production.json"):
structure = Dict().from_json(File(structure_file).read())
return self._update_structure_from_config(structure)
elif structure_file.endswith(".json"):
return Dict().from_json(File(structure_file).read())
return self._update_structure_from_config(Dict().from_json(req.text))
def parse_env_file(cls, env_file_location):
"""
Parse the environment file into something we understand.
:param str env_file_location: The location of the file we have to parse.
"""
result = {}
content = ""
if PyFunceble.path.isfile(env_file_location):
content = File(env_file_location).read()
for line in content.splitlines():
line = line.strip()
if line.startswith("#"):
continue
if "#" in line:
line = line[:line.find("#")]
if "=" in line:
splited = line.split("=")
result[splited[0]] = splited[1]
return result
def test_retrieve_file_exist(self):
"""
Test the case that we want to retrieve a file that exist.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
PyFunceble.INTERN["to_test_type"] = "domain"
Dict(self.excepted_content).to_json(self.file)
Mining()._retrieve()
self.assertEqual(self.excepted_content, PyFunceble.INTERN["mined"])
del PyFunceble.INTERN["mined"]
del PyFunceble.INTERN["to_test_type"]
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_restore_old_system(self):
"""
This method test AutoContinue().restore() for the case that we run the
most recent version but with data from the old system.
"""
PyFunceble.CONFIGURATION["auto_continue"] = True
File(self.file).delete()
old_system = {
PyFunceble.CONFIGURATION["file_to_test"]: {
"number_of_up": 15,
"number_of_down": 18,
"number_of_invalid": 5,
"number_of_tested": 38,
}
}
Dict(old_system).to_json(self.file)
AutoContinue().restore()
expected = {"up": 15, "down": 18, "invalid": 5, "tested": 38}
actual = PyFunceble.CONFIGURATION["counter"]["number"]
self.assertEqual(expected, actual)
self.set_counter(0)
expected = {"up": 0, "down": 0, "invalid": 0, "tested": 0}
actual = PyFunceble.CONFIGURATION["counter"]["number"]
self.assertEqual(expected, actual)
PyFunceble.CONFIGURATION["auto_continue"] = False
File(self.file).delete()
def __init__(self):
if (PyFunceble.CONFIGURATION["auto_continue"]
and not PyFunceble.CONFIGURATION["no_files"]):
# * The auto_continue subsystem is activated.
# and
# * We are authorized to generate files.
# We set the log file location.
self.autocontinue_log_file = (
PyFunceble.OUTPUT_DIRECTORY +
PyFunceble.OUTPUTS["parent_directory"] +
PyFunceble.OUTPUTS["logs"]["filenames"]["auto_continue"])
if PyFunceble.path.isfile(self.autocontinue_log_file):
# The log file already exist.
# We get its content and save it inside backup_content.
self.backup_content = Dict().from_json(
File(self.autocontinue_log_file).read())
else:
# The log file does not exist.
# We initiate the backup content.
self.backup_content = {}
# And we save our empty backup_content to the log file.
File(self.autocontinue_log_file).write(str(
self.backup_content))
def test_remove(self):
"""
Test the deletion subsystem.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
PyFunceble.INTERN["mined"] = self.excepted_content
PyFunceble.INTERN["to_test_type"] = "domain"
PyFunceble.INTERN["to_test"] = "www.google.com"
expected = {
PyFunceble.INTERN["file_to_test"]: {
"myètherwället.com": ["www.facebook.com"]
}
}
Mining().remove()
self.assertEqual(expected, PyFunceble.INTERN["mined"])
del PyFunceble.INTERN["mined"]
del PyFunceble.INTERN["to_test"]
del PyFunceble.INTERN["to_test_type"]
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
def test_to_yaml(self):
"""
Test Dict().to_yaml.
"""
file_to_read = "this_yaml_is_a_ghost.yaml"
expected = False
actual = PyFunceble.path.isfile(file_to_read)
self.assertEqual(expected, actual)
to_write = {"hello": ["This is PyFunceble!", "Uhh!"], "world": "Fun Ilrys"}
expected = "{hello: [This is PyFunceble!, Uhh!], world: Fun Ilrys}\n"
Dict(to_write).to_yaml(file_to_read, flow_style=True)
expected = """hello:
- This is PyFunceble!
- Uhh!
world: Fun Ilrys
"""
Dict(to_write).to_yaml(file_to_read, flow_style=False)
actual = File(file_to_read).read()
self.assertEqual(expected, actual)
File(file_to_read).delete()
expected = False
actual = PyFunceble.path.isfile(file_to_read)
self.assertEqual(expected, actual)
def test_retrieve_file_not_exist(self):
"""
Test the case that we want to retrieve a file that does not exist.
"""
File(self.file).delete()
actual = PyFunceble.path.isfile(self.file)
expected = False
self.assertEqual(expected, actual)
Mining()._retrieve()
excepted = {}
self.assertEqual(excepted, PyFunceble.INTERN["mined"])
PyFunceble.INTERN["mined"] = {}
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_write_overwrite_delete(self):
"""
Test File.write() along with File.write() for the case that
we want to overwrite the content of a file.
"""
expected = "Hello, World! I'm domain2idna"
File("hi").write(expected)
with open("hi") as file:
actual = file.read()
self.assertEqual(expected, actual)
expected = "Hello, World! Python is great, you should consider learning it!"
File("hi").write(expected, overwrite=True)
with open("hi") as file:
actual = file.read()
self.assertEqual(expected, actual)
expected = False
File("hi").delete()
actual = PyFunceble.path.isfile("hi")
self.assertEqual(expected, actual)
def before_header(self):
"""
Print informations about PyFunceble and the date of generation of a file
into a given path, if doesn't exist.
"""
if not PyFunceble.CONFIGURATION[
"no_files"] and self.output and not path.isfile(self.output):
link = ("# File generated with %s\n" % PyFunceble.LINKS["repo"])
date_of_generation = ("# Date of generation: %s \n\n" %
PyFunceble.CURRENT_TIME)
if self.template in [
"Generic_File",
PyFunceble.STATUS["official"]["up"],
PyFunceble.STATUS["official"]["down"],
PyFunceble.STATUS["official"]["invalid"],
"Less",
]:
header = self._header_constructor(self.currently_used_header,
None)[0] + "\n"
try:
File(self.output).write(link + date_of_generation + header)
except UnboundLocalError:
File(self.output).write(link + date_of_generation)
def testhash_file(self):
"""
Test Hash._hash_file().
"""
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
File(self.file).write("\n".join(self.data_to_write))
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
for algo, result in self.expected_hashed.items():
self.assertEqual(
result,
Hash(self.file)._hash_file(algo),
msg="%s did not passed the test" % repr(algo),
)
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_save(self):
"""
Test the saving system.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
self.mining.database = self.excepted_content
self.mining.save()
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(
self.excepted_content, Dict().from_json(File(self.file).read())
)
File(self.file).delete()
actual = PyFunceble.path.isfile(self.file)
expected = False
self.assertEqual(expected, actual)
def _load_config_file(self):
"""
Load .PyFunceble.yaml into the system.
"""
try:
# We try to load the configuration file.
PyFunceble.CONFIGURATION.update(
Dict.from_yaml(File(self.path_to_config).read()))
# We install the latest iana configuration file.
self._install_iana_config()
# We install the latest public suffix configuration file.
self._install_psl_config()
# We install the latest directory structure file.
self._install_directory_structure_file()
except FileNotFoundError as exception:
# But if the configuration file is not found.
if PyFunceble.path.isfile(self.path_to_default_config):
# The `DEFAULT_CONFIGURATION_FILENAME` file exists.
# We copy it as the configuration file.
File(self.path_to_default_config).copy(self.path_to_config)
# And we load the configuration file as it does exist (yet).
self._load_config_file()
else:
# The `DEFAULT_CONFIGURATION_FILENAME` file does not exists.
# We raise the exception we were handling.
raise exception
def test_list_of_mined(self):
"""
Test Mining.list_of_mined
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
self.mining.database = self.excepted_content
expected = [
("myètherwället.com", "www.google.com"),
("myètherwället.com", "www.facebook.com"),
]
self.assertEqual(expected, self.mining.list_of_mined())
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
def test_remove(self):
"""
Test the deletion subsystem.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
expected = {
self.file_to_test: {
"myètherwället.com": ["www.facebook.com", "facebook.com"],
"example.org": ["facebook.com"],
}
}
self.mining.database = {self.file_to_test: {}}
self.mining["myètherwället.com"] = ["www.facebook.com", "facebook.com"]
self.mining["example.org"] = ["www.facebook.com", "facebook.com"]
self.mining.remove("example.org", "www.facebook.com")
self.assertEqual(expected, self.mining.database)
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
def test_add(self):
"""
Test the addition subsystem.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
expected = {
self.file_to_test: {"www.google.com": ["facebook.com", "www.facebook.com"]}
}
self.mining["www.google.com"] = ["facebook.com", "www.facebook.com"]
self.assertEqual(expected, self.mining.database)
self.mining["www.google.com"] = ["github.com"]
expected[self.file_to_test]["www.google.com"].append("github.com")
self.assertEqual(expected, self.mining.database)
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_get_specific_algo(self):
"""
Test Hash.get() for the case that we want a specifig
algorithm.
"""
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
File(self.file).write("\n".join(self.data_to_write))
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
expected = self.expected_hashed["sha512"]
actual = Hash(self.file, algorithm="sha512", only_hash=True).get()
self.assertEqual(expected, actual)
expected = self.expected_hashed["sha512"]
actual = Hash(
data="\n".join(self.data_to_write), algorithm="sha512", only_hash=True
).get()
self.assertEqual(expected, actual)
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_backup(self):
"""
This function test AutoContinue().backup().
"""
PyFunceble.CONFIGURATION["auto_continue"] = True
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
self.set_counter(to_set=25)
AutoContinue().backup()
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
expected = {
PyFunceble.CONFIGURATION["file_to_test"]: {
"up": 25, "down": 25, "invalid": 25, "tested": 25
}
}
actual = Dict().from_json(File(self.file).read())
self.assertEqual(expected, actual)
PyFunceble.CONFIGURATION["auto_continue"] = False
File(self.file).delete()
def test_get_all(self):
"""
Test Hash.get() for the case that we want all.
"""
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
File(self.file).write("\n".join(self.data_to_write))
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
expected = self.expected_hashed
actual = Hash(self.file, algorithm="all").get()
self.assertEqual(expected, actual)
actual = Hash(data="\n".join(self.data_to_write), algorithm="all").get()
self.assertEqual(expected, actual)
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_backup(self):
"""
Test the backup system.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
PyFunceble.INTERN["mined"] = self.excepted_content
Mining()._backup()
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(self.excepted_content,
Dict().from_json(File(self.file).read()))
del PyFunceble.INTERN["mined"]
File(self.file).delete()
actual = PyFunceble.path.isfile(self.file)
expected = False
self.assertEqual(expected, actual)
def test_hash_file(self):
"""
Tests the method which let us the content of a given file.
"""
file_instance = File(self.file)
expected = False
actual = file_instance.exists()
self.assertEqual(expected, actual)
file_instance.write("\n".join(self.data_to_write))
expected = True
actual = file_instance.exists()
self.assertEqual(expected, actual)
for algo, expected in self.expected_hashed.items():
self.assertEqual(
expected,
Hash(algo=algo).file(self.file),
)
file_instance.delete()
expected = False
actual = file_instance.exists()
self.assertEqual(expected, actual)
def _before_header(self):
"""
Print informations about PyFunceble and the date of generation of a file
into a given path, if doesn't exist.
"""
if (
not PyFunceble.CONFIGURATION["no_files"]
and self.output
and not PyFunceble.path.isfile(self.output)
):
# * We are allowed to generate files.
# and
# * And output is given.
# and
# * The given output does not exist.
# We initiate the information about what generated the file.
link = "# File generated by %s\n" % PyFunceble.LINKS["repo"]
# We initiate the information about the generation date of this file.
date_of_generation = (
"# Date of generation: %s \n\n" % PyFunceble.CURRENT_TIME
)
# We initiate a variable which will save the list of
# templates which have to meet in order to write the before
# header informations.
authorized_templates = [
"Generic_File",
PyFunceble.STATUS["official"]["up"],
PyFunceble.STATUS["official"]["down"],
PyFunceble.STATUS["official"]["invalid"],
PyFunceble.STATUS["official"]["valid"],
"Less",
]
if self.template in authorized_templates:
# The current header is in our list of authorized templated.
# We get the header.
header = (
self._header_constructor(self.currently_used_header, None)[0] + "\n"
)
try:
# We try to print the link, the date of generation and the header in the
# given file.
File(self.output).write(link + date_of_generation + header)
except UnboundLocalError:
# We don't have any header.
# We print the link and the date in the given file.
File(self.output).write(link + date_of_generation)
def setUp(self):
"""
Setups everything needed for the tests.
"""
self.file = "this_file_is_a_ghost"
self.file_instance = File(self.file)
self.file_instance_2 = File(self.file + "_2")
self.file_instance.delete()
self.file_instance_2.delete()
def test_before_header(self, header_constructor_patch):
"""
Test the functionability of Prints().before_header()
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
expected = """# File generated by {0} (v{1}) / {2}
# Date of generation: {3}
""".format(
PyFunceble.NAME,
PyFunceble.VERSION.split()[0],
PyFunceble.LINKS["repo"],
PyFunceble.CURRENT_TIME + " ",
)
Prints(None, None, output_file=self.file, only_on_file=False)._before_header()
self.assertEqual(expected, File(self.file).read())
# Test of the case that we have a Generic_File template
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
expected = """# File generated by {0} (v{1}) / {2}
# Date of generation: {3}
Hello World!
""".format(
PyFunceble.NAME,
PyFunceble.VERSION.split()[0],
PyFunceble.LINKS["repo"],
PyFunceble.CURRENT_TIME + " ",
)
header_constructor_patch.return_value = ["Hello World!"]
Prints(
None, "Generic_File", output_file=self.file, only_on_file=False
)._before_header()
self.assertEqual(expected, File(self.file).read())
def __init__(self):
if PyFunceble.CONFIGURATION["auto_continue"]:
self.autocontinue_log_file = PyFunceble.CURRENT_DIRECTORY + PyFunceble.OUTPUTS[
"parent_directory"] + PyFunceble.OUTPUTS["logs"]["filenames"][
"auto_continue"]
if path.isfile(self.autocontinue_log_file):
self.backup_content = Dict().from_json(
File(self.autocontinue_log_file).read())
else:
self.backup_content = {}
File(self.autocontinue_log_file).write(str(
self.backup_content))
def restore(self):
"""
Restore the 'output/' directory structure based on the `dir_structure.json` file.
"""
structure = self._get_structure()
list_of_key = list(structure.keys())
structure = structure[list_of_key[0]]
parent_path = list_of_key[0] + directory_separator
for directory in structure:
base = self.base + parent_path + directory + directory_separator
self._create_directory(base)
for file in structure[directory]:
file_path = base + file
content_to_write = structure[directory][file]["content"]
online_sha = structure[directory][file]["sha512"]
content_to_write = Regex(content_to_write,
"@@@",
escape=True,
replace_with="\\n").replace()
git_to_keep = file_path.replace("gitignore", "keep")
keep_to_git = file_path.replace("keep", "gitignore")
if self._restore_replace():
if path.isfile(file_path) and Hash(
file_path, "sha512", True).get() == online_sha:
rename(file_path, git_to_keep)
write = False
else:
File(file_path).delete()
file_path = git_to_keep
write = True
else:
if path.isfile(keep_to_git) and Hash(
file_path, "sha512", True).get() == online_sha:
rename(file_path, keep_to_git)
write = False
else:
File(keep_to_git).delete()
file_path = keep_to_git
write = True
if write:
File(file_path).write(content_to_write + "\n", True)
def __sort_generated_files(cls):
"""
Sort the content of all files we generated.
"""
for root, _, files in PyFunceble.walk(
PyFunceble.OUTPUT_DIRECTORY + PyFunceble.OUTPUTS["parent_directory"]
):
# We loop through the list of directories of the output directory.
for file in files:
# We loop through the list of file of the
# currently read directory.
if file.endswith(".json"):
# The currently read filename ends
# with .json.
# We continue the loop.
continue
if file in [".keep", ".gitignore"]:
# The currently read filename is
# into a list of filename that are not relevant
# for us.
# We continue the loop.
continue
# We create an instance of our File().
file_instance = File(
"{0}{1}{2}".format(root, PyFunceble.directory_separator, file)
)
# We get the content of the current file.
file_content = file_instance.read().splitlines()
if not PyFunceble.CONFIGURATION["hierarchical_sorting"]:
# We do not have to sort hierarchicaly.
# We sort the lines of the file standarly.
formatted = List(file_content[3:]).custom_format(Sort.standard)
else:
# We do have to sort hierarchicaly.
# We sort the lines of the file hierarchicaly.
formatted = List(file_content[3:]).custom_format(Sort.hierarchical)
# We finally put the formatted data in place.
file_instance.write(
"\n".join(file_content[:3] + formatted), overwrite=True
)
def test_copy(self):
"""
Test File.copy().
"""
file_to_write = "hello_world"
copy_destination = "world_hello"
expected = False
actual = PyFunceble.path.isfile(file_to_write)
self.assertEqual(expected, actual)
expected = "Hello, World! Python is great, you should consider learning it!"
File(file_to_write).write(expected)
with open(file_to_write) as file:
actual = file.read()
self.assertEqual(expected, actual)
expected = False
actual = PyFunceble.path.isfile(copy_destination)
self.assertEqual(expected, actual)
File(file_to_write).copy(copy_destination)
expected = True
actual = PyFunceble.path.isfile(copy_destination)
self.assertEqual(expected, actual)
expected = "Hello, World! Python is great, you should consider learning it!"
with open(copy_destination) as file:
actual = file.read()
self.assertEqual(expected, actual)
File(copy_destination).delete()
File(file_to_write).delete()
expected = False
actual = PyFunceble.path.isfile(copy_destination)
self.assertEqual(expected, actual)
actual = PyFunceble.path.isfile(file_to_write)
self.assertEqual(expected, actual)
def _update_docs(self, file_to_update):
"""
Update the given documentation file or :code:`README.rst` so that
it always gives branch related URL and informations.
.. note::
This only apply to :code:`dev` and :code:`master` branch.
:param str file_to_update: The file to update.
"""
if self.is_dev_version():
# The current version is the dev version.
# We map what we have to replace.
# Format: {match:replacement}
regexes = {
"/%s/" % "dev": r"\/%s\/" % "master",
"=%s" % "dev": "=%s" % "master",
}
elif self.is_master_version():
# The current version is the master version.
# We map what we have to replace.
regexes = {
"/%s/" % "master": r"\/%s\/" % "dev",
"=%s" % "master": "=%s" % "dev",
}
else:
# The current version is not the master nor the dev version.
# We raise an exception as the branch we are currently is not meaned
# for production.
raise Exception("Please switch to `dev` or `master` branch.")
file_instance = File(file_to_update)
# We get the content of the file to fix.
to_update = file_instance.read()
for replacement, regex in regexes.items():
# We loop through reach element of the map.
# We process the replacement.
to_update = Regex(to_update, regex, replace_with=replacement).replace()
# We finally overwrite the file to fix with the filtered.
# content.
file_instance.write(to_update, overwrite=True)
