def _get_structure(self):
"""
This method return the structure we are goinng to work with.
"""
structure_file = ""
req = ""
if path.isfile(self.structure):
structure_file = self.structure
elif path.isfile(self.base + "dir_structure_production.json"):
structure_file = self.base + "dir_structure_production.json"
else:
if "dev" not in PyFunceble.VERSION:
req = requests.get(PyFunceble.LINKS["dir_structure"])
else:
req = requests.get(PyFunceble.LINKS["dir_structure"].replace(
"master", "dev"))
if structure_file.endswith("_production.json"):
structure = Dict().from_json(File(structure_file).read())
return self._update_structure_from_config(structure)
elif structure_file.endswith(".json"):
return Dict().from_json(File(structure_file).read())
return self._update_structure_from_config(Dict().from_json(req.text))
def test_merge_multilevel(self):
"""
Test of Dict().merge() for the case that we have a multi level dict/list.
"""
origin = {
"hello": {"world": ["This is PyFunceble!", "Uhh!"]},
"world": "Fun Ilrys",
"hello_world": {"author": "funilrys", "name": "Fun"},
}
to_merge = {
"hello": {"world": ["hello", "Uhh"]},
"hello_world": {"author": "nobody", "surname": "body"},
}
expected = {
"hello": {"world": ["hello", "Uhh"]},
"world": "Fun Ilrys",
"hello_world": {"author": "nobody", "name": "Fun", "surname": "body"},
}
actual = Dict(origin).merge(to_merge, strict=True)
self.assertEqual(expected, actual)
expected = {
"hello": {"world": ["This is PyFunceble!", "Uhh!", "hello", "Uhh"]},
"world": "Fun Ilrys",
"hello_world": {"author": "nobody", "name": "Fun", "surname": "body"},
}
actual = Dict(origin).merge(to_merge, strict=False)
self.assertEqual(expected, actual)
def test_rename_key_single(self):
"""
This method will test Dict().rename_key() for the case that we want to
rename only one key.
"""
# Test of the strict case
expected = {
"Hello": "world",
"World": {"world", "hello"},
"funilrys": ["Fun", "Ilrys"],
"PyFunceble": "Funceble",
"pyfunceble": ["funilrys"],
}
actual = Dict(self.to_test).rename_key({"Py": "PyFunceble"})
self.assertEqual(expected, actual)
# Test of the non-strict case
expected = {
"Hello": "world",
"World": {"world", "hello"},
"funilrys": ["Fun", "Ilrys"],
"PyFunceble": "Funceble",
"pyfunceble": ["funilrys"],
}
actual = Dict(self.to_test).rename_key({"fun": "nuf"}, strict=False)
def test_to_yaml(self):
"""
Test Dict().to_yaml.
"""
file_to_read = "this_yaml_is_a_ghost.yaml"
expected = False
actual = PyFunceble.path.isfile(file_to_read)
self.assertEqual(expected, actual)
to_write = {"hello": ["This is PyFunceble!", "Uhh!"], "world": "Fun Ilrys"}
expected = "{hello: [This is PyFunceble!, Uhh!], world: Fun Ilrys}\n"
Dict(to_write).to_yaml(file_to_read, flow_style=True)
expected = """hello:
- This is PyFunceble!
- Uhh!
world: Fun Ilrys
"""
Dict(to_write).to_yaml(file_to_read, flow_style=False)
actual = File(file_to_read).read()
self.assertEqual(expected, actual)
File(file_to_read).delete()
expected = False
actual = PyFunceble.path.isfile(file_to_read)
self.assertEqual(expected, actual)
def test_to_json(self):
"""
Tests the method which let us get the JSON
representation of the given dict.
"""
expected = """{
"Hello": "world",
"Py": "Funceble",
"World": {
"world": "hello"
},
"funilrys": [
"Fun",
"Ilrys"
],
"pyfunceble": [
"funilrys"
]
}"""
actual = Dict(self.test_subject.copy()).to_json()
self.assertEqual(expected, actual)
actual = Dict().from_json(expected)
expected = self.test_subject.copy()
self.assertEqual(expected, actual)
def test_has_same_keys_as(self):
"""
Tests the method which let us know if the keys of
2 dicts are the same.
"""
# This is a.
origin = {"a": 1, "b": 1}
# This is b.
target = {"a": 1, "b": 2, "c": {"a": 1, "b": 3, "c": {"x": "x"}}}
# We want to test that all keys of a are into b.
self.assertEqual(True, Dict(target).has_same_keys_as(origin))
# We want to test that all keys of b are into a.
self.assertEqual(False, Dict(origin).has_same_keys_as(target))
origin["c"] = {"a": 1, "b": 3, "c": {"x": "x"}}
# We want to test that all keys of a are in b.
self.assertEqual(True, Dict(target).has_same_keys_as(origin))
# We want to test that all keys of b are in a.
self.assertEqual(True, Dict(origin).has_same_keys_as(target))
del origin["c"]["c"]
# We want to test that all keys of b are in a.
self.assertEqual(False, Dict(origin).has_same_keys_as(target))
def _merge_values(self):
"""
Simply merge the older into the new one.
"""
to_remove = []
self.new_config = Dict(
Dict(self.upstream_config).merge(
self.local_config)).remove_key(to_remove)
def _merge_values(self):
"""
Simply merge the older into the new one.
"""
to_remove = []
self.new_config = Dict(
Dict(self.upstream_config).merge(
PyFunceble.CONFIGURATION)).remove_key(to_remove)
def _json_print(self): # pragma: no cover
"""
Management of the json template.
"""
if self.output:
# The given output is not empty.
if PyFunceble.path.isfile(self.output):
# The given output already exist.
# We get the content of the output.
content = Dict().from_json(self.file_output_instance.read())
if isinstance(content, list):
# The content is a list.
# We extend the content with our data to print.
content.extend(self.data_to_print)
# We format our list.
content = List(content).custom_format(Sort.standard)
if PyFunceble.CONFIGURATION["hierarchical_sorting"]:
# The hierarchical sorting is activated.
# We format our content hierarchicaly
content = List(content).custom_format(
Sort.hierarchical)
# We finally save our content into the file.
Dict(content).to_json(self.output)
else:
# The content is not a list.
# We raise an exception.
raise Exception("Output not correctly formatted.")
else:
# The given output does not already exist.
# We save our data to print into the output.
#
# Note: We do not have to take care if self.data_to_print is a list
# formatted or not because this method should not be called if it is
# not the case.
Dict(self.data_to_print).to_json(self.output)
else:
# The given output is empty.
# We raise an exception.
raise Exception("Empty output given.")
def _get_current_version_yaml(cls):
"""
Get and return the content of version.yaml
"""
return Dict().from_yaml(
File(PyFunceble.CURRENT_DIRECTORY + "version.yaml").read())
def test_to_yaml_file_non_dict(self):
"""
Tests the method which let us save a dict into a YAML file
for the case that we don't given a dict.
"""
output_file = "this_file_is_a_ghost"
File(output_file).delete()
self.assertRaises(TypeError, lambda: Dict(1).to_yaml_file(output_file))
self.assertRaises(TypeError, lambda: Dict("100").to_yaml_file(output_file))
self.assertRaises(
TypeError, lambda: Dict("{'hello': 'world'}").to_yaml_file(output_file)
)
File(output_file).delete()
def test_to_test__path_exist_time_future(self): # pylint: disable=invalid-name
"""
Test Inactive.to_test() for the case that the path exist but
the timestamp is in the future.
"""
self.test_file_not_exist()
PyFunceble.INTERN["inactive_db"] = {
PyFunceble.INTERN["file_to_test"]: {
self.time_future: ["hello.world", "world.hello"]
}
}
expected = {
PyFunceble.INTERN["file_to_test"]: {
self.time_future: ["hello.world", "world.hello"],
"to_test": [],
}
}
Dict(PyFunceble.INTERN["inactive_db"]).to_json(self.file)
Inactive().to_test()
self.assertEqual(expected, PyFunceble.INTERN["inactive_db"])
del PyFunceble.INTERN["inactive_db"]
self.test_file_not_exist()
def _timestamp(self):
"""
Return the timestamp where we are going to save our current list.
Returns: int or str
The timestamp to append with the currently tested domains.
"""
result = 0
to_delete = []
if self.file_path in PyFunceble.CONFIGURATION[
"inactive_db"] and PyFunceble.CONFIGURATION["inactive_db"][
self.file_path]:
for data in PyFunceble.CONFIGURATION["inactive_db"][
self.file_path]:
if data != "to_test":
if self.current_time < int(data) + self.day_in_seconds:
result = int(data)
else:
result = self.current_time
to_delete.append(data)
for element in to_delete:
self._add_to_test(PyFunceble.CONFIGURATION["inactive_db"][
self.file_path][element])
Dict(PyFunceble.CONFIGURATION["inactive_db"][
self.file_path]).remove_key(to_delete)
return result
return self.current_time
def update(self):
"""
Update of the content of the :code:`public-suffix.json`.
"""
if not PyFunceble.CONFIGURATION["quiet"]:
# The quiet mode is not activated.
# We print a message for the user on screen.
print(
"Update of %s" %
PyFunceble.OUTPUTS["default_files"]["public_suffix"],
end=" ",
)
# We loop through the line of the upstream file.
list(map(self._extensions, self._data().split("\n")))
# We save the content of our database in the final testination.
Dict(self.public_suffix_db).to_json(self.destination)
if not PyFunceble.CONFIGURATION["quiet"]:
# The quiet mode is not activated.
# We inform the user that everything goes right.
print(PyFunceble.INTERN["done"])
def test_backup(self):
"""
Test AutoContinue().backup().
"""
self.test_delete_file()
PyFunceble.CONFIGURATION["auto_continue"] = True
self.set_counter(to_set=25)
AutoContinue().backup()
expected = True
actual = PyFunceble.path.isfile(self.file_to_work_with)
self.assertEqual(expected, actual)
expected = {
PyFunceble.INTERN["file_to_test"]: {
"up": 25,
"down": 25,
"invalid": 25,
"tested": 25,
}
}
actual = Dict().from_json(File(self.file_to_work_with).read())
self.assertEqual(expected, actual)
del PyFunceble.CONFIGURATION["auto_continue"]
self.test_delete_file()
def _load_config_file(self):
"""
Load .PyFunceble.yaml into the system.
"""
try:
# We try to load the configuration file.
PyFunceble.CONFIGURATION.update(
Dict.from_yaml(File(self.path_to_config).read()))
# We install the latest iana configuration file.
self._install_iana_config()
# We install the latest public suffix configuration file.
self._install_psl_config()
# We install the latest directory structure file.
self._install_directory_structure_file()
except FileNotFoundError as exception:
# But if the configuration file is not found.
if PyFunceble.path.isfile(self.path_to_default_config):
# The `DEFAULT_CONFIGURATION_FILENAME` file exists.
# We copy it as the configuration file.
File(self.path_to_default_config).copy(self.path_to_config)
# And we load the configuration file as it does exist (yet).
self._load_config_file()
else:
# The `DEFAULT_CONFIGURATION_FILENAME` file does not exists.
# We raise the exception we were handling.
raise exception
def test_save(self):
"""
Test the saving system.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
self.mining.database = self.excepted_content
self.mining.save()
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(
self.excepted_content, Dict().from_json(File(self.file).read())
)
File(self.file).delete()
actual = PyFunceble.path.isfile(self.file)
expected = False
self.assertEqual(expected, actual)
def load_config_file(self):
"""
This method will load .PyFunceble.yaml.
"""
PyFunceble.CONFIGURATION.update(
Dict.from_yaml(File(self.path_to_config).read()))
def to_test(self):
"""
Get the list to test for the next session.
"""
result = []
to_delete = []
self._retrieve()
if self.file_path in PyFunceble.CONFIGURATION["inactive_db"]:
for data in PyFunceble.CONFIGURATION["inactive_db"][
self.file_path]:
if data != "to_test":
if self.current_time > int(data) + self.day_in_seconds:
result.extend(PyFunceble.CONFIGURATION["inactive_db"][
self.file_path][data])
to_delete.append(data)
Dict(PyFunceble.CONFIGURATION["inactive_db"][
self.file_path]).remove_key(to_delete)
self._add_to_test(result)
else:
PyFunceble.CONFIGURATION["inactive_db"].update(
{self.file_path: {}})
self._backup()
def update(self):
"""
Update the content of the `iana-domains-db` file.
"""
if not PyFunceble.CONFIGURATION["quiet"]:
# * The quiet mode is not activated.
# We print on screen what we are doing.
print("Update of iana-domains-db", end=" ")
# We loop through the line of the iana website.
for extension, referer in self._extensions():
if extension not in self.iana_db or self.iana_db[
extension] != referer:
# We add the extension to the databae.
self.iana_db[extension] = referer
# We save the content of the constructed database.
Dict(self.iana_db).to_json(self.destination)
if not PyFunceble.CONFIGURATION["quiet"]:
# The quiet mode is not activated.
# We indicate that the work is done without any issue.
print(PyFunceble.INTERN["done"])
def _retrieve(self):
"""
Retrieve the mining informations.
"""
if PyFunceble.CONFIGURATION["mining"]:
# The mining is activated.
if "mined" not in PyFunceble.INTERN:
PyFunceble.INTERN["mined"] = {}
if PyFunceble.path.isfile(self.file):
# Our backup file exist.
# We return the information from our backup.
data = Dict().from_json(File(self.file).read())
# We clean the empty elements.
for file_path in data:
PyFunceble.INTERN["mined"][file_path] = {}
for element in data[file_path]:
if data[file_path][element]:
PyFunceble.INTERN["mined"][file_path][
element] = data[file_path][element]
return
# * The mining is not activated.
# or
# * Our backup file does not exist.
# We return nothing.
PyFunceble.INTERN["mined"] = {}
return
def clean(self):
"""
Clean the database.
"""
if self.authorized:
# We are authorized to operate.
if PyFunceble.CONFIGURATION["db_type"] == "json":
# We empty the database.
self.database[self.filename] = {}
# And we save the current database state.
Dict(self.database).to_json(self.database_file)
elif PyFunceble.CONFIGURATION["db_type"] == "sqlite":
# We construct the query we are going to execute.
query = "DELETE FROM {0} WHERE file_path = :file".format(
self.table_name)
# We execute it.
self.sqlite_db.cursor.execute(query, {"file": self.filename})
# We commit everything.
self.sqlite_db.connection.commit()
elif PyFunceble.CONFIGURATION["db_type"] in ["mariadb", "mysql"]:
# We construct the query we are going to execute.
query = "DELETE FROM {0} WHERE file_path = %(file)s".format(
self.table_name)
with self.mysql_db.get_connection() as cursor:
cursor.execute(query, {"file": self.filename})
def test_retrieve_file_exist(self):
"""
Test the case that we want to retrieve a file that exist.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
PyFunceble.INTERN["to_test_type"] = "domain"
Dict(self.excepted_content).to_json(self.file)
Mining()._retrieve()
self.assertEqual(self.excepted_content, PyFunceble.INTERN["mined"])
del PyFunceble.INTERN["mined"]
del PyFunceble.INTERN["to_test_type"]
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_backup(self):
"""
This function test AutoContinue().backup().
"""
PyFunceble.CONFIGURATION["auto_continue"] = True
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
self.set_counter(to_set=25)
AutoContinue().backup()
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
expected = {
PyFunceble.CONFIGURATION["file_to_test"]: {
"up": 25, "down": 25, "invalid": 25, "tested": 25
}
}
actual = Dict().from_json(File(self.file).read())
self.assertEqual(expected, actual)
PyFunceble.CONFIGURATION["auto_continue"] = False
File(self.file).delete()
def test_add_file_path_not_present(self): # pylint: disable=invalid-name
"""
Test Inactive.add() for the case that the path is not
present into the Inactive.
"""
self.test_file_not_exist()
timestamp = str(Inactive()._timestamp())
PyFunceble.INTERN["to_test"] = "hello.world"
expected = {
PyFunceble.INTERN["file_to_test"]: {
timestamp: ["hello.world"]
}
}
Inactive().add()
actual = Dict().from_json(File(self.file).read())
self.assertEqual(expected, actual)
del PyFunceble.INTERN["to_test"]
del PyFunceble.INTERN["inactive_db"]
self.test_file_not_exist()
def backup(cls):
"""
Backup the developer state of `output/` in order to make it restorable
and portable for user.
"""
output_path = PyFunceble.CURRENT_DIRECTORY + PyFunceble.OUTPUTS[
"parent_directory"]
result = {PyFunceble.OUTPUTS["parent_directory"]: {}}
for root, _, files in walk(output_path):
directories = root.split(output_path)[1]
local_result = result[PyFunceble.OUTPUTS["parent_directory"]]
for file in files:
file_path = root + directory_separator + file
file_hash = Hash(file_path, "sha512", True).get()
lines_in_list = [line.rstrip("\n") for line in open(file_path)]
formated_content = "@@@".join(lines_in_list)
local_result = local_result.setdefault(
directories,
{file: {
"sha512": file_hash,
"content": formated_content
}},
)
Dict(result).to_json(PyFunceble.CURRENT_DIRECTORY +
"dir_structure_production.json")
def test_backup(self):
"""
Test the backup system.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
PyFunceble.INTERN["mined"] = self.excepted_content
Mining()._backup()
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(self.excepted_content,
Dict().from_json(File(self.file).read()))
del PyFunceble.INTERN["mined"]
File(self.file).delete()
actual = PyFunceble.path.isfile(self.file)
expected = False
self.assertEqual(expected, actual)
def __init__(self, configuration_path):
config_link = Version(True).right_url_from_version(
"https://raw.githubusercontent.com/funilrys/PyFunceble/master/.PyFunceble_production.yaml" # pylint: disable=line-too-long
)
self.path_to_config = configuration_path
if not self.path_to_config.endswith(PyFunceble.directory_separator):
self.path_to_config += PyFunceble.directory_separator
self.path_to_config += PyFunceble.CONFIGURATION_FILENAME
dict_instance = Dict()
self.local_config = dict_instance.from_yaml(
File(self.path_to_config).read())
self.upstream_config = dict_instance.from_yaml(
Download(config_link, return_data=True).text())
if self.upstream_config["links"]["config"] != config_link:
self.upstream_config = dict_instance.from_yaml(
Download(self.upstream_config["links"]["config"],
return_data=True).text())
self.new_config = {}
self._load()
def test_restore_old_system(self):
"""
This method test AutoContinue().restore() for the case that we run the
most recent version but with data from the old system.
"""
PyFunceble.CONFIGURATION["auto_continue"] = True
File(self.file).delete()
old_system = {
PyFunceble.CONFIGURATION["file_to_test"]: {
"number_of_up": 15,
"number_of_down": 18,
"number_of_invalid": 5,
"number_of_tested": 38,
}
}
Dict(old_system).to_json(self.file)
AutoContinue().restore()
expected = {"up": 15, "down": 18, "invalid": 5, "tested": 38}
actual = PyFunceble.CONFIGURATION["counter"]["number"]
self.assertEqual(expected, actual)
self.set_counter(0)
expected = {"up": 0, "down": 0, "invalid": 0, "tested": 0}
actual = PyFunceble.CONFIGURATION["counter"]["number"]
self.assertEqual(expected, actual)
PyFunceble.CONFIGURATION["auto_continue"] = False
File(self.file).delete()
def __init__(self):
if (PyFunceble.CONFIGURATION["auto_continue"]
and not PyFunceble.CONFIGURATION["no_files"]):
# * The auto_continue subsystem is activated.
# and
# * We are authorized to generate files.
# We set the log file location.
self.autocontinue_log_file = (
PyFunceble.OUTPUT_DIRECTORY +
PyFunceble.OUTPUTS["parent_directory"] +
PyFunceble.OUTPUTS["logs"]["filenames"]["auto_continue"])
if PyFunceble.path.isfile(self.autocontinue_log_file):
# The log file already exist.
# We get its content and save it inside backup_content.
self.backup_content = Dict().from_json(
File(self.autocontinue_log_file).read())
else:
# The log file does not exist.
# We initiate the backup content.
self.backup_content = {}
# And we save our empty backup_content to the log file.
File(self.autocontinue_log_file).write(str(
self.backup_content))
