def test_backup(self):
"""
Test the backup system.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
PyFunceble.INTERN["mined"] = self.excepted_content
Mining()._backup()
expected = True
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(self.excepted_content,
Dict().from_json(File(self.file).read()))
del PyFunceble.INTERN["mined"]
File(self.file).delete()
actual = PyFunceble.path.isfile(self.file)
expected = False
self.assertEqual(expected, actual)
def test_remove(self):
"""
Test the deletion subsystem.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
PyFunceble.INTERN["mined"] = self.excepted_content
PyFunceble.INTERN["to_test_type"] = "domain"
PyFunceble.INTERN["to_test"] = "www.google.com"
expected = {
PyFunceble.INTERN["file_to_test"]: {
"myètherwället.com": ["www.facebook.com"]
}
}
Mining().remove()
self.assertEqual(expected, PyFunceble.INTERN["mined"])
del PyFunceble.INTERN["mined"]
del PyFunceble.INTERN["to_test"]
del PyFunceble.INTERN["to_test_type"]
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
def test_retrieve_file_exist(self):
"""
Test the case that we want to retrieve a file that exist.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
PyFunceble.INTERN["to_test_type"] = "domain"
Dict(self.excepted_content).to_json(self.file)
Mining()._retrieve()
self.assertEqual(self.excepted_content, PyFunceble.INTERN["mined"])
del PyFunceble.INTERN["mined"]
del PyFunceble.INTERN["to_test_type"]
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_retrieve_file_not_exist(self):
"""
Test the case that we want to retrieve a file that does not exist.
"""
File(self.file).delete()
actual = PyFunceble.path.isfile(self.file)
expected = False
self.assertEqual(expected, actual)
Mining()._retrieve()
excepted = {}
self.assertEqual(excepted, PyFunceble.INTERN["mined"])
PyFunceble.INTERN["mined"] = {}
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_add(self):
"""
Test the addition subsystem.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
to_add = {"www.google.com": ["facebook.com", "www.facebook.com"]}
expected = {PyFunceble.INTERN["file_to_test"]: to_add}
Mining()._add(to_add)
self.assertEqual(expected, PyFunceble.INTERN["mined"])
to_add["www.google.com"].append("github.com")
expected = {
PyFunceble.INTERN["file_to_test"]: {
"www.google.com":
["facebook.com", "github.com", "www.facebook.com"]
}
}
Mining()._add(to_add)
self.assertEqual(expected, PyFunceble.INTERN["mined"])
del PyFunceble.INTERN["mined"]
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def __init__(self, file, file_type="domain"):
# We share the file we are working with.
self.file = file
# We share the file/test type.
self.file_type = file_type
# We construct the list of UP statuses.
self.list_of_up_statuses = PyFunceble.STATUS["list"]["up"]
self.list_of_up_statuses.extend(PyFunceble.STATUS["list"]["valid"])
# We get/initiate the db.
self.sqlite_db = SQLite()
self.mysql_db = MySQL()
# We get/initiate the preset class.
self.preset = PyFunceble.Preset()
# We get/initiate the autosave database/subsyste..
self.autosave = AutoSave(start_time=PyFunceble.INTERN["start"])
# We get/initiate the inactive database.
self.inactive_db = InactiveDB(self.file,
sqlite_db=self.sqlite_db,
mysql_db=self.mysql_db)
# We get/initiate the whois database.
self.whois_db = WhoisDB(sqlite_db=self.sqlite_db,
mysql_db=self.mysql_db)
# We get/initiate the mining subsystem.
self.mining = Mining(self.file,
sqlite_db=self.sqlite_db,
mysql_db=self.mysql_db)
# We get/initiate the autocontinue subsystem.
self.autocontinue = AutoContinue(
self.file,
parent_process=True,
sqlite_db=self.sqlite_db,
mysql_db=self.mysql_db,
)
# We initiate a variable which will tell us when
# we start testing for complements.
self.complements_test_started = False
# We download the file if it is a list.
self.download_link()
def __init__(self, **args):
# We initiate our list of optional arguments with their default values.
optional_arguments = {
"domain_or_ip_to_test": None,
"file_path": None,
"url_to_test": None,
"url_file": None,
"modulo_test": False,
"link_to_test": None,
}
# We initiate our optional_arguments in order to be usable all over the
# class.
for (arg, default) in optional_arguments.items():
setattr(self, arg, args.get(arg, default))
# We initiate a variable in order to avoid having to recall/declare
# Status() over and over.
self.status = Status()
# We initiate a variable in order to avoid having to recall/declare
# Check() over and over.
self.checker = Check()
# We initiate a variable in order to avoid having to recall/declare
# Percentage() over and over.
self.percentage = Percentage()
# We initiate a variable in order to avoid having to recall/declare
# URL() over and over.
self.url_status = URL()
# We initiate a variable in order to avoid having to recall/declare
# Mining() over and over.
self.mining = Mining()
# We initiate a variable in order to avoid having to recall/declare
# AutoContinue() over and over.
self.auto_continue = None
# We initiate a variable in order to avoid having to recall/declare
# Syntax() over and over.
self.syntax_status = Syntax()
# We initiate a variable in order to avoid having to recall/declare
# Inactive() over and over.
self.inactive_database = Inactive()
# We manage the entries.
self._entry_management()
def test_list_of_mined(self):
"""
Test Mining.list_of_mined
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
PyFunceble.INTERN["mined"] = self.excepted_content
expected = ["www.facebook.com", "www.google.com"]
self.assertEqual(expected, Mining().list_of_mined())
del PyFunceble.INTERN["mined"]
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
def setUp(self):
"""
Setup everything needeed for the test.
"""
PyFunceble.load_config(
generate_directory_structure=False, custom={"db_type": "json"}
)
PyFunceble.CONFIGURATION["mining"] = True
self.file_to_test = "this_file_is_a_ghost"
self.file = (
PyFunceble.CONFIG_DIRECTORY + PyFunceble.OUTPUTS["default_files"]["mining"]
)
self.excepted_content = {
self.file_to_test: {
"myètherwället.com": ["www.google.com", "www.facebook.com"]
}
}
self.mining = Mining(self.file_to_test)
def _test_line(self, line, manager_data=None): # pylint: disable=too-many-branches # pragma: no cover
"""
Given a line, we test it.
:param str line: A line to work with.
:param multiprocessing.Manager.list manager_data: A Server process.
"""
if PyFunceble.CONFIGURATION[
"db_type"] == "json" and manager_data is not None:
autocontinue = AutoContinue(self.file, parent_process=False)
inactive_db = InactiveDB(self.file)
mining = Mining(self.file)
else:
# We use the previously initiated autocontinue instance.
autocontinue = self.autocontinue
# We use the previously initiated inactive database instance.
inactive_db = self.inactive_db
# We use the previously initiated mining instance.
mining = self.mining
# We remove cariage from the given line.
line = line.strip()
if not line or line[0] == "#":
# We line is a comment line.
# We return None, there is nothing to test.
return None
if Regex(line, self.regex_ignore, escape=False,
return_data=False).match():
# The line match our list of elemenet
# to ignore.
# We return None, there is nothing to test.
return None
# We format the line, it's the last
# rush before starting to filter and test.
subject = self._format_line(line)
if (not PyFunceble.CONFIGURATION["local"]
and PyFunceble.Check(subject).is_reserved_ipv4()):
# * We are not testing for local components.
# and
# * The subject is a reserved IPv4.
# We return None, there is nothing to test.
return None
if PyFunceble.CONFIGURATION["filter"]:
# We have to filter.
if Regex(subject,
PyFunceble.CONFIGURATION["filter"],
return_data=False).match():
# The line match the given filter.
# We get the status of the current line.
status = self.__process_test(subject)
else:
# The line does not match the given filter.
# We return None.
return None
else:
# We do not have to filter.
# We get the status of the current line.
status = self.__process_test(subject)
# We add the line into the auto continue database.
autocontinue.add(subject, status)
if status.lower() in self.list_of_up_statuses:
# The status is in the list of UP status.
# We mine if necessary.
mining.mine(subject, self.file_type)
if subject in inactive_db:
# The subject is in the inactive database.
# We generate the suspicous file.
Generate(subject, "file_domain", PyFunceble.STATUS["official"]
["up"]).analytic_file("suspicious")
# And we remove the current subject from
# the inactive database.
inactive_db.remove(subject)
else:
# The status is not in the list of UP status.
# We add the current subject into the
# inactive database.
inactive_db.add(subject, status)
if (self.complements_test_started
and PyFunceble.CONFIGURATION["db_type"] == "json"):
# We started the test of the complements.
if "complements" in autocontinue.database:
# The complement index is present.
while subject in autocontinue.database["complements"]:
# We loop untill the line is not present into the
# database.
# We remove the currently tested element.
autocontinue.database["complements"].remove(subject)
# We save the current state.
autocontinue.save()
if manager_data is None:
# We are not in a multiprocess environment.
# We update the counters
autocontinue.update_counters()
# We process the autosaving if it is necessary.
self.autosave.process(test_completed=False)
elif PyFunceble.CONFIGURATION["db_type"] == "json":
# We are in a multiprocess environment.
# We save everything we initiated into the server process
manager_data.append({
"autocontinue": autocontinue.database,
"inactive_db": inactive_db.database,
"mining": mining.database,
})
# We return None.
return None
def _file_list_to_test_filtering(self):
"""
Unify the way we work before testing file contents.
"""
# We get the list to test from the file we have to test.
list_to_test = self._extract_domain_from_file()
# We save the original list to test globally.
PyFunceble.INTERN["extracted_list_to_test"] = list_to_test
# We get the list of mined.
mined_list = Mining().list_of_mined()
if mined_list:
list_to_test.extend(mined_list)
# We generate the directory structure.
DirectoryStructure()
# We restore the data from the last session if it does exist.
AutoContinue().restore()
if PyFunceble.CONFIGURATION["adblock"]:
# The adblock decoder is activated.
# We get the list of domain to test (decoded).
list_to_test = AdBlock(list_to_test).decode()
else:
# The adblock decoder is not activated.
# We get the formatted list of domain to test.
list_to_test = list(map(self._format_domain, list_to_test))
# We clean the output directory if it is needed.
PyFunceble.Clean(list_to_test)
# We set the start time.
ExecutionTime("start")
# We get the list we have to test in the current session (from the database).
Inactive().to_test()
if (
PyFunceble.CONFIGURATION["inactive_database"]
and PyFunceble.INTERN["file_to_test"] in PyFunceble.INTERN["inactive_db"]
and "to_test"
in PyFunceble.INTERN["inactive_db"][PyFunceble.INTERN["file_to_test"]]
and PyFunceble.INTERN["inactive_db"][PyFunceble.INTERN["file_to_test"]][
"to_test"
]
):
# * The current file to test in into the database.
# and
# * The `to_test` index is present into the database
# related to the file we are testing.
# and
# * The `to_test` index content is not empty.
# We extend our list to test with the content of the `to_test` index
# of the current file database.
list_to_test.extend(
PyFunceble.INTERN["inactive_db"][PyFunceble.INTERN["file_to_test"]][
"to_test"
]
)
# We set a regex of element to delete.
# Understand with this variable that we don't want to test those.
regex_delete = r"localhost$|localdomain$|local$|broadcasthost$|0\.0\.0\.0$|allhosts$|allnodes$|allrouters$|localnet$|loopback$|mcastprefix$|ip6-mcastprefix$|ip6-localhost$|ip6-loopback$|ip6-allnodes$|ip6-allrouters$|ip6-localnet$" # pylint: disable=line-too-long
# We load the flatten version of the database.
PyFunceble.INTERN.update({"flatten_inactive_db": Inactive().content()})
# We initiate a local variable which will save the current state of the list.
not_filtered = list_to_test
try:
# We remove the element which are in the database from the
# current list to test.
list_to_test = List(
list(
set(Regex(list_to_test, regex_delete).not_matching_list())
- set(PyFunceble.INTERN["flatten_inactive_db"])
)
).format()
_ = list_to_test[-1]
except IndexError:
# We test without the database removing.
list_to_test = List(
Regex(not_filtered, regex_delete).not_matching_list()
).format()
# We delete the not_filtered variable.
del not_filtered
if PyFunceble.CONFIGURATION["filter"]:
# The filter is not empty.
# We get update our list to test. Indeed we only keep the elements which
# matches the given filter.
list_to_test = List(
Regex(
list_to_test, PyFunceble.CONFIGURATION["filter"], escape=False
).matching_list()
).format()
list_to_test = List(list(list_to_test)).custom_format(Sort.standard)
if PyFunceble.CONFIGURATION["hierarchical_sorting"]:
# The hierarchical sorting is desired by the user.
# We format the list.
list_to_test = List(list(list_to_test)).custom_format(Sort.hierarchical)
# We return the final list to test.
return list_to_test
def _file_decision(self, current, last, status=None):
"""
Manage the database, autosave and autocontinue systems for the case that we are reading
a file.
:param current: The currently tested element.
:type current: str
:param last: The last element of the list.
:type last: str
:param status: The status of the currently tested element.
:type status: str
"""
if (
status
and not PyFunceble.CONFIGURATION["simple"]
and PyFunceble.INTERN["file_to_test"]
):
# * The status is given.
# and
# * The simple mode is deactivated.
# and
# * A file to test is set.
# We run the mining logic.
Mining().process()
# We delete the currently tested element from the mining
# database.
# Indeed, as it is tested, it is already in our
# testing process which means that we don't need it into
# the mining database.
Mining().remove()
if (
status.lower() in PyFunceble.STATUS["list"]["up"]
or status.lower() in PyFunceble.STATUS["list"]["valid"]
):
# The status is in the list of up status.
if Inactive().is_present():
# The currently tested element is in the database.
# We generate the suspicious file(s).
Generate("strange").analytic_file(
"suspicious", PyFunceble.STATUS["official"]["up"]
)
# We remove the currently tested element from the
# database.
Inactive().remove()
else:
# The status is not in the list of up status.
# We add the currently tested element to the
# database.
Inactive().add()
# We backup the current state of the file reading
# for the case that we need to continue later.
AutoContinue().backup()
if current != last:
# The current element is not the last one.
# We run the autosave logic.
AutoSave()
else:
# The current element is the last one.
# We stop and log the execution time.
ExecutionTime("stop", True)
# We show/log the percentage.
Percentage().log()
# We reset the counters as we end the process.
self.reset_counters()
# We backup the current state of the file reading
# for the case that we need to continue later.
AutoContinue().backup()
# We show the colored logo.
self.colorify_logo()
# We save and stop the script if we are under
# Travis CI.
AutoSave(True)
for index in ["http_code", "referer"]:
# We loop through some configuration index we have to empty.
if index in PyFunceble.INTERN:
# The index is in the configuration.
# We empty the configuration index.
PyFunceble.INTERN[index] = ""