def __init__(self, domain=None, file_path=None, **args):
optional_arguments = {
"url_to_test": None,
"file_urls": None,
"modulo_test": False
}
# We initiate our optional_arguments in order to be usable all over the
# class
for (arg, default) in optional_arguments.items():
setattr(self, arg, args.get(arg, default))
if not self.modulo_test: # pylint: disable=no-member
PyFunceble.CONFIGURATION["file_to_test"] = file_path # pylint: disable=no-member
if self.file_urls: # pylint: disable=no-member
PyFunceble.CONFIGURATION["file_to_test"] = self.file_urls # pylint: disable=no-member
if PyFunceble.CONFIGURATION["travis"]:
AutoSave().travis_permissions()
self.bypass()
ExecutionTime("start")
if domain:
PyFunceble.CONFIGURATION["show_percentage"] = False
PyFunceble.CONFIGURATION["domain"] = domain.lower()
self.domain()
elif self.url_to_test and not file_path: # pylint: disable=no-member
PyFunceble.CONFIGURATION["show_percentage"] = False
PyFunceble.CONFIGURATION["URL"] = self.url_to_test # pylint: disable=no-member
self.url()
elif self.file_urls: # pylint: disable=no-member
PyFunceble.CONFIGURATION[
"no_whois"] = PyFunceble.CONFIGURATION[
"plain_list_domain"] = PyFunceble.CONFIGURATION[
"split"] = True
PyFunceble.CONFIGURATION["generate_hosts"] = False
self.url_file()
elif file_path:
self.file()
ExecutionTime("stop")
Percentage().log()
if domain:
self.colored_logo()
else:
PyFunceble.CONFIGURATION["simple"] = True
PyFunceble.CONFIGURATION["quiet"] = True
PyFunceble.CONFIGURATION["no_files"] = True
if domain:
PyFunceble.CONFIGURATION["domain"] = domain.lower()
def test_calculate(self, _):
"""
Test the calculation of the execution time.
"""
expected = PyFunceble.OrderedDict([("days", "00"), ("hours", "00"),
("minutes", "00"),
("seconds", "15")])
actual = ExecutionTime("stop")._calculate()
self.assertEqual(expected, actual)
def test_format_execution_time(self, calculate):
"""
This method test if the printed format is the one we want.
"""
calculate.return_value = PyFunceble.OrderedDict(
[("days", "01"), ("hours", "12"), ("minutes", "25"), ("seconds", "15")]
)
actual = ExecutionTime("start").format_execution_time()
expected = "01:12:25:15"
self.assertEqual(expected, actual)
def test_calculate_consequent(self, _):
"""
Test the calculation of the execution time for more consequent
time.
"""
day_in_second = 60 * 60 * 24
fifty_hours_in_second = 60 * 60 * 50
expected = PyFunceble.OrderedDict([("days", "03"), ("hours", "02"),
("minutes", "00"),
("seconds", "00")])
end = int(PyFunceble.time()) + day_in_second + fifty_hours_in_second
actual = ExecutionTime()._calculate(start=int(PyFunceble.time()),
end=end)
self.assertEqual(expected, actual)
def _file_decision(self, current, last, status=None):
"""
Manage the database, autosave and autocontinue systems for the case that we are reading
a file.
Arguments:
- status: str
The current status of current.
- current: str
The current domain or URL we are testing.
- last: str
The last domain or URL of the file we are testing.
"""
if status:
if not PyFunceble.CONFIGURATION[
"simple"] and PyFunceble.CONFIGURATION["file_to_test"]:
if PyFunceble.CONFIGURATION["inactive_database"]:
if status.lower() in PyFunceble.STATUS["list"]["up"]:
Database().remove()
else:
Database().add()
AutoContinue().backup()
if current != last:
AutoSave()
else:
ExecutionTime("stop")
Percentage().log()
self.reset_counters()
AutoContinue().backup()
self.colored_logo()
AutoSave(True)
for index in ["http_code", "referer"]:
if index in PyFunceble.CONFIGURATION:
PyFunceble.CONFIGURATION[index] = ""
def _file_list_to_test_filtering(self):
"""
Unify the way we work before testing file contents.
"""
# We get the list to test from the file we have to test.
list_to_test = self._extract_domain_from_file()
# We save the original list to test globally.
PyFunceble.INTERN["extracted_list_to_test"] = list_to_test
# We get the list of mined.
mined_list = Mining().list_of_mined()
if mined_list:
list_to_test.extend(mined_list)
# We generate the directory structure.
DirectoryStructure()
# We restore the data from the last session if it does exist.
AutoContinue().restore()
if PyFunceble.CONFIGURATION["adblock"]:
# The adblock decoder is activated.
# We get the list of domain to test (decoded).
list_to_test = AdBlock(list_to_test).decode()
else:
# The adblock decoder is not activated.
# We get the formatted list of domain to test.
list_to_test = list(map(self._format_domain, list_to_test))
# We clean the output directory if it is needed.
PyFunceble.Clean(list_to_test)
# We set the start time.
ExecutionTime("start")
# We get the list we have to test in the current session (from the database).
Inactive().to_test()
if (
PyFunceble.CONFIGURATION["inactive_database"]
and PyFunceble.INTERN["file_to_test"] in PyFunceble.INTERN["inactive_db"]
and "to_test"
in PyFunceble.INTERN["inactive_db"][PyFunceble.INTERN["file_to_test"]]
and PyFunceble.INTERN["inactive_db"][PyFunceble.INTERN["file_to_test"]][
"to_test"
]
):
# * The current file to test in into the database.
# and
# * The `to_test` index is present into the database
# related to the file we are testing.
# and
# * The `to_test` index content is not empty.
# We extend our list to test with the content of the `to_test` index
# of the current file database.
list_to_test.extend(
PyFunceble.INTERN["inactive_db"][PyFunceble.INTERN["file_to_test"]][
"to_test"
]
)
# We set a regex of element to delete.
# Understand with this variable that we don't want to test those.
regex_delete = r"localhost$|localdomain$|local$|broadcasthost$|0\.0\.0\.0$|allhosts$|allnodes$|allrouters$|localnet$|loopback$|mcastprefix$|ip6-mcastprefix$|ip6-localhost$|ip6-loopback$|ip6-allnodes$|ip6-allrouters$|ip6-localnet$" # pylint: disable=line-too-long
# We load the flatten version of the database.
PyFunceble.INTERN.update({"flatten_inactive_db": Inactive().content()})
# We initiate a local variable which will save the current state of the list.
not_filtered = list_to_test
try:
# We remove the element which are in the database from the
# current list to test.
list_to_test = List(
list(
set(Regex(list_to_test, regex_delete).not_matching_list())
- set(PyFunceble.INTERN["flatten_inactive_db"])
)
).format()
_ = list_to_test[-1]
except IndexError:
# We test without the database removing.
list_to_test = List(
Regex(not_filtered, regex_delete).not_matching_list()
).format()
# We delete the not_filtered variable.
del not_filtered
if PyFunceble.CONFIGURATION["filter"]:
# The filter is not empty.
# We get update our list to test. Indeed we only keep the elements which
# matches the given filter.
list_to_test = List(
Regex(
list_to_test, PyFunceble.CONFIGURATION["filter"], escape=False
).matching_list()
).format()
list_to_test = List(list(list_to_test)).custom_format(Sort.standard)
if PyFunceble.CONFIGURATION["hierarchical_sorting"]:
# The hierarchical sorting is desired by the user.
# We format the list.
list_to_test = List(list(list_to_test)).custom_format(Sort.hierarchical)
# We return the final list to test.
return list_to_test
def _file_decision(self, current, last, status=None):
"""
Manage the database, autosave and autocontinue systems for the case that we are reading
a file.
:param current: The currently tested element.
:type current: str
:param last: The last element of the list.
:type last: str
:param status: The status of the currently tested element.
:type status: str
"""
if (
status
and not PyFunceble.CONFIGURATION["simple"]
and PyFunceble.INTERN["file_to_test"]
):
# * The status is given.
# and
# * The simple mode is deactivated.
# and
# * A file to test is set.
# We run the mining logic.
Mining().process()
# We delete the currently tested element from the mining
# database.
# Indeed, as it is tested, it is already in our
# testing process which means that we don't need it into
# the mining database.
Mining().remove()
if (
status.lower() in PyFunceble.STATUS["list"]["up"]
or status.lower() in PyFunceble.STATUS["list"]["valid"]
):
# The status is in the list of up status.
if Inactive().is_present():
# The currently tested element is in the database.
# We generate the suspicious file(s).
Generate("strange").analytic_file(
"suspicious", PyFunceble.STATUS["official"]["up"]
)
# We remove the currently tested element from the
# database.
Inactive().remove()
else:
# The status is not in the list of up status.
# We add the currently tested element to the
# database.
Inactive().add()
# We backup the current state of the file reading
# for the case that we need to continue later.
AutoContinue().backup()
if current != last:
# The current element is not the last one.
# We run the autosave logic.
AutoSave()
else:
# The current element is the last one.
# We stop and log the execution time.
ExecutionTime("stop", True)
# We show/log the percentage.
Percentage().log()
# We reset the counters as we end the process.
self.reset_counters()
# We backup the current state of the file reading
# for the case that we need to continue later.
AutoContinue().backup()
# We show the colored logo.
self.colorify_logo()
# We save and stop the script if we are under
# Travis CI.
AutoSave(True)
for index in ["http_code", "referer"]:
# We loop through some configuration index we have to empty.
if index in PyFunceble.INTERN:
# The index is in the configuration.
# We empty the configuration index.
PyFunceble.INTERN[index] = ""
def _entry_management(self): # pylint: disable=too-many-branches
"""
Avoid to have 1 millions line into self.__init__()
"""
if not self.modulo_test: # pylint: disable=no-member
# We are not in a module usage.
# We set the file_path as the file we have to test.
PyFunceble.INTERN[
"file_to_test"
] = self.file_path # pylint: disable=no-member
# We check if the given file_path is an url.
# If it is an URL we update the file to test and download
# the given URL.
self._entry_management_url()
# We fix the environnement permissions.
AutoSave().travis_permissions()
# We check if we need to bypass the execution of PyFunceble.
self.bypass()
# We set the start time.
ExecutionTime("start")
if PyFunceble.CONFIGURATION["syntax"]:
# We are checking for syntax.
# We deactivate the http status code.
PyFunceble.HTTP_CODE["active"] = False
if self.domain_or_ip_to_test: # pylint: disable=no-member
# The given domain is not empty or None.
# We initiate a variable which will tell the system the type
# of the tested element.
PyFunceble.INTERN["to_test_type"] = "domain"
# We set the start time.
ExecutionTime("start")
# We deactivate the showing of percentage as we are in a single
# test run.
PyFunceble.CONFIGURATION["show_percentage"] = False
# We deactivate the whois database as it is not needed.
PyFunceble.CONFIGURATION["whois_database"] = False
if PyFunceble.CONFIGURATION["idna_conversion"]:
domain_or_ip_to_test = domain2idna(
self.domain_or_ip_to_test.lower() # pylint: disable=no-member
)
else:
domain_or_ip_to_test = (
self.domain_or_ip_to_test.lower() # pylint: disable=no-member
) # pylint: disable=no-member
# We test the domain after converting it to lower case.
self.domain(domain_or_ip_to_test)
elif self.url_to_test and not self.file_path: # pylint: disable=no-member
# An url to test is given and the file path is empty.
# We initiate a variable which will tell the system the type
# of the tested element.
PyFunceble.INTERN["to_test_type"] = "url"
# We set the start time.
ExecutionTime("start")
# We deactivate the showing of percentage as we are in a single
# test run.
PyFunceble.CONFIGURATION["show_percentage"] = False
# We test the url to test after converting it if needed (IDNA).
self.url(
Check().is_url_valid(
self.url_to_test, # pylint: disable=no-member
return_formatted=True,
)
)
elif (
self._entry_management_url_download(
self.url_file # pylint: disable=no-member
)
or self.url_file # pylint: disable=no-member
):
# * A file full of URL is given.
# or
# * the given file full of URL is a URL.
# * We deactivate the whois subsystem as it is not needed for url testing.
# * We activate the generation of plain list element.
# * We activate the generation of splited data instead of unified data.
PyFunceble.CONFIGURATION["no_whois"] = PyFunceble.CONFIGURATION[
"plain_list_domain"
] = PyFunceble.CONFIGURATION["split"] = True
# We deactivate the generation of hosts file as it is not relevant for
# url testing.
PyFunceble.CONFIGURATION["generate_hosts"] = False
# We initiate a variable which will tell the system the type
# of the tested element.
PyFunceble.INTERN["to_test_type"] = "url"
# And we test the given or the downloaded file.
self.file_url()
elif (
self._entry_management_url_download(
self.link_to_test # pylint: disable=no-member
)
or self._entry_management_url_download(
self.file_path # pylint: disable=no-member
) # pylint: disable=no-member
or self.file_path # pylint: disable=no-member
):
# * A file path is given.
# or
# * The given file path is an URL.
# or
# * A link to test is given.
# We initiate a variable which will tell the system the type
# of the tested element.
PyFunceble.INTERN["to_test_type"] = "domain"
# We test the given or the downloaded file.
self.file()
else:
# No file, domain, single url or file or url is given.
# We print a message on screen.
print(
PyFunceble.Fore.CYAN + PyFunceble.Style.BRIGHT + "Nothing to test."
)
# We stop and log the execution time.
ExecutionTime("stop", last=True)
# We log the current percentage state.
Percentage().log()
if (
self.domain_or_ip_to_test # pylint: disable=no-member
or self.url_to_test # pylint: disable=no-member
):
# We are testing a domain.
# We show the colored logo.
self.colorify_logo()
# We print our friendly message :)
PyFunceble.stay_safe()
else:
# We are used as an imported module.
# * We activate the simple mode as the table or any full
# details on screen are irrelevant.
# * We activate the quiet mode.
# And we deactivate the generation of files.
PyFunceble.CONFIGURATION["simple"] = PyFunceble.CONFIGURATION[
"quiet"
] = PyFunceble.CONFIGURATION["no_files"] = True
# * We deactivate the whois database as it is not needed.
# * We deactivate the database as it is not needed.
PyFunceble.CONFIGURATION["whois_database"] = PyFunceble.CONFIGURATION[
"inactive_database"
] = False
if self.domain_or_ip_to_test: # pylint: disable=no-member
# A domain is given.
# We initiate a variable which will tell the system the type
# of the tested element.
PyFunceble.INTERN["to_test_type"] = "domain"
# We set the domain to test.
PyFunceble.INTERN[
"to_test"
] = self.domain_or_ip_to_test.lower() # pylint: disable=no-member
elif self.url_to_test: # pylint: disable=no-member
# A url is given,
# We initiate a variable which will tell the system the type
# of the tested element.
PyFunceble.INTERN["to_test_type"] = "url"
# We set the url to test.
PyFunceble.INTERN[
"to_test"
] = self.url_to_test # pylint: disable=no-member
def __init__(
self,
domain_or_ip=None,
file_path=None,
link_to_test=None,
url_file_path=None,
url_to_test=None,
): # pylint: disable=too-many-branches
if domain_or_ip or file_path or link_to_test or url_file_path or url_to_test:
preset = PyFunceble.Preset()
CLICore.logs_sharing()
ExecutionTime("start")
if domain_or_ip:
SimpleCore(domain_or_ip).domain()
elif file_path:
PyFunceble.DirectoryStructure()
if PyFunceble.CONFIGURATION["multiprocess"]:
preset.maximal_processes()
preset.multiprocess()
FileMultiprocessCore(
file_path, "domain"
).read_and_test_file_content()
else:
FileCore(file_path, "domain").read_and_test_file_content()
elif link_to_test:
PyFunceble.DirectoryStructure()
if PyFunceble.CONFIGURATION["multiprocess"]:
preset.maximal_processes()
preset.multiprocess()
FileMultiprocessCore(
link_to_test, "domain"
).read_and_test_file_content()
else:
FileCore(link_to_test, "domain").read_and_test_file_content()
elif url_file_path:
PyFunceble.DirectoryStructure()
preset.file_url()
if PyFunceble.CONFIGURATION["multiprocess"]:
preset.maximal_processes()
preset.multiprocess()
FileMultiprocessCore(
url_file_path, "url"
).read_and_test_file_content()
else:
FileCore(url_file_path, "url").read_and_test_file_content()
elif url_to_test:
SimpleCore(url_to_test).url()
Percentage().log()
ExecutionTime("stop")
PyFunceble.CLICore.stay_safe()
else:
PyFunceble.CLICore.print_nothing_to_test()
