class TestInactiveDB(TestCase):
"""
Test PyFunceble.database.Inactive
"""
def setUp(self):
"""
Setup everything needed for the test
"""
PyFunceble.load_config(generate_directory_structure=False,
custom={"db_type": "json"})
self.file_to_test = "this_file_is_a_ghost"
self.file = (PyFunceble.CONFIG_DIRECTORY +
PyFunceble.OUTPUTS["default_files"]["inactive_db"])
self.expected_content = {
self.file_to_test: {
"0": {
"mÿethèrwallét.com":
PyFunceble.STATUS["official"]["invalid"],
"||google.com^": PyFunceble.STATUS["official"]["invalid"],
}
}
}
self.time_past = str(int(PyFunceble.time()) - (365 * 24 * 3600))
self.time_future = str(int(PyFunceble.time()) + (365 * 24 * 3600))
self.inactive_db = InactiveDB(self.file_to_test)
def test_file_not_exist(self):
"""
Test if everything is right with the generated
file.
"""
File(self.file).delete()
expected = False
actual = PyFunceble.path.isfile(self.file)
self.assertEqual(expected, actual)
def test_load_file_not_exist(self):
"""
Test the case that we want to load the database file that does not exist.
"""
self.test_file_not_exist()
self.inactive_db.load()
expected = {self.file_to_test: {}}
self.assertEqual(expected, self.inactive_db.database)
self.test_file_not_exist()
def test_load_file_exist(self):
"""
Test the case that we want to load a file that exist.
"""
self.test_file_not_exist()
Dict(self.expected_content).to_json(self.file)
self.inactive_db.database = {}
self.inactive_db.load()
self.assertEqual(self.expected_content, self.inactive_db.database)
self.test_file_not_exist()
def test_save(self):
"""
Test the saving of the inactive database.
"""
self.test_file_not_exist()
self.inactive_db.database = self.expected_content
self.inactive_db.save()
self.assertEqual(self.expected_content,
Dict().from_json(File(self.file).read()))
self.test_file_not_exist()
def test_initiate__path_not_exist(self): # pylint: disable=invalid-name
"""
Test Inactive.initiate() for the case that the path does not exist.
"""
self.test_file_not_exist()
self.inactive_db.database = {}
expected = {self.file_to_test: {}}
self.inactive_db.initiate()
self.assertEqual(expected, self.inactive_db.database)
self.test_file_not_exist()
def test_initiate__path_exist_time_past(self): # pylint: disable=invalid-name
"""
Test Inactive.initiate() for the case that the path exist but
the timestamp is in the past.
"""
self.test_file_not_exist()
new_time = str(
int(PyFunceble.time()) - self.inactive_db.one_day_in_seconds - 100)
self.inactive_db.database = {
self.file_to_test: {
self.time_past: {
"hello.world": PyFunceble.STATUS["official"]["down"],
"world.hello": PyFunceble.STATUS["official"]["down"],
},
"to_test": {
"github.com": ""
},
}
}
expected = {
self.file_to_test: {
self.time_past: {
"hello.world": PyFunceble.STATUS["official"]["down"],
"world.hello": PyFunceble.STATUS["official"]["down"],
},
new_time: {
"github.com": ""
},
}
}
self.inactive_db.save()
self.inactive_db.initiate()
self.assertEqual(expected, self.inactive_db.database)
self.test_file_not_exist()
def test_initiate__path_exist_time_future(self): # pylint: disable=invalid-name
"""
Test Inactive.initiate() for the case that the path exist but
the timestamp is in the future.
"""
self.test_file_not_exist()
self.inactive_db.database = {
self.file_to_test: {
self.time_future: {
"hello.world": PyFunceble.STATUS["official"]["down"],
"world.hello": PyFunceble.STATUS["official"]["down"],
}
}
}
expected = {
self.file_to_test: {
self.time_future: {
"hello.world": PyFunceble.STATUS["official"]["down"],
"world.hello": PyFunceble.STATUS["official"]["down"],
}
}
}
self.inactive_db.save()
self.inactive_db.initiate()
self.assertEqual(expected, self.inactive_db.database)
self.test_file_not_exist()
def test_timestamp_path_does_not_exit(self): # pylint: disable=invalid-name
"""
Test Inactive._timestamp() for the case that the path does
not exist but the time is in the past.
"""
self.test_file_not_exist()
self.inactive_db.database = {}
expected = int(PyFunceble.time())
actual = self.inactive_db._timestamp()
self.assertGreaterEqual(expected, actual)
self.test_file_not_exist()
def test_timestamp_path_exist_time_past(self): # pylint: disable=invalid-name
"""
Test Inactive._timestamp() for the case that the path exist but
the time is in the past.
"""
self.test_file_not_exist()
self.inactive_db.database = {
self.file_to_test: {
self.time_past: ["hello.world", "world.hello"]
}
}
expected = int(PyFunceble.time())
actual = self.inactive_db._timestamp()
self.assertGreaterEqual(expected, actual)
self.test_file_not_exist()
def test_timestamp_path_exist_time_future(self): # pylint: disable=invalid-name
"""
Test Inactive._timestamp() for the case that the path exist but
the time is in the future.
"""
self.test_file_not_exist()
self.inactive_db.database = {
self.file_to_test: {
self.time_future: ["hello.world", "world.hello"]
}
}
expected = int(self.time_future)
actual = self.inactive_db._timestamp()
self.assertEqual(expected, actual)
self.test_file_not_exist()
def test_add_path_does_not_exist(self): # pylint: disable=invalid-name
"""
Test Inactive.add() for the case that the path does not exist.
"""
self.test_file_not_exist()
self.inactive_db.database = {}
subject = "hello.world"
expected = {
self.file_to_test: {
str(self.inactive_db._timestamp()): {
"hello.world": PyFunceble.STATUS["official"]["down"]
}
}
}
self.inactive_db.add(subject, PyFunceble.STATUS["official"]["down"])
self.assertEqual(expected, self.inactive_db.database)
self.inactive_db.database = {}
subject = "http://hello.world"
expected = {
self.file_to_test: {
str(self.inactive_db._timestamp()): {
"http://hello.world": PyFunceble.STATUS["official"]["down"]
}
}
}
self.inactive_db.add(subject, PyFunceble.STATUS["official"]["down"])
self.assertEqual(expected, self.inactive_db.database)
self.test_file_not_exist()
def test_add_file_path_not_present(self): # pylint: disable=invalid-name
"""
Test Inactive.add() for the case that the path is not
present into the Inactive.
"""
self.test_file_not_exist()
timestamp = str(self.inactive_db._timestamp())
subject = "hello.world"
expected = {
self.file_to_test: {
timestamp: {
"hello.world": PyFunceble.STATUS["official"]["down"]
}
}
}
self.inactive_db.add(subject, PyFunceble.STATUS["official"]["down"])
self.assertEqual(expected, self.inactive_db.database)
self.test_file_not_exist()
def test_add_file_path_present(self): # pylint: disable=invalid-name
"""
Test Inactive.add() for the case that the path is present
into the Inactive.
"""
self.test_file_not_exist()
timestamp = str(self.inactive_db._timestamp())
subject = "hello.world"
expected = {
self.file_to_test: {
timestamp: {
"hello.world": PyFunceble.STATUS["official"]["down"],
"world.hello": PyFunceble.STATUS["official"]["down"],
}
}
}
self.inactive_db.database = {
self.file_to_test: {
timestamp: {
"world.hello": PyFunceble.STATUS["official"]["down"]
}
}
}
self.inactive_db.add(subject, PyFunceble.STATUS["official"]["down"])
self.assertEqual(expected, self.inactive_db.database)
self.test_file_not_exist()
def test_remove(self):
"""
Test Inactive.remove().
"""
timestamp = str(self.inactive_db._timestamp())
self.test_file_not_exist()
self.inactive_db.database = {
self.file_to_test: {
timestamp: {
"hello.world": PyFunceble.STATUS["official"]["down"]
}
}
}
subject = "hello.world"
expected = {self.file_to_test: {timestamp: {}}}
self.inactive_db.remove(subject)
self.assertEqual(expected, self.inactive_db.database)
self.test_file_not_exist()
def test_is_present(self):
"""
Test the presence of element in the databse.
"""
self.test_file_not_exist()
# Test of the case that everything goes right !
timestamp = str(self.inactive_db._timestamp())
self.inactive_db.database = {
self.file_to_test: {
timestamp: ["hello.world", "world.hello", "hello-world.com"]
}
}
subject = "hello.world"
expected = True
actual = subject in self.inactive_db
self.assertEqual(expected, actual)
del self.inactive_db.database[self.file_to_test][timestamp]
subject = "world.hello.world"
expected = False
actual = subject in self.inactive_db
self.assertEqual(expected, actual)
self.assertEqual(expected, actual)
self.test_file_not_exist()
def _test_line(self, line, manager_data=None): # pylint: disable=too-many-branches # pragma: no cover
"""
Given a line, we test it.
:param str line: A line to work with.
:param multiprocessing.Manager.list manager_data: A Server process.
"""
if PyFunceble.CONFIGURATION[
"db_type"] == "json" and manager_data is not None:
autocontinue = AutoContinue(self.file, parent_process=False)
inactive_db = InactiveDB(self.file)
mining = Mining(self.file)
else:
# We use the previously initiated autocontinue instance.
autocontinue = self.autocontinue
# We use the previously initiated inactive database instance.
inactive_db = self.inactive_db
# We use the previously initiated mining instance.
mining = self.mining
# We remove cariage from the given line.
line = line.strip()
if not line or line[0] == "#":
# We line is a comment line.
# We return None, there is nothing to test.
return None
if Regex(line, self.regex_ignore, escape=False,
return_data=False).match():
# The line match our list of elemenet
# to ignore.
# We return None, there is nothing to test.
return None
# We format the line, it's the last
# rush before starting to filter and test.
subject = self._format_line(line)
if (not PyFunceble.CONFIGURATION["local"]
and PyFunceble.Check(subject).is_reserved_ipv4()):
# * We are not testing for local components.
# and
# * The subject is a reserved IPv4.
# We return None, there is nothing to test.
return None
if PyFunceble.CONFIGURATION["filter"]:
# We have to filter.
if Regex(subject,
PyFunceble.CONFIGURATION["filter"],
return_data=False).match():
# The line match the given filter.
# We get the status of the current line.
status = self.__process_test(subject)
else:
# The line does not match the given filter.
# We return None.
return None
else:
# We do not have to filter.
# We get the status of the current line.
status = self.__process_test(subject)
# We add the line into the auto continue database.
autocontinue.add(subject, status)
if status.lower() in self.list_of_up_statuses:
# The status is in the list of UP status.
# We mine if necessary.
mining.mine(subject, self.file_type)
if subject in inactive_db:
# The subject is in the inactive database.
# We generate the suspicous file.
Generate(subject, "file_domain", PyFunceble.STATUS["official"]
["up"]).analytic_file("suspicious")
# And we remove the current subject from
# the inactive database.
inactive_db.remove(subject)
else:
# The status is not in the list of UP status.
# We add the current subject into the
# inactive database.
inactive_db.add(subject, status)
if (self.complements_test_started
and PyFunceble.CONFIGURATION["db_type"] == "json"):
# We started the test of the complements.
if "complements" in autocontinue.database:
# The complement index is present.
while subject in autocontinue.database["complements"]:
# We loop untill the line is not present into the
# database.
# We remove the currently tested element.
autocontinue.database["complements"].remove(subject)
# We save the current state.
autocontinue.save()
if manager_data is None:
# We are not in a multiprocess environment.
# We update the counters
autocontinue.update_counters()
# We process the autosaving if it is necessary.
self.autosave.process(test_completed=False)
elif PyFunceble.CONFIGURATION["db_type"] == "json":
# We are in a multiprocess environment.
# We save everything we initiated into the server process
manager_data.append({
"autocontinue": autocontinue.database,
"inactive_db": inactive_db.database,
"mining": mining.database,
})
# We return None.
return None
class APICore:
"""
Provide the logic and interface for the tests from the API.
:param str subject: The element we are testing.
:param bool complete:
Activate the return of a dictionnary with signigicant - if not all -
data about the test.
:param dict configuration:
The configuration to
"""
# The subject we are working with.
subject = None
# Tell us if we have to return all possible data.
complete = False
# Saves the configuration.
configuration = None
def __init__(self, subject, complete=False, configuration=None):
# We share the subject.
self.subject = subject
# We share the complete option.
self.complete = complete
# We share the configuration.
self.configuration = configuration
# We load the global configuration
# if it was not alreay done.
PyFunceble.load_config(generate_directory_structure=False,
custom=self.configuration)
if ("api_file_generation" in PyFunceble.CONFIGURATION
and PyFunceble.CONFIGURATION["api_file_generation"]):
PyFunceble.load_config(generate_directory_structure=True)
# We update the configuration with the given
# configuration.
PyFunceble.Preset().api()
# We get an instance of the DB connection.
self.sqlite_db = SQLite()
self.mysql_db = MySQL()
# We create an instance of the whois database.
self.whois_db = WhoisDB(sqlite_db=self.sqlite_db,
mysql_db=self.mysql_db)
# We create an instance of the inactive database.
self.inactive_db = InactiveDB("api_call",
sqlite_db=self.sqlite_db,
mysql_db=self.mysql_db)
def __inactive_database_management(self, subject, status):
"""
Given the subject and status, we add or remove the subject
from the inactive database.
"""
if self.inactive_db.authorized:
# We are authorized to operate with the
# inactive database.s
if status.lower() in PyFunceble.STATUS["list"]["up"]:
# The status is in the list of UP status.
# We remove it from the database.
self.inactive_db.remove(subject)
else:
# The status is not in the list of UP status.
# We add it into the database.
self.inactive_db.add(subject, status)
def domain_and_ip(self):
"""
Run a domain/IP avaibility check over the given subject.
"""
if isinstance(self.subject, list):
# The given subject is a list of subjects.
# We initiate a variable which save our result.
result = {}
for subject in self.subject:
# We loop through the list of subject.
# We get the complete data related to the status
# of the subject.
data = Status(subject,
subject_type="domain",
whois_db=self.whois_db).get()
if self.complete:
# The user want a copy of the complete data.
# We set it.
result[subject] = data
else:
# The user do not want a copy of the complete data.
# We only set the status.
result[subject] = data["status"]
self.__inactive_database_management(subject, data["status"])
# We return our local result.
return result
# We get the status of the given subject.
data = Status(self.subject,
subject_type="domain",
whois_db=self.whois_db).get()
self.__inactive_database_management(self.subject, data["status"])
if self.complete:
# The user want a copy of the compelte data.
# We return them
return data
# We only return the status.
return data["status"]
def domain_syntax(self):
"""
Run a domain syntax check over the given subject.
"""
if isinstance(self.subject, list):
# The given subject is a list of subject.
# We return the validity of each subjects.
return {
subject: PyFunceble.Check(subject).is_domain()
for subject in self.subject
}
# We return the validity of the the given subject.
return PyFunceble.Check(self.subject).is_domain()
def subdomain_syntax(self):
"""
Run a subdomain syntax check over the given subject.
"""
if isinstance(self.subject, list):
# The given subjet is a list of subject.
# We return the validity of each subjects.
return {
subject: PyFunceble.Check(subject).is_subdomain()
for subject in self.subject
}
# We return the validity of the given subject.
return PyFunceble.Check(self.subject).is_subdomain()
def ipv4_syntax(self):
"""
Run an IPv4 syntax check over the given subject.
"""
if isinstance(self.subject, list):
# The given subjet is a list of subject.
# We return the validity of each subjects.
return {
subject: PyFunceble.Check(subject).is_ipv4()
for subject in self.subject
}
# We return the validity of the given subject.
return PyFunceble.Check(self.subject).is_ipv4()
def ipv4_range_syntax(self):
"""
Run an IPv4 range syntax check over the given subject.
"""
if isinstance(self.subject, list):
# The given subjet is a list of subject.
# We return the validity of each subjects.
return {
subject: PyFunceble.Check(subject).is_ipv4_range()
for subject in self.subject
}
# We return the validity of the given subject.
return PyFunceble.Check(self.subject).is_ipv4_range()
def url(self):
"""
Run an URL avaibility check over the given subject.
"""
if isinstance(self.subject, list):
# The given subjet is a list of subject.
# We initiate a local variable which will save
# what we are going to return.
result = {}
for subject in self.subject:
# We loop through the list of subjects.
# We get the complete data about the status.
data = URLStatus(subject, subject_type="url").get()
if self.complete:
# The user want a complete copy of the data.
# We set it.
result[subject] = data
else:
# The user does not want a complete copy of the data.
# We only set the status.
result[subject] = data["status"]
self.__inactive_database_management(subject, data["status"])
# We return the result of each subjects.
return result
# We get the complete data about the status of the subject.
data = URLStatus(self.subject, subject_type="url").get()
self.__inactive_database_management(self.subject, data["status"])
if self.complete:
# The user want a complete copy of the data.
# We return them.
return data
# We return the result of each subjects.
return data["status"]
def url_syntax(self):
"""
Run an IPv4 syntax check over the given subject.
"""
if isinstance(self.subject, list):
# The given subjet is a list of subject.
# We return the validity of each subjects.
return {
subject: PyFunceble.Check(subject).is_url()
for subject in self.subject
}
# We return the validity of the subject.
return PyFunceble.Check(self.subject).is_url()