def __handle_http_code(self, previous_state):
"""
Handle the HTTP Code status escalation.
:param str previous_state: The previously catched status.
:return:
:code:`(new status, new source)` or :code:`None` if there is any
change to apply.
:rtype: tuple|None
"""
try:
if self.status_code in PyFunceble.HTTP_CODE["list"]["up"]:
# The extracted http code is in the list of up codes.
# We generate the analytics files.
Generate(self.subject, self.subject_type,
previous_state).analytic_file(
PyFunceble.STATUS["official"]["up"])
if previous_state.lower(
) not in PyFunceble.STATUS["list"]["up"]:
# And we return the new status and source
return self.__http_status_code_up()
if self.status_code in PyFunceble.HTTP_CODE["list"][
"potentially_up"]:
# The extracted http status code is in the list of potentially up status.
# We generate the analytics files.
Generate(self.subject, self.subject_type,
previous_state).analytic_file("potentially_up")
if previous_state.lower(
) not in PyFunceble.STATUS["list"]["up"]:
# And we return the new status and source
return self.__http_status_code_up()
if (previous_state.lower() not in PyFunceble.STATUS["list"]["down"]
or previous_state.lower() not in PyFunceble.STATUS["list"]
["invalid"]) and self.status_code in PyFunceble.HTTP_CODE[
"list"]["potentially_down"]:
# The extracted http code is in the list of potentially down status code.
# We generate the analytics files.
Generate(self.subject, self.subject_type,
previous_state).analytic_file("potentially_down")
except KeyError:
pass
# We return None, there is no changes.
return None
def handle(self):
"""
Handle the backend of the given status.
"""
source = "URL"
if self.catched.lower() not in PyFunceble.STATUS["list"]["invalid"]:
Generate(self.catched, source).status_file()
else:
Generate(self.catched, "IANA").status_file()
return self.catched
def handle(self):
"""
Handle the lack of WHOIS. :smile_cat:
:return:
The strus of the domain after generating the files desired
by the user.
:rtype: str
"""
if self.matched_status.lower(
) not in PyFunceble.STATUS["list"]["invalid"]:
# The matched status is not in the list of invalid status.
# We initiate the source we are going to parse to the Generate class.
source = "NSLOOKUP"
if Lookup().nslookup():
# We could execute the nslookup logic.
# We generate the status files with the up status.
Generate(PyFunceble.STATUS["official"]["up"],
source).status_file()
# We return the up status.
return PyFunceble.STATUS["official"]["up"]
# We could not execute the nslookup logic.
# * We generate the status file with the down status.
Generate(PyFunceble.STATUS["official"]["down"],
source).status_file()
# We return the down status.
return PyFunceble.STATUS["official"]["down"]
# The matched status is in the list of invalid status.
# We generate the status file with the invalid status.
Generate(PyFunceble.STATUS["official"]["invalid"],
self.invalid_source).status_file()
# We return the invalid status.
return PyFunceble.STATUS["official"]["invalid"]
def handle(self):
"""
Handle the backend of the given status.
"""
# We generate the status file with the catched status.
Generate(self.catched, "SYNTAX").status_file()
# We return the parsed status.
return self.catched
def __handle_potentially_inactive(self, previous_state):
"""
Handle the potentially inactive case.
:param previous_state: The previously catched status.
:type previous_state: str
:return:
:code:`(new status, new source)` or :code:`None` if there is any
change to apply.
:rtype: tuple|None
"""
if (
PyFunceble.HTTP_CODE["active"]
and PyFunceble.INTERN["http_code"]
in PyFunceble.HTTP_CODE["list"]["potentially_down"]
):
# * The http status request is activated.
# and
# * The extracted http status code is in the list of
# potentially down list.
# We generate the analytics files.
Generate(domain_status=previous_state).analytic_file("potentially_down")
if not PyFunceble.CONFIGURATION["no_special"]:
# We are authorized to play with the SPEICIAL rules.
for regx in self.regexes_active_to_inactive_potentially_down:
# We loop through the list of available regex.
if Regex(
data=PyFunceble.INTERN["to_test"],
regex=regx,
return_data=False,
escape=False,
).match():
# The element we are currently testing match the
# regex we are currently reading.
# We get the output of the function associated
# with the regex.
output = self.regexes_active_to_inactive_potentially_down[
regx
]()
if output is not None:
# The output is not None.
# We return the new source and state.
return output
# We return None, there is no changes.
return None
def handle(self):
"""
Handle the backend of the given status.
"""
# We initiate the source we are going to parse to the Generate class.
source = "URL"
if self.catched.lower() not in PyFunceble.STATUS["list"]["invalid"]:
# The parsed status is not in the list of invalid.
# We generate the status file with the catched status.
Generate(self.catched, source).status_file()
else:
# The parsed status is in the list of invalid.
# We generate the status file with the parsed status.
Generate(self.catched, "SYNTAX").status_file()
# We return the parsed status.
return self.catched
def handle(self):
"""
Handle the backend of the found status.
"""
# We generate the status file with the catched status.
Generate(
self.subject,
self.subject_type,
self.output["status"],
source=self.output["status_source"],
filename=self.filename,
ip_validation=self.output["ipv4_syntax_validation"],
).status_file()
def handle(self):
"""
Handle the backend of the given status.
"""
# We generate the status file with the catched status.
Generate(
self.subject,
self.subject_type,
self.output["status"],
source=self.output["status_source"],
http_status_code=self.output["http_status_code"],
filename=self.filename,
).status_file()
def handle(self):
"""
Handle the lack of WHOIS. :)
Returns: str
The status of the domains after generating the files.
"""
source = "NSLOOKUP"
if self.matched_status.lower(
) not in PyFunceble.STATUS["list"]["invalid"]:
if Lookup().nslookup():
Generate(PyFunceble.STATUS["official"]["up"],
source).status_file()
return PyFunceble.STATUS["official"]["up"]
Generate(PyFunceble.STATUS["official"]["down"],
source).status_file()
return PyFunceble.STATUS["official"]["down"]
Generate(PyFunceble.STATUS["official"]["invalid"],
"IANA").status_file()
return PyFunceble.STATUS["official"]["invalid"]
def domain(self, subject): # pragma: no cover
"""
Handle the test of a single domain.
:param str subject: The subject we are testing.
"""
if subject:
# The given subject is not empty nor None.
if PyFunceble.CONFIGURATION["syntax"]:
# The syntax mode is activated.
# We get the status from SyntaxStatus.
status = SyntaxStatus(subject,
subject_type="file_domain",
filename=self.file).get()["status"]
else:
# We test and get the status of the domain.
status = Status(
subject,
subject_type="file_domain",
filename=self.file,
whois_db=self.whois_db,
inactive_db=self.inactive_db,
).get()["status"]
if PyFunceble.CONFIGURATION["simple"]:
# The simple mode is activated.
# We print the domain and the status.
print("{0} {1}".format(
self.get_simple_coloration(status) + subject, status))
if self.complements_test_started:
# We started to test the complements.
# We generate the complement file(s).
Generate(subject, "file_domain", status).complements_file()
# We return the status.
return status
# We return None, there is nothing to test.
return None
def _test_line(self, line, manager_data=None): # pylint: disable=too-many-branches # pragma: no cover
"""
Given a line, we test it.
:param str line: A line to work with.
:param multiprocessing.Manager.list manager_data: A Server process.
"""
if PyFunceble.CONFIGURATION[
"db_type"] == "json" and manager_data is not None:
autocontinue = AutoContinue(self.file, parent_process=False)
inactive_db = InactiveDB(self.file)
mining = Mining(self.file)
else:
# We use the previously initiated autocontinue instance.
autocontinue = self.autocontinue
# We use the previously initiated inactive database instance.
inactive_db = self.inactive_db
# We use the previously initiated mining instance.
mining = self.mining
# We remove cariage from the given line.
line = line.strip()
if not line or line[0] == "#":
# We line is a comment line.
# We return None, there is nothing to test.
return None
if Regex(line, self.regex_ignore, escape=False,
return_data=False).match():
# The line match our list of elemenet
# to ignore.
# We return None, there is nothing to test.
return None
# We format the line, it's the last
# rush before starting to filter and test.
subject = self._format_line(line)
if (not PyFunceble.CONFIGURATION["local"]
and PyFunceble.Check(subject).is_reserved_ipv4()):
# * We are not testing for local components.
# and
# * The subject is a reserved IPv4.
# We return None, there is nothing to test.
return None
if PyFunceble.CONFIGURATION["filter"]:
# We have to filter.
if Regex(subject,
PyFunceble.CONFIGURATION["filter"],
return_data=False).match():
# The line match the given filter.
# We get the status of the current line.
status = self.__process_test(subject)
else:
# The line does not match the given filter.
# We return None.
return None
else:
# We do not have to filter.
# We get the status of the current line.
status = self.__process_test(subject)
# We add the line into the auto continue database.
autocontinue.add(subject, status)
if status.lower() in self.list_of_up_statuses:
# The status is in the list of UP status.
# We mine if necessary.
mining.mine(subject, self.file_type)
if subject in inactive_db:
# The subject is in the inactive database.
# We generate the suspicous file.
Generate(subject, "file_domain", PyFunceble.STATUS["official"]
["up"]).analytic_file("suspicious")
# And we remove the current subject from
# the inactive database.
inactive_db.remove(subject)
else:
# The status is not in the list of UP status.
# We add the current subject into the
# inactive database.
inactive_db.add(subject, status)
if (self.complements_test_started
and PyFunceble.CONFIGURATION["db_type"] == "json"):
# We started the test of the complements.
if "complements" in autocontinue.database:
# The complement index is present.
while subject in autocontinue.database["complements"]:
# We loop untill the line is not present into the
# database.
# We remove the currently tested element.
autocontinue.database["complements"].remove(subject)
# We save the current state.
autocontinue.save()
if manager_data is None:
# We are not in a multiprocess environment.
# We update the counters
autocontinue.update_counters()
# We process the autosaving if it is necessary.
self.autosave.process(test_completed=False)
elif PyFunceble.CONFIGURATION["db_type"] == "json":
# We are in a multiprocess environment.
# We save everything we initiated into the server process
manager_data.append({
"autocontinue": autocontinue.database,
"inactive_db": inactive_db.database,
"mining": mining.database,
})
# We return None.
return None
def _file_decision(self, current, last, status=None):
"""
Manage the database, autosave and autocontinue systems for the case that we are reading
a file.
:param current: The currently tested element.
:type current: str
:param last: The last element of the list.
:type last: str
:param status: The status of the currently tested element.
:type status: str
"""
if (
status
and not PyFunceble.CONFIGURATION["simple"]
and PyFunceble.INTERN["file_to_test"]
):
# * The status is given.
# and
# * The simple mode is deactivated.
# and
# * A file to test is set.
# We run the mining logic.
Mining().process()
# We delete the currently tested element from the mining
# database.
# Indeed, as it is tested, it is already in our
# testing process which means that we don't need it into
# the mining database.
Mining().remove()
if (
status.lower() in PyFunceble.STATUS["list"]["up"]
or status.lower() in PyFunceble.STATUS["list"]["valid"]
):
# The status is in the list of up status.
if Inactive().is_present():
# The currently tested element is in the database.
# We generate the suspicious file(s).
Generate("strange").analytic_file(
"suspicious", PyFunceble.STATUS["official"]["up"]
)
# We remove the currently tested element from the
# database.
Inactive().remove()
else:
# The status is not in the list of up status.
# We add the currently tested element to the
# database.
Inactive().add()
# We backup the current state of the file reading
# for the case that we need to continue later.
AutoContinue().backup()
if current != last:
# The current element is not the last one.
# We run the autosave logic.
AutoSave()
else:
# The current element is the last one.
# We stop and log the execution time.
ExecutionTime("stop", True)
# We show/log the percentage.
Percentage().log()
# We reset the counters as we end the process.
self.reset_counters()
# We backup the current state of the file reading
# for the case that we need to continue later.
AutoContinue().backup()
# We show the colored logo.
self.colorify_logo()
# We save and stop the script if we are under
# Travis CI.
AutoSave(True)
for index in ["http_code", "referer"]:
# We loop through some configuration index we have to empty.
if index in PyFunceble.INTERN:
# The index is in the configuration.
# We empty the configuration index.
PyFunceble.INTERN[index] = ""
def handle(cls, status, invalid_source="IANA"):
"""
Handle the lack of WHOIS and expiration date. :smile_cat:
:param matched_status: The status that we have to handle.
:type status: str
:param invalid_source:
The source to set when we handle INVALID element.
:type invalid_source: str
:return:
The strus of the domain after generating the files desired
by the user.
:rtype: str
"""
if status.lower() not in PyFunceble.STATUS["list"]["invalid"]:
# The matched status is not in the list of invalid status.
# We initiate the source we are going to parse to the Generate class.
source = "NSLOOKUP"
if Lookup().nslookup():
# We could execute the nslookup logic.
# We get the status and source after extra rules check.
status, source = cls.extra_rules.handle(
PyFunceble.STATUS["official"]["up"], source
)
# We generate the status files with the up status.
Generate(status, source).status_file()
# We return the up status.
return status, source
# We could not execute the nslookup logic.
# We get the status and source after extra rules check.
status, source = cls.extra_rules.handle(
PyFunceble.STATUS["official"]["down"], source
)
# We generate the status file with the down status.
Generate(status, source).status_file()
# We return the down status.
return status, source
# The matched status is in the list of invalid status.
# We get the status and source after extra rules check.
status, source = cls.extra_rules.handle(
PyFunceble.STATUS["official"]["invalid"], invalid_source
)
# We generate the status file with the invalid status.
Generate(status, source).status_file()
# We return the status.
return status, source
def handle(self, status, ip_validation_status):
"""
Handle the lack of WHOIS and expiration date. :smile_cat:
:param str matched_status: The status that we have to handle.
:param str ip_validation_status:
The IP syntax validation.
:return:
The status of the domain after generating the files desired
by the user.
:rtype: str
"""
# We get the dns_lookup state.
self.output["dns_lookup"] = PyFunceble.DNSLookup(
self.subject,
dns_server=PyFunceble.CONFIGURATION["dns_server"]).request()
if status.lower() not in PyFunceble.STATUS["list"]["invalid"]:
# The matched status is not in the list of invalid status.
if self.output["dns_lookup"]:
# We could execute the dns_lookup logic.
# We set the status we got.
self.output["_status"] = PyFunceble.STATUS["official"]["up"]
else:
# We could not get something.
# We set the status we got.
self.output["_status"] = PyFunceble.STATUS["official"]["down"]
# We get the status and source after extra rules check.
self.output["status"], self.output["status_source"] = ExtraRules(
self.subject, self.subject_type,
self.output["http_status_code"]).handle(
self.output["_status"], self.output["_status_source"])
else:
if self.output["dns_lookup"]:
# We could execute the dns_lookup logic.
# We set the status we got.
self.output["_status"] = PyFunceble.STATUS["official"]["up"]
# We set the status source.
self.output["_status_source"] = "DNSLOOKUP"
self.output["status"], self.output["status_source"] = (
self.output["_status"],
self.output["_status_source"],
)
# We generate the status file with the invalid status.
Generate(
self.subject,
self.subject_type,
self.output["status"],
source=self.output["status_source"],
expiration_date=self.output["expiration_date"],
http_status_code=self.output["http_status_code"],
whois_server=self.output["whois_server"],
filename=self.filename,
ip_validation=ip_validation_status,
).status_file()
def get(self):
"""
Get the status while testing for an IP or domain.
"""
if self.subject:
self.output.update({
"domain_syntax_validation":
self.checker.is_domain(),
"expiration_date":
None,
"http_status_code":
"***",
"ipv4_range_syntax_validation":
self.checker.is_ipv4_range(),
"ipv4_syntax_validation":
self.checker.is_ipv4(),
"subdomain_syntax_validation":
self.checker.is_subdomain(),
"tested":
self.subject,
"url_syntax_validation":
self.checker.is_url(),
"whois_server":
Referer(self.subject).get(),
})
if PyFunceble.CONFIGURATION["local"] or (
self.output["domain_syntax_validation"]
or self.output["ipv4_syntax_validation"]):
self.output["http_status_code"] = HTTPCode(
self.subject, self.subject_type).get()
if not self.output["subdomain_syntax_validation"]:
self.output["expiration_date"], self.output[
"whois_record"] = ExpirationDate(
self.subject,
self.output["whois_server"],
whois_db=self.whois_db,
).get()
if isinstance(self.output["expiration_date"], str):
self.output["_status_source"] = self.output[
"status_source"] = "WHOIS"
self.output["_status"] = self.output[
"status"] = PyFunceble.STATUS["official"]["up"]
Generate(
self.subject,
self.subject_type,
self.output["status"],
source=self.output["status_source"],
expiration_date=self.output["expiration_date"],
http_status_code=self.output["http_status_code"],
whois_server=self.output["whois_server"],
filename=self.filename,
ip_validation=self.
output["ipv4_syntax_validation"],
).status_file()
else:
self.output["_status_source"] = "DNSLOOKUP"
self.handle(
status="inactive",
ip_validation_status=self.
output["ipv4_syntax_validation"],
)
else:
self.output["_status_source"] = "DNSLOOKUP"
self.handle(
status="inactive",
ip_validation_status=self.
output["ipv4_syntax_validation"],
)
else:
self.output["_status_source"] = "SYNTAX"
self.output["_status"] = PyFunceble.STATUS["official"][
"invalid"]
self.handle(
status="invalid",
ip_validation_status=self.output["ipv4_syntax_validation"],
)
return self.output
raise ValueError("Subject should be given.")
def _extract(self): # pragma: no cover
"""
Extract the expiration date from the whois record.
:return: The status of the domain.
:rtype: str
"""
# We try to get the expiration date from the database.
expiration_date_from_database = Whois().get_expiration_date()
if expiration_date_from_database:
# The hash of the current whois record did not changed and the
# expiration date from the database is not empty not equal to
# None or False.
# We generate the files and print the status.
# It's an active element!
Generate(
PyFunceble.STATUS["official"]["up"],
"WHOIS",
expiration_date_from_database,
).status_file()
# We handle und return the official up status.
return PyFunceble.STATUS["official"]["up"]
# We get the whois record.
self.whois_record = Lookup().whois(PyFunceble.INTERN["referer"])
# We list the list of regex which will help us get an unformatted expiration date.
to_match = [
r"expire:(.*)",
r"expire on:(.*)",
r"Expiry Date:(.*)",
r"free-date(.*)",
r"expires:(.*)",
r"Expiration date:(.*)",
r"Expiry date:(.*)",
r"Expire Date:(.*)",
r"renewal date:(.*)",
r"Expires:(.*)",
r"validity:(.*)",
r"Expiration Date :(.*)",
r"Expiry :(.*)",
r"expires at:(.*)",
r"domain_datebilleduntil:(.*)",
r"Data de expiração \/ Expiration Date \(dd\/mm\/yyyy\):(.*)",
r"Fecha de expiración \(Expiration date\):(.*)",
r"\[Expires on\](.*)",
r"Record expires on(.*)(\(YYYY-MM-DD\))",
r"status: OK-UNTIL(.*)",
r"renewal:(.*)",
r"expires............:(.*)",
r"expire-date:(.*)",
r"Exp date:(.*)",
r"Valid-date(.*)",
r"Expires On:(.*)",
r"Fecha de vencimiento:(.*)",
r"Expiration:.........(.*)",
r"Fecha de Vencimiento:(.*)",
r"Registry Expiry Date:(.*)",
r"Expires on..............:(.*)",
r"Expiration Time:(.*)",
r"Expiration Date:(.*)",
r"Expired:(.*)",
r"Date d'expiration:(.*)",
]
if self.whois_record:
# The whois record is not empty.
if "current_test_data" in PyFunceble.INTERN:
# The end-user want more information whith his test.
# We update the whois_record index.
PyFunceble.INTERN["current_test_data"][
"whois_record"] = self.whois_record
for string in to_match:
# We loop through the list of regex.
# We try tro extract the expiration date from the WHOIS record.
expiration_date = Regex(self.whois_record,
string,
return_data=True,
rematch=True,
group=0).match()
if expiration_date:
# The expiration date could be extracted.
# We get the extracted expiration date.
self.expiration_date = expiration_date[0].strip()
# We initate a regex which will help us know if a number
# is present into the extracted expiration date.
regex_rumbers = r"[0-9]"
if Regex(self.expiration_date,
regex_rumbers,
return_data=False).match():
# The extracted expiration date has a number.
# We format the extracted expiration date.
self.expiration_date = self._format()
if (self.expiration_date and not Regex(
self.expiration_date,
r"[0-9]{2}\-[a-z]{3}\-2[0-9]{3}",
return_data=False,
).match()):
# The formatted expiration date does not match our unified format.
# We log the problem.
Logs().expiration_date(self.expiration_date)
# We log the whois record.
Logs().whois(self.whois_record)
if "current_test_data" in PyFunceble.INTERN:
# The end-user want more information whith his test.
# We update the expiration_date index.
PyFunceble.INTERN["current_test_data"][
"expiration_date"] = self.expiration_date
# We generate the files and print the status.
# It's an active element!
Generate(
PyFunceble.STATUS["official"]["up"],
"WHOIS",
self.expiration_date,
).status_file()
# We log the whois record.
Logs().whois(self.whois_record)
# We save the whois record into the database.
Whois(expiration_date=self.expiration_date).add()
# We handle und return the official up status.
return PyFunceble.STATUS["official"]["up"]
# The extracted expiration date does not have a number.
# We log the whois record.
Logs().whois(self.whois_record)
# We handle and return and h the official down status.
return Status(
PyFunceble.STATUS["official"]["down"]).handle()
# The whois record is empty.
# We handle and return the official down status.
return Status(PyFunceble.STATUS["official"]["down"]).handle()
def _extract(self): # pragma: no cover
"""
Extract the expiration date from the whois record.
"""
self.whois_record = Lookup().whois(PyFunceble.CONFIGURATION["referer"])
to_match = [
r"expire:(.*)",
r"expire on:(.*)",
r"Expiry Date:(.*)",
r"free-date(.*)",
r"expires:(.*)",
r"Expiration date:(.*)",
r"Expiry date:(.*)",
r"Expire Date:(.*)",
r"renewal date:(.*)",
r"Expires:(.*)",
r"validity:(.*)",
r"Expiration Date :(.*)",
r"Expiry :(.*)",
r"expires at:(.*)",
r"domain_datebilleduntil:(.*)",
r"Data de expiração \/ Expiration Date \(dd\/mm\/yyyy\):(.*)",
r"Fecha de expiración \(Expiration date\):(.*)",
r"\[Expires on\](.*)",
r"Record expires on(.*)(\(YYYY-MM-DD\))",
r"status: OK-UNTIL(.*)",
r"renewal:(.*)",
r"expires............:(.*)",
r"expire-date:(.*)",
r"Exp date:(.*)",
r"Valid-date(.*)",
r"Expires On:(.*)",
r"Fecha de vencimiento:(.*)",
r"Expiration:.........(.*)",
r"Fecha de Vencimiento:(.*)",
r"Registry Expiry Date:(.*)",
r"Expires on..............:(.*)",
r"Expiration Time:(.*)",
r"Expiration Date:(.*)",
r"Expired:(.*)",
r"Date d'expiration:(.*)",
]
if self.whois_record:
for string in to_match:
expiration_date = Regex(
self.whois_record, string, return_data=True, rematch=True, group=0
).match()
if expiration_date:
self.expiration_date = expiration_date[0].strip()
regex_rumbers = r"[0-9]"
if Regex(
self.expiration_date, regex_rumbers, return_data=False
).match():
self.expiration_date = self._format()
if self.expiration_date and not Regex(
self.expiration_date,
r"[0-9]{2}\-[a-z]{3}\-2[0-9]{3}",
return_data=False,
).match():
self.log()
self._whois_log()
Generate(
PyFunceble.STATUS["official"]["up"],
"WHOIS",
self.expiration_date,
).status_file()
self._whois_log()
return PyFunceble.STATUS["official"]["up"]
self._whois_log()
return Status(PyFunceble.STATUS["official"]["down"]).handle()
self._whois_log()
return Status(PyFunceble.STATUS["official"]["down"]).handle()
