def __handle_http_code(self, previous_state): """ Handle the HTTP Code status escalation. :param str previous_state: The previously catched status. :return: :code:`(new status, new source)` or :code:`None` if there is any change to apply. :rtype: tuple|None """ try: if self.status_code in PyFunceble.HTTP_CODE["list"]["up"]: # The extracted http code is in the list of up codes. # We generate the analytics files. Generate(self.subject, self.subject_type, previous_state).analytic_file( PyFunceble.STATUS["official"]["up"]) if previous_state.lower( ) not in PyFunceble.STATUS["list"]["up"]: # And we return the new status and source return self.__http_status_code_up() if self.status_code in PyFunceble.HTTP_CODE["list"][ "potentially_up"]: # The extracted http status code is in the list of potentially up status. # We generate the analytics files. Generate(self.subject, self.subject_type, previous_state).analytic_file("potentially_up") if previous_state.lower( ) not in PyFunceble.STATUS["list"]["up"]: # And we return the new status and source return self.__http_status_code_up() if (previous_state.lower() not in PyFunceble.STATUS["list"]["down"] or previous_state.lower() not in PyFunceble.STATUS["list"] ["invalid"]) and self.status_code in PyFunceble.HTTP_CODE[ "list"]["potentially_down"]: # The extracted http code is in the list of potentially down status code. # We generate the analytics files. Generate(self.subject, self.subject_type, previous_state).analytic_file("potentially_down") except KeyError: pass # We return None, there is no changes. return None
def handle(self): """ Handle the backend of the given status. """ source = "URL" if self.catched.lower() not in PyFunceble.STATUS["list"]["invalid"]: Generate(self.catched, source).status_file() else: Generate(self.catched, "IANA").status_file() return self.catched
def handle(self): """ Handle the lack of WHOIS. :smile_cat: :return: The strus of the domain after generating the files desired by the user. :rtype: str """ if self.matched_status.lower( ) not in PyFunceble.STATUS["list"]["invalid"]: # The matched status is not in the list of invalid status. # We initiate the source we are going to parse to the Generate class. source = "NSLOOKUP" if Lookup().nslookup(): # We could execute the nslookup logic. # We generate the status files with the up status. Generate(PyFunceble.STATUS["official"]["up"], source).status_file() # We return the up status. return PyFunceble.STATUS["official"]["up"] # We could not execute the nslookup logic. # * We generate the status file with the down status. Generate(PyFunceble.STATUS["official"]["down"], source).status_file() # We return the down status. return PyFunceble.STATUS["official"]["down"] # The matched status is in the list of invalid status. # We generate the status file with the invalid status. Generate(PyFunceble.STATUS["official"]["invalid"], self.invalid_source).status_file() # We return the invalid status. return PyFunceble.STATUS["official"]["invalid"]
def handle(self): """ Handle the backend of the given status. """ # We generate the status file with the catched status. Generate(self.catched, "SYNTAX").status_file() # We return the parsed status. return self.catched
def __handle_potentially_inactive(self, previous_state): """ Handle the potentially inactive case. :param previous_state: The previously catched status. :type previous_state: str :return: :code:`(new status, new source)` or :code:`None` if there is any change to apply. :rtype: tuple|None """ if ( PyFunceble.HTTP_CODE["active"] and PyFunceble.INTERN["http_code"] in PyFunceble.HTTP_CODE["list"]["potentially_down"] ): # * The http status request is activated. # and # * The extracted http status code is in the list of # potentially down list. # We generate the analytics files. Generate(domain_status=previous_state).analytic_file("potentially_down") if not PyFunceble.CONFIGURATION["no_special"]: # We are authorized to play with the SPEICIAL rules. for regx in self.regexes_active_to_inactive_potentially_down: # We loop through the list of available regex. if Regex( data=PyFunceble.INTERN["to_test"], regex=regx, return_data=False, escape=False, ).match(): # The element we are currently testing match the # regex we are currently reading. # We get the output of the function associated # with the regex. output = self.regexes_active_to_inactive_potentially_down[ regx ]() if output is not None: # The output is not None. # We return the new source and state. return output # We return None, there is no changes. return None
def handle(self): """ Handle the backend of the given status. """ # We initiate the source we are going to parse to the Generate class. source = "URL" if self.catched.lower() not in PyFunceble.STATUS["list"]["invalid"]: # The parsed status is not in the list of invalid. # We generate the status file with the catched status. Generate(self.catched, source).status_file() else: # The parsed status is in the list of invalid. # We generate the status file with the parsed status. Generate(self.catched, "SYNTAX").status_file() # We return the parsed status. return self.catched
def handle(self): """ Handle the backend of the found status. """ # We generate the status file with the catched status. Generate( self.subject, self.subject_type, self.output["status"], source=self.output["status_source"], filename=self.filename, ip_validation=self.output["ipv4_syntax_validation"], ).status_file()
def handle(self): """ Handle the backend of the given status. """ # We generate the status file with the catched status. Generate( self.subject, self.subject_type, self.output["status"], source=self.output["status_source"], http_status_code=self.output["http_status_code"], filename=self.filename, ).status_file()
def handle(self): """ Handle the lack of WHOIS. :) Returns: str The status of the domains after generating the files. """ source = "NSLOOKUP" if self.matched_status.lower( ) not in PyFunceble.STATUS["list"]["invalid"]: if Lookup().nslookup(): Generate(PyFunceble.STATUS["official"]["up"], source).status_file() return PyFunceble.STATUS["official"]["up"] Generate(PyFunceble.STATUS["official"]["down"], source).status_file() return PyFunceble.STATUS["official"]["down"] Generate(PyFunceble.STATUS["official"]["invalid"], "IANA").status_file() return PyFunceble.STATUS["official"]["invalid"]
def domain(self, subject): # pragma: no cover """ Handle the test of a single domain. :param str subject: The subject we are testing. """ if subject: # The given subject is not empty nor None. if PyFunceble.CONFIGURATION["syntax"]: # The syntax mode is activated. # We get the status from SyntaxStatus. status = SyntaxStatus(subject, subject_type="file_domain", filename=self.file).get()["status"] else: # We test and get the status of the domain. status = Status( subject, subject_type="file_domain", filename=self.file, whois_db=self.whois_db, inactive_db=self.inactive_db, ).get()["status"] if PyFunceble.CONFIGURATION["simple"]: # The simple mode is activated. # We print the domain and the status. print("{0} {1}".format( self.get_simple_coloration(status) + subject, status)) if self.complements_test_started: # We started to test the complements. # We generate the complement file(s). Generate(subject, "file_domain", status).complements_file() # We return the status. return status # We return None, there is nothing to test. return None
def _test_line(self, line, manager_data=None): # pylint: disable=too-many-branches # pragma: no cover """ Given a line, we test it. :param str line: A line to work with. :param multiprocessing.Manager.list manager_data: A Server process. """ if PyFunceble.CONFIGURATION[ "db_type"] == "json" and manager_data is not None: autocontinue = AutoContinue(self.file, parent_process=False) inactive_db = InactiveDB(self.file) mining = Mining(self.file) else: # We use the previously initiated autocontinue instance. autocontinue = self.autocontinue # We use the previously initiated inactive database instance. inactive_db = self.inactive_db # We use the previously initiated mining instance. mining = self.mining # We remove cariage from the given line. line = line.strip() if not line or line[0] == "#": # We line is a comment line. # We return None, there is nothing to test. return None if Regex(line, self.regex_ignore, escape=False, return_data=False).match(): # The line match our list of elemenet # to ignore. # We return None, there is nothing to test. return None # We format the line, it's the last # rush before starting to filter and test. subject = self._format_line(line) if (not PyFunceble.CONFIGURATION["local"] and PyFunceble.Check(subject).is_reserved_ipv4()): # * We are not testing for local components. # and # * The subject is a reserved IPv4. # We return None, there is nothing to test. return None if PyFunceble.CONFIGURATION["filter"]: # We have to filter. if Regex(subject, PyFunceble.CONFIGURATION["filter"], return_data=False).match(): # The line match the given filter. # We get the status of the current line. status = self.__process_test(subject) else: # The line does not match the given filter. # We return None. return None else: # We do not have to filter. # We get the status of the current line. status = self.__process_test(subject) # We add the line into the auto continue database. autocontinue.add(subject, status) if status.lower() in self.list_of_up_statuses: # The status is in the list of UP status. # We mine if necessary. mining.mine(subject, self.file_type) if subject in inactive_db: # The subject is in the inactive database. # We generate the suspicous file. Generate(subject, "file_domain", PyFunceble.STATUS["official"] ["up"]).analytic_file("suspicious") # And we remove the current subject from # the inactive database. inactive_db.remove(subject) else: # The status is not in the list of UP status. # We add the current subject into the # inactive database. inactive_db.add(subject, status) if (self.complements_test_started and PyFunceble.CONFIGURATION["db_type"] == "json"): # We started the test of the complements. if "complements" in autocontinue.database: # The complement index is present. while subject in autocontinue.database["complements"]: # We loop untill the line is not present into the # database. # We remove the currently tested element. autocontinue.database["complements"].remove(subject) # We save the current state. autocontinue.save() if manager_data is None: # We are not in a multiprocess environment. # We update the counters autocontinue.update_counters() # We process the autosaving if it is necessary. self.autosave.process(test_completed=False) elif PyFunceble.CONFIGURATION["db_type"] == "json": # We are in a multiprocess environment. # We save everything we initiated into the server process manager_data.append({ "autocontinue": autocontinue.database, "inactive_db": inactive_db.database, "mining": mining.database, }) # We return None. return None
def _file_decision(self, current, last, status=None): """ Manage the database, autosave and autocontinue systems for the case that we are reading a file. :param current: The currently tested element. :type current: str :param last: The last element of the list. :type last: str :param status: The status of the currently tested element. :type status: str """ if ( status and not PyFunceble.CONFIGURATION["simple"] and PyFunceble.INTERN["file_to_test"] ): # * The status is given. # and # * The simple mode is deactivated. # and # * A file to test is set. # We run the mining logic. Mining().process() # We delete the currently tested element from the mining # database. # Indeed, as it is tested, it is already in our # testing process which means that we don't need it into # the mining database. Mining().remove() if ( status.lower() in PyFunceble.STATUS["list"]["up"] or status.lower() in PyFunceble.STATUS["list"]["valid"] ): # The status is in the list of up status. if Inactive().is_present(): # The currently tested element is in the database. # We generate the suspicious file(s). Generate("strange").analytic_file( "suspicious", PyFunceble.STATUS["official"]["up"] ) # We remove the currently tested element from the # database. Inactive().remove() else: # The status is not in the list of up status. # We add the currently tested element to the # database. Inactive().add() # We backup the current state of the file reading # for the case that we need to continue later. AutoContinue().backup() if current != last: # The current element is not the last one. # We run the autosave logic. AutoSave() else: # The current element is the last one. # We stop and log the execution time. ExecutionTime("stop", True) # We show/log the percentage. Percentage().log() # We reset the counters as we end the process. self.reset_counters() # We backup the current state of the file reading # for the case that we need to continue later. AutoContinue().backup() # We show the colored logo. self.colorify_logo() # We save and stop the script if we are under # Travis CI. AutoSave(True) for index in ["http_code", "referer"]: # We loop through some configuration index we have to empty. if index in PyFunceble.INTERN: # The index is in the configuration. # We empty the configuration index. PyFunceble.INTERN[index] = ""
def handle(cls, status, invalid_source="IANA"): """ Handle the lack of WHOIS and expiration date. :smile_cat: :param matched_status: The status that we have to handle. :type status: str :param invalid_source: The source to set when we handle INVALID element. :type invalid_source: str :return: The strus of the domain after generating the files desired by the user. :rtype: str """ if status.lower() not in PyFunceble.STATUS["list"]["invalid"]: # The matched status is not in the list of invalid status. # We initiate the source we are going to parse to the Generate class. source = "NSLOOKUP" if Lookup().nslookup(): # We could execute the nslookup logic. # We get the status and source after extra rules check. status, source = cls.extra_rules.handle( PyFunceble.STATUS["official"]["up"], source ) # We generate the status files with the up status. Generate(status, source).status_file() # We return the up status. return status, source # We could not execute the nslookup logic. # We get the status and source after extra rules check. status, source = cls.extra_rules.handle( PyFunceble.STATUS["official"]["down"], source ) # We generate the status file with the down status. Generate(status, source).status_file() # We return the down status. return status, source # The matched status is in the list of invalid status. # We get the status and source after extra rules check. status, source = cls.extra_rules.handle( PyFunceble.STATUS["official"]["invalid"], invalid_source ) # We generate the status file with the invalid status. Generate(status, source).status_file() # We return the status. return status, source
def handle(self, status, ip_validation_status): """ Handle the lack of WHOIS and expiration date. :smile_cat: :param str matched_status: The status that we have to handle. :param str ip_validation_status: The IP syntax validation. :return: The status of the domain after generating the files desired by the user. :rtype: str """ # We get the dns_lookup state. self.output["dns_lookup"] = PyFunceble.DNSLookup( self.subject, dns_server=PyFunceble.CONFIGURATION["dns_server"]).request() if status.lower() not in PyFunceble.STATUS["list"]["invalid"]: # The matched status is not in the list of invalid status. if self.output["dns_lookup"]: # We could execute the dns_lookup logic. # We set the status we got. self.output["_status"] = PyFunceble.STATUS["official"]["up"] else: # We could not get something. # We set the status we got. self.output["_status"] = PyFunceble.STATUS["official"]["down"] # We get the status and source after extra rules check. self.output["status"], self.output["status_source"] = ExtraRules( self.subject, self.subject_type, self.output["http_status_code"]).handle( self.output["_status"], self.output["_status_source"]) else: if self.output["dns_lookup"]: # We could execute the dns_lookup logic. # We set the status we got. self.output["_status"] = PyFunceble.STATUS["official"]["up"] # We set the status source. self.output["_status_source"] = "DNSLOOKUP" self.output["status"], self.output["status_source"] = ( self.output["_status"], self.output["_status_source"], ) # We generate the status file with the invalid status. Generate( self.subject, self.subject_type, self.output["status"], source=self.output["status_source"], expiration_date=self.output["expiration_date"], http_status_code=self.output["http_status_code"], whois_server=self.output["whois_server"], filename=self.filename, ip_validation=ip_validation_status, ).status_file()
def get(self): """ Get the status while testing for an IP or domain. """ if self.subject: self.output.update({ "domain_syntax_validation": self.checker.is_domain(), "expiration_date": None, "http_status_code": "***", "ipv4_range_syntax_validation": self.checker.is_ipv4_range(), "ipv4_syntax_validation": self.checker.is_ipv4(), "subdomain_syntax_validation": self.checker.is_subdomain(), "tested": self.subject, "url_syntax_validation": self.checker.is_url(), "whois_server": Referer(self.subject).get(), }) if PyFunceble.CONFIGURATION["local"] or ( self.output["domain_syntax_validation"] or self.output["ipv4_syntax_validation"]): self.output["http_status_code"] = HTTPCode( self.subject, self.subject_type).get() if not self.output["subdomain_syntax_validation"]: self.output["expiration_date"], self.output[ "whois_record"] = ExpirationDate( self.subject, self.output["whois_server"], whois_db=self.whois_db, ).get() if isinstance(self.output["expiration_date"], str): self.output["_status_source"] = self.output[ "status_source"] = "WHOIS" self.output["_status"] = self.output[ "status"] = PyFunceble.STATUS["official"]["up"] Generate( self.subject, self.subject_type, self.output["status"], source=self.output["status_source"], expiration_date=self.output["expiration_date"], http_status_code=self.output["http_status_code"], whois_server=self.output["whois_server"], filename=self.filename, ip_validation=self. output["ipv4_syntax_validation"], ).status_file() else: self.output["_status_source"] = "DNSLOOKUP" self.handle( status="inactive", ip_validation_status=self. output["ipv4_syntax_validation"], ) else: self.output["_status_source"] = "DNSLOOKUP" self.handle( status="inactive", ip_validation_status=self. output["ipv4_syntax_validation"], ) else: self.output["_status_source"] = "SYNTAX" self.output["_status"] = PyFunceble.STATUS["official"][ "invalid"] self.handle( status="invalid", ip_validation_status=self.output["ipv4_syntax_validation"], ) return self.output raise ValueError("Subject should be given.")
def _extract(self): # pragma: no cover """ Extract the expiration date from the whois record. :return: The status of the domain. :rtype: str """ # We try to get the expiration date from the database. expiration_date_from_database = Whois().get_expiration_date() if expiration_date_from_database: # The hash of the current whois record did not changed and the # expiration date from the database is not empty not equal to # None or False. # We generate the files and print the status. # It's an active element! Generate( PyFunceble.STATUS["official"]["up"], "WHOIS", expiration_date_from_database, ).status_file() # We handle und return the official up status. return PyFunceble.STATUS["official"]["up"] # We get the whois record. self.whois_record = Lookup().whois(PyFunceble.INTERN["referer"]) # We list the list of regex which will help us get an unformatted expiration date. to_match = [ r"expire:(.*)", r"expire on:(.*)", r"Expiry Date:(.*)", r"free-date(.*)", r"expires:(.*)", r"Expiration date:(.*)", r"Expiry date:(.*)", r"Expire Date:(.*)", r"renewal date:(.*)", r"Expires:(.*)", r"validity:(.*)", r"Expiration Date :(.*)", r"Expiry :(.*)", r"expires at:(.*)", r"domain_datebilleduntil:(.*)", r"Data de expiração \/ Expiration Date \(dd\/mm\/yyyy\):(.*)", r"Fecha de expiración \(Expiration date\):(.*)", r"\[Expires on\](.*)", r"Record expires on(.*)(\(YYYY-MM-DD\))", r"status: OK-UNTIL(.*)", r"renewal:(.*)", r"expires............:(.*)", r"expire-date:(.*)", r"Exp date:(.*)", r"Valid-date(.*)", r"Expires On:(.*)", r"Fecha de vencimiento:(.*)", r"Expiration:.........(.*)", r"Fecha de Vencimiento:(.*)", r"Registry Expiry Date:(.*)", r"Expires on..............:(.*)", r"Expiration Time:(.*)", r"Expiration Date:(.*)", r"Expired:(.*)", r"Date d'expiration:(.*)", ] if self.whois_record: # The whois record is not empty. if "current_test_data" in PyFunceble.INTERN: # The end-user want more information whith his test. # We update the whois_record index. PyFunceble.INTERN["current_test_data"][ "whois_record"] = self.whois_record for string in to_match: # We loop through the list of regex. # We try tro extract the expiration date from the WHOIS record. expiration_date = Regex(self.whois_record, string, return_data=True, rematch=True, group=0).match() if expiration_date: # The expiration date could be extracted. # We get the extracted expiration date. self.expiration_date = expiration_date[0].strip() # We initate a regex which will help us know if a number # is present into the extracted expiration date. regex_rumbers = r"[0-9]" if Regex(self.expiration_date, regex_rumbers, return_data=False).match(): # The extracted expiration date has a number. # We format the extracted expiration date. self.expiration_date = self._format() if (self.expiration_date and not Regex( self.expiration_date, r"[0-9]{2}\-[a-z]{3}\-2[0-9]{3}", return_data=False, ).match()): # The formatted expiration date does not match our unified format. # We log the problem. Logs().expiration_date(self.expiration_date) # We log the whois record. Logs().whois(self.whois_record) if "current_test_data" in PyFunceble.INTERN: # The end-user want more information whith his test. # We update the expiration_date index. PyFunceble.INTERN["current_test_data"][ "expiration_date"] = self.expiration_date # We generate the files and print the status. # It's an active element! Generate( PyFunceble.STATUS["official"]["up"], "WHOIS", self.expiration_date, ).status_file() # We log the whois record. Logs().whois(self.whois_record) # We save the whois record into the database. Whois(expiration_date=self.expiration_date).add() # We handle und return the official up status. return PyFunceble.STATUS["official"]["up"] # The extracted expiration date does not have a number. # We log the whois record. Logs().whois(self.whois_record) # We handle and return and h the official down status. return Status( PyFunceble.STATUS["official"]["down"]).handle() # The whois record is empty. # We handle and return the official down status. return Status(PyFunceble.STATUS["official"]["down"]).handle()
def _extract(self): # pragma: no cover """ Extract the expiration date from the whois record. """ self.whois_record = Lookup().whois(PyFunceble.CONFIGURATION["referer"]) to_match = [ r"expire:(.*)", r"expire on:(.*)", r"Expiry Date:(.*)", r"free-date(.*)", r"expires:(.*)", r"Expiration date:(.*)", r"Expiry date:(.*)", r"Expire Date:(.*)", r"renewal date:(.*)", r"Expires:(.*)", r"validity:(.*)", r"Expiration Date :(.*)", r"Expiry :(.*)", r"expires at:(.*)", r"domain_datebilleduntil:(.*)", r"Data de expiração \/ Expiration Date \(dd\/mm\/yyyy\):(.*)", r"Fecha de expiración \(Expiration date\):(.*)", r"\[Expires on\](.*)", r"Record expires on(.*)(\(YYYY-MM-DD\))", r"status: OK-UNTIL(.*)", r"renewal:(.*)", r"expires............:(.*)", r"expire-date:(.*)", r"Exp date:(.*)", r"Valid-date(.*)", r"Expires On:(.*)", r"Fecha de vencimiento:(.*)", r"Expiration:.........(.*)", r"Fecha de Vencimiento:(.*)", r"Registry Expiry Date:(.*)", r"Expires on..............:(.*)", r"Expiration Time:(.*)", r"Expiration Date:(.*)", r"Expired:(.*)", r"Date d'expiration:(.*)", ] if self.whois_record: for string in to_match: expiration_date = Regex( self.whois_record, string, return_data=True, rematch=True, group=0 ).match() if expiration_date: self.expiration_date = expiration_date[0].strip() regex_rumbers = r"[0-9]" if Regex( self.expiration_date, regex_rumbers, return_data=False ).match(): self.expiration_date = self._format() if self.expiration_date and not Regex( self.expiration_date, r"[0-9]{2}\-[a-z]{3}\-2[0-9]{3}", return_data=False, ).match(): self.log() self._whois_log() Generate( PyFunceble.STATUS["official"]["up"], "WHOIS", self.expiration_date, ).status_file() self._whois_log() return PyFunceble.STATUS["official"]["up"] self._whois_log() return Status(PyFunceble.STATUS["official"]["down"]).handle() self._whois_log() return Status(PyFunceble.STATUS["official"]["down"]).handle()