def status_file(self): # pylint: disable=inconsistent-return-statements """ Generate a file according to the domain status. """ # We generate the hosts file. self.info_files() # We are testing a file content. # We increase the percentage count. Percentage(self.status).count() # We print on screen if needed. self._prints_status_screen() if self._do_not_produce_file(): return None if (not PyFunceble.CONFIGURATION["no_files"] and PyFunceble.CONFIGURATION["split"]): # * The file non-generation of file is globaly deactivated. # and # * We have to split the outputs. # We print or generate the files. self._prints_status_file() else: # * The file non-generation of file is globaly activated. # or # * We do not have to split the outputs. # We print or generate the unified files. self.unified_file()
def test_count(self): """ This method test if the counter can be set proprely. """ expected = {} for i, element in enumerate(["tested", "up", "down", "invalid"]): PyFunceble.CONFIGURATION["counter"]["number"][element] = 12 + i expected.update({element: 12 + i}) for element in ["tested", "up", "down", "invalid"]: try: Percentage( domain_status=PyFunceble.STATUS["official"][element], init=None).count() expected[element] += 1 expected["tested"] += 1 actual = PyFunceble.CONFIGURATION["counter"]["number"] self.assertEqual(expected, actual) except KeyError: pass PyFunceble.CONFIGURATION["counter"]["number"] = {}
def _travis(self): """ Logic behind autosave under Travis CI. """ if PyFunceble.CONFIGURATION["travis"]: try: _ = PyFunceble.environ["TRAVIS_BUILD_DIR"] time_autorisation = False try: time_autorisation = int(PyFunceble.time( )) >= int(PyFunceble.INTERN["start"]) + (int( PyFunceble.CONFIGURATION["travis_autosave_minutes"]) * 60) except KeyError: if self.last and not self.bypass: raise Exception( "Please review the way `ExecutionTime()` is called." ) if self.last or time_autorisation or self.bypass: Percentage().log() self.travis_permissions() command = 'git add --all && git commit -a -m "%s"' if self.last or self.bypass: if PyFunceble.CONFIGURATION["command_before_end"]: print( Command(PyFunceble.CONFIGURATION[ "command_before_end"]).execute()) self.travis_permissions() message = ( PyFunceble. CONFIGURATION["travis_autosave_final_commit"] + " [ci skip]") Command(command % message).execute() else: if PyFunceble.CONFIGURATION["command"]: print( Command(PyFunceble.CONFIGURATION["command"]). execute()) self.travis_permissions() Command( command % PyFunceble.CONFIGURATION["travis_autosave_commit"] ).execute() Command( "git push origin %s" % PyFunceble.CONFIGURATION["travis_branch"]).execute() exit(0) except KeyError: pass
def test_log_syntax(self): """ Test the log system for the case that we are checking for syntax. """ PyFunceble.CONFIGURATION["syntax"] = PyFunceble.CONFIGURATION[ "show_percentage"] = True BaseStdout.setUp(self) expected = """ Status Percentage Numbers%s ----------- ------------ ------------ VALID 36%% 45%s INVALID 1%% 2%s """ % ( " " * 5, " " * 10, " " * 11, ) PyFunceble.INTERN["counter"]["number"].update({ "up": 45, "down": 78, "invalid": 2, "tested": 125 }) Percentage(domain_status=None, init=None).log() actual = sys.stdout.getvalue() self.assertEqual(expected, actual) # Test for the case that we do not show_percentage PyFunceble.CONFIGURATION["show_percentage"] = False PyFunceble.INTERN["counter"]["number"].update({ "up": 45, "down": 78, "invalid": 2, "tested": 125 }) Percentage(domain_status=None, init=None).log() actual = PyFunceble.INTERN["counter"]["percentage"] expected = {"up": 36, "down": 62, "invalid": 1} self.assertEqual(expected, actual)
def status_file(self): """ Generate a file according to the domain status. """ try: PyFunceble.CONFIGURATION["http_code"] except KeyError: PyFunceble.CONFIGURATION["http_code"] = "*" * 3 if not PyFunceble.CONFIGURATION["http_code"]: PyFunceble.CONFIGURATION["http_code"] = "*" * 3 if self.domain_status.lower() in PyFunceble.STATUS["list"]["up"]: self.up_status_file() elif self.domain_status.lower() in PyFunceble.STATUS["list"]["down"]: self.down_status_file() elif self.domain_status.lower() in PyFunceble.STATUS["list"]["invalid"]: self.invalid_status_file() Generate(self.domain_status, self.source, self.expiration_date).hosts_file() Percentage(self.domain_status).count() if not PyFunceble.CONFIGURATION["quiet"]: if PyFunceble.CONFIGURATION["less"]: Prints( [ self.tested, self.domain_status, PyFunceble.CONFIGURATION["http_code"], ], "Less", ).data() else: if PyFunceble.HTTP_CODE["active"]: data_to_print = [ self.tested, self.domain_status, self.expiration_date, self.source, PyFunceble.CONFIGURATION["http_code"], ] else: data_to_print = [ self.tested, self.domain_status, self.expiration_date, self.source, PyFunceble.CURRENT_TIME, ] Prints(data_to_print, "Generic").data() if not PyFunceble.CONFIGURATION["no_files"] and PyFunceble.CONFIGURATION[ "split" ]: self._prints_status_file() else: self.unified_file()
def __init__(self, domain=None, file_path=None, **args): optional_arguments = { "url_to_test": None, "file_urls": None, "modulo_test": False } # We initiate our optional_arguments in order to be usable all over the # class for (arg, default) in optional_arguments.items(): setattr(self, arg, args.get(arg, default)) if not self.modulo_test: # pylint: disable=no-member PyFunceble.CONFIGURATION["file_to_test"] = file_path # pylint: disable=no-member if self.file_urls: # pylint: disable=no-member PyFunceble.CONFIGURATION["file_to_test"] = self.file_urls # pylint: disable=no-member if PyFunceble.CONFIGURATION["travis"]: AutoSave().travis_permissions() self.bypass() ExecutionTime("start") if domain: PyFunceble.CONFIGURATION["show_percentage"] = False PyFunceble.CONFIGURATION["domain"] = domain.lower() self.domain() elif self.url_to_test and not file_path: # pylint: disable=no-member PyFunceble.CONFIGURATION["show_percentage"] = False PyFunceble.CONFIGURATION["URL"] = self.url_to_test # pylint: disable=no-member self.url() elif self.file_urls: # pylint: disable=no-member PyFunceble.CONFIGURATION[ "no_whois"] = PyFunceble.CONFIGURATION[ "plain_list_domain"] = PyFunceble.CONFIGURATION[ "split"] = True PyFunceble.CONFIGURATION["generate_hosts"] = False self.url_file() elif file_path: self.file() ExecutionTime("stop") Percentage().log() if domain: self.colored_logo() else: PyFunceble.CONFIGURATION["simple"] = True PyFunceble.CONFIGURATION["quiet"] = True PyFunceble.CONFIGURATION["no_files"] = True if domain: PyFunceble.CONFIGURATION["domain"] = domain.lower()
def test_log(self): """ This method test the log system. """ BaseStdout.setUp(self) expected = """ Status Percentage Numbers%s ----------- ------------ ------------%s ACTIVE 36%% 45%s INACTIVE 62%% 78%s INVALID 1%% 2%s """ % (" " * 6, " " * 1, " " * 11, " " * 11, " " * 12) PyFunceble.CONFIGURATION["counter"]["number"].update({ "up": 45, "down": 78, "invalid": 2, "tested": 125 }) Percentage(domain_status=None, init=None).log() actual = sys.stdout.getvalue() self.assertEqual(expected, actual) # Test for the case that we do not show_percentage PyFunceble.CONFIGURATION["show_percentage"] = False PyFunceble.CONFIGURATION["counter"]["number"].update({ "up": 45, "down": 78, "invalid": 2, "tested": 125 }) Percentage(domain_status=None, init=None).log() actual = PyFunceble.CONFIGURATION["counter"]["percentage"] expected = {"up": 36, "down": 62, "invalid": 1} self.assertEqual(expected, actual)
def test_init(self): """ Test the :code:`init` argument of Percentage() """ expected = {"up": 15, "down": 2, "invalid": 0, "tested": 75} Percentage(domain_status=None, init=expected) self.assertEqual(expected, PyFunceble.INTERN["counter"]["percentage"]) PyFunceble.INTERN["counter"]["percentage"] = {}
def test_init(self): """ This method test the `init` argument of Percentage() """ expected = {"up": 15, "down": 2, "invalid": 0, "tested": 75} Percentage(domain_status=None, init=expected) self.assertEqual(expected, PyFunceble.CONFIGURATION["counter"]["percentage"]) PyFunceble.CONFIGURATION["counter"]["percentage"] = {}
def test_calculate(self): """ Test the calculation system. """ PyFunceble.CONFIGURATION["counter"]["number"].update( {"up": 45, "down": 78, "invalid": 2, "tested": 125} ) expected = {"up": 36, "down": 62, "invalid": 1} Percentage(domain_status=None, init=None)._calculate() actual = PyFunceble.CONFIGURATION["counter"]["percentage"] self.assertEqual(expected, actual) PyFunceble.CONFIGURATION["counter"]["number"] = {}
def __init__(self, **args): # We initiate our list of optional arguments with their default values. optional_arguments = { "domain_or_ip_to_test": None, "file_path": None, "url_to_test": None, "url_file": None, "modulo_test": False, "link_to_test": None, } # We initiate our optional_arguments in order to be usable all over the # class. for (arg, default) in optional_arguments.items(): setattr(self, arg, args.get(arg, default)) # We initiate a variable in order to avoid having to recall/declare # Status() over and over. self.status = Status() # We initiate a variable in order to avoid having to recall/declare # Check() over and over. self.checker = Check() # We initiate a variable in order to avoid having to recall/declare # Percentage() over and over. self.percentage = Percentage() # We initiate a variable in order to avoid having to recall/declare # URL() over and over. self.url_status = URL() # We initiate a variable in order to avoid having to recall/declare # Mining() over and over. self.mining = Mining() # We initiate a variable in order to avoid having to recall/declare # AutoContinue() over and over. self.auto_continue = None # We initiate a variable in order to avoid having to recall/declare # Syntax() over and over. self.syntax_status = Syntax() # We initiate a variable in order to avoid having to recall/declare # Inactive() over and over. self.inactive_database = Inactive() # We manage the entries. self._entry_management()
def _travis(self): """ Logic behind travis autosave. """ current_time = int(strftime("%s")) time_autorisation = False try: time_autorisation = current_time >= int( PyFunceble.CONFIGURATION["start"]) + (int( PyFunceble.CONFIGURATION["travis_autosave_minutes"]) * 60) except KeyError: if self.last and not self.bypass: raise Exception( "Please review the way `ExecutionTime()` is called.") if self.last or time_autorisation or self.bypass: Percentage().log() self.travis_permissions() command = 'git add --all && git commit -a -m "%s"' if self.last or self.bypass: if PyFunceble.CONFIGURATION["command_before_end"]: print( Command(PyFunceble.CONFIGURATION["command_before_end"] ).execute()) self.travis_permissions() message = PyFunceble.CONFIGURATION[ "travis_autosave_final_commit"] + " [ci skip]" Command(command % message).execute() else: Command(command % PyFunceble.CONFIGURATION["travis_autosave_commit"] ).execute() Command("git push origin %s" % PyFunceble.CONFIGURATION["travis_branch"]).execute() exit(0)
def test_log_quiet(self): """ Test the log system for the case that we quiet mode is activated. """ output = (PyFunceble.OUTPUT_DIRECTORY + PyFunceble.OUTPUTS["parent_directory"] + PyFunceble.OUTPUTS["logs"]["directories"]["parent"] + PyFunceble.OUTPUTS["logs"]["directories"]["percentage"] + PyFunceble.OUTPUTS["logs"]["filenames"]["percentage"]) PyFunceble.CONFIGURATION["quiet"] = True expected = """ Status Percentage Numbers%s ----------- ------------ ------------ ACTIVE 36%% 45%s INACTIVE 62%% 78%s INVALID 1%% 2%s """ % ( " " * 5, " " * 10, " " * 10, " " * 11, ) PyFunceble.INTERN["counter"]["number"].update({ "up": 45, "down": 78, "invalid": 2, "tested": 125 }) Percentage(domain_status=None, init=None).log() actual = "\n" + "\n".join([ x for x in File(output).read().split("\n") if not x.startswith("#") ]) self.assertEqual(expected, actual) PyFunceble.CONFIGURATION["quiet"] = False
def _file_decision(self, current, last, status=None): """ Manage the database, autosave and autocontinue systems for the case that we are reading a file. Arguments: - status: str The current status of current. - current: str The current domain or URL we are testing. - last: str The last domain or URL of the file we are testing. """ if status: if not PyFunceble.CONFIGURATION[ "simple"] and PyFunceble.CONFIGURATION["file_to_test"]: if PyFunceble.CONFIGURATION["inactive_database"]: if status.lower() in PyFunceble.STATUS["list"]["up"]: Database().remove() else: Database().add() AutoContinue().backup() if current != last: AutoSave() else: ExecutionTime("stop") Percentage().log() self.reset_counters() AutoContinue().backup() self.colored_logo() AutoSave(True) for index in ["http_code", "referer"]: if index in PyFunceble.CONFIGURATION: PyFunceble.CONFIGURATION[index] = ""
def not_end_commit(self): """ Commit and push at on the middle of the test. """ if self.authorized: Percentage().log() self.permissions() command = 'git add --all && git commit -a -m "{0}"'.format( PyFunceble.CONFIGURATION["travis_autosave_commit"] ) if PyFunceble.CONFIGURATION["command"]: for line in Command(PyFunceble.CONFIGURATION["command"]).run(): sys_stdout.write("{}\n".format(line)) self.permissions() for line in Command(command).run(): sys_stdout.write("{}\n".format(line)) self.push()
def __init__( self, domain_or_ip=None, file_path=None, link_to_test=None, url_file_path=None, url_to_test=None, ): # pylint: disable=too-many-branches if domain_or_ip or file_path or link_to_test or url_file_path or url_to_test: preset = PyFunceble.Preset() CLICore.logs_sharing() ExecutionTime("start") if domain_or_ip: SimpleCore(domain_or_ip).domain() elif file_path: PyFunceble.DirectoryStructure() if PyFunceble.CONFIGURATION["multiprocess"]: preset.maximal_processes() preset.multiprocess() FileMultiprocessCore( file_path, "domain" ).read_and_test_file_content() else: FileCore(file_path, "domain").read_and_test_file_content() elif link_to_test: PyFunceble.DirectoryStructure() if PyFunceble.CONFIGURATION["multiprocess"]: preset.maximal_processes() preset.multiprocess() FileMultiprocessCore( link_to_test, "domain" ).read_and_test_file_content() else: FileCore(link_to_test, "domain").read_and_test_file_content() elif url_file_path: PyFunceble.DirectoryStructure() preset.file_url() if PyFunceble.CONFIGURATION["multiprocess"]: preset.maximal_processes() preset.multiprocess() FileMultiprocessCore( url_file_path, "url" ).read_and_test_file_content() else: FileCore(url_file_path, "url").read_and_test_file_content() elif url_to_test: SimpleCore(url_to_test).url() Percentage().log() ExecutionTime("stop") PyFunceble.CLICore.stay_safe() else: PyFunceble.CLICore.print_nothing_to_test()
def _entry_management(self): # pylint: disable=too-many-branches """ Avoid to have 1 millions line into self.__init__() """ if not self.modulo_test: # pylint: disable=no-member # We are not in a module usage. # We set the file_path as the file we have to test. PyFunceble.INTERN[ "file_to_test" ] = self.file_path # pylint: disable=no-member # We check if the given file_path is an url. # If it is an URL we update the file to test and download # the given URL. self._entry_management_url() # We fix the environnement permissions. AutoSave().travis_permissions() # We check if we need to bypass the execution of PyFunceble. self.bypass() # We set the start time. ExecutionTime("start") if PyFunceble.CONFIGURATION["syntax"]: # We are checking for syntax. # We deactivate the http status code. PyFunceble.HTTP_CODE["active"] = False if self.domain_or_ip_to_test: # pylint: disable=no-member # The given domain is not empty or None. # We initiate a variable which will tell the system the type # of the tested element. PyFunceble.INTERN["to_test_type"] = "domain" # We set the start time. ExecutionTime("start") # We deactivate the showing of percentage as we are in a single # test run. PyFunceble.CONFIGURATION["show_percentage"] = False # We deactivate the whois database as it is not needed. PyFunceble.CONFIGURATION["whois_database"] = False if PyFunceble.CONFIGURATION["idna_conversion"]: domain_or_ip_to_test = domain2idna( self.domain_or_ip_to_test.lower() # pylint: disable=no-member ) else: domain_or_ip_to_test = ( self.domain_or_ip_to_test.lower() # pylint: disable=no-member ) # pylint: disable=no-member # We test the domain after converting it to lower case. self.domain(domain_or_ip_to_test) elif self.url_to_test and not self.file_path: # pylint: disable=no-member # An url to test is given and the file path is empty. # We initiate a variable which will tell the system the type # of the tested element. PyFunceble.INTERN["to_test_type"] = "url" # We set the start time. ExecutionTime("start") # We deactivate the showing of percentage as we are in a single # test run. PyFunceble.CONFIGURATION["show_percentage"] = False # We test the url to test after converting it if needed (IDNA). self.url( Check().is_url_valid( self.url_to_test, # pylint: disable=no-member return_formatted=True, ) ) elif ( self._entry_management_url_download( self.url_file # pylint: disable=no-member ) or self.url_file # pylint: disable=no-member ): # * A file full of URL is given. # or # * the given file full of URL is a URL. # * We deactivate the whois subsystem as it is not needed for url testing. # * We activate the generation of plain list element. # * We activate the generation of splited data instead of unified data. PyFunceble.CONFIGURATION["no_whois"] = PyFunceble.CONFIGURATION[ "plain_list_domain" ] = PyFunceble.CONFIGURATION["split"] = True # We deactivate the generation of hosts file as it is not relevant for # url testing. PyFunceble.CONFIGURATION["generate_hosts"] = False # We initiate a variable which will tell the system the type # of the tested element. PyFunceble.INTERN["to_test_type"] = "url" # And we test the given or the downloaded file. self.file_url() elif ( self._entry_management_url_download( self.link_to_test # pylint: disable=no-member ) or self._entry_management_url_download( self.file_path # pylint: disable=no-member ) # pylint: disable=no-member or self.file_path # pylint: disable=no-member ): # * A file path is given. # or # * The given file path is an URL. # or # * A link to test is given. # We initiate a variable which will tell the system the type # of the tested element. PyFunceble.INTERN["to_test_type"] = "domain" # We test the given or the downloaded file. self.file() else: # No file, domain, single url or file or url is given. # We print a message on screen. print( PyFunceble.Fore.CYAN + PyFunceble.Style.BRIGHT + "Nothing to test." ) # We stop and log the execution time. ExecutionTime("stop", last=True) # We log the current percentage state. Percentage().log() if ( self.domain_or_ip_to_test # pylint: disable=no-member or self.url_to_test # pylint: disable=no-member ): # We are testing a domain. # We show the colored logo. self.colorify_logo() # We print our friendly message :) PyFunceble.stay_safe() else: # We are used as an imported module. # * We activate the simple mode as the table or any full # details on screen are irrelevant. # * We activate the quiet mode. # And we deactivate the generation of files. PyFunceble.CONFIGURATION["simple"] = PyFunceble.CONFIGURATION[ "quiet" ] = PyFunceble.CONFIGURATION["no_files"] = True # * We deactivate the whois database as it is not needed. # * We deactivate the database as it is not needed. PyFunceble.CONFIGURATION["whois_database"] = PyFunceble.CONFIGURATION[ "inactive_database" ] = False if self.domain_or_ip_to_test: # pylint: disable=no-member # A domain is given. # We initiate a variable which will tell the system the type # of the tested element. PyFunceble.INTERN["to_test_type"] = "domain" # We set the domain to test. PyFunceble.INTERN[ "to_test" ] = self.domain_or_ip_to_test.lower() # pylint: disable=no-member elif self.url_to_test: # pylint: disable=no-member # A url is given, # We initiate a variable which will tell the system the type # of the tested element. PyFunceble.INTERN["to_test_type"] = "url" # We set the url to test. PyFunceble.INTERN[ "to_test" ] = self.url_to_test # pylint: disable=no-member
def _file_decision(self, current, last, status=None): """ Manage the database, autosave and autocontinue systems for the case that we are reading a file. :param current: The currently tested element. :type current: str :param last: The last element of the list. :type last: str :param status: The status of the currently tested element. :type status: str """ if ( status and not PyFunceble.CONFIGURATION["simple"] and PyFunceble.INTERN["file_to_test"] ): # * The status is given. # and # * The simple mode is deactivated. # and # * A file to test is set. # We run the mining logic. Mining().process() # We delete the currently tested element from the mining # database. # Indeed, as it is tested, it is already in our # testing process which means that we don't need it into # the mining database. Mining().remove() if ( status.lower() in PyFunceble.STATUS["list"]["up"] or status.lower() in PyFunceble.STATUS["list"]["valid"] ): # The status is in the list of up status. if Inactive().is_present(): # The currently tested element is in the database. # We generate the suspicious file(s). Generate("strange").analytic_file( "suspicious", PyFunceble.STATUS["official"]["up"] ) # We remove the currently tested element from the # database. Inactive().remove() else: # The status is not in the list of up status. # We add the currently tested element to the # database. Inactive().add() # We backup the current state of the file reading # for the case that we need to continue later. AutoContinue().backup() if current != last: # The current element is not the last one. # We run the autosave logic. AutoSave() else: # The current element is the last one. # We stop and log the execution time. ExecutionTime("stop", True) # We show/log the percentage. Percentage().log() # We reset the counters as we end the process. self.reset_counters() # We backup the current state of the file reading # for the case that we need to continue later. AutoContinue().backup() # We show the colored logo. self.colorify_logo() # We save and stop the script if we are under # Travis CI. AutoSave(True) for index in ["http_code", "referer"]: # We loop through some configuration index we have to empty. if index in PyFunceble.INTERN: # The index is in the configuration. # We empty the configuration index. PyFunceble.INTERN[index] = ""
def status_file(self): # pylint: disable=inconsistent-return-statements """ Generate a file according to the domain status. """ if not PyFunceble.INTERN["http_code"]: # The http code is equal to None. # We initiate an empty http code. PyFunceble.INTERN["http_code"] = "*" * 3 if self.domain_status.lower() in PyFunceble.STATUS["list"]["up"]: # The status is in the list of up status. # We generate the status file(s). self.up_status_file() elif self.domain_status.lower() in PyFunceble.STATUS["list"]["valid"]: # The status is in the list of valid status. # We generate the status file(s). self.valid_status_file() elif self.domain_status.lower() in PyFunceble.STATUS["list"]["down"]: # The status is in the list of down status. # We generate the status file(s). self.down_status_file() elif self.domain_status.lower( ) in PyFunceble.STATUS["list"]["invalid"]: # The status is in the list of invalid status. # We generate the status file(s). self.invalid_status_file() # We generate the hosts file. Generate(self.domain_status, self.source, self.expiration_date).info_files() # We increase the percentage count. Percentage(self.domain_status).count() # We print on screen if needed. self._prints_status_screen() if self._do_not_produce_file(): return None if (not PyFunceble.CONFIGURATION["no_files"] and PyFunceble.CONFIGURATION["split"]): # * The file non-generation of file is globaly deactivated. # and # * We have to split the outputs. # We print or generate the files. self._prints_status_file() else: # * The file non-generation of file is globaly activated. # or # * We do not have to split the outputs. # We print or generate the unified files. self.unified_file()