def save_to_env_file(cls, envs, env_file_location): """ Save the given dict of environment variable into our environment file. :param dict envs: A dict of environment variables to save. :param str env_file_location: The location of the file we have to update. """ file_instance = File(env_file_location) try: content = file_instance.read() except FileNotFoundError: content = "" if content: for environment_variable, value in envs.items(): to_write = "{0}={1}".format(environment_variable, value) regex = r"{0}=.*".format(environment_variable) if Regex(content, regex, return_data=False).match(): content = Regex(content, regex, replace_with=to_write).replace() else: if not content.endswith("\n"): content += "\n{0}\n".format(to_write) else: content += "{0}\n".format(to_write) else: for environment_variable, value in envs.items(): to_write = "{0}={1}".format(environment_variable, value) content += "{0}\n".format(to_write) file_instance.write(content, overwrite=True)
def _get_structure(self): """ This method return the structure we are goinng to work with. """ structure_file = "" req = "" if path.isfile(self.structure): structure_file = self.structure elif path.isfile(self.base + "dir_structure_production.json"): structure_file = self.base + "dir_structure_production.json" else: if "dev" not in PyFunceble.VERSION: req = requests.get(PyFunceble.LINKS["dir_structure"]) else: req = requests.get(PyFunceble.LINKS["dir_structure"].replace( "master", "dev")) if structure_file.endswith("_production.json"): structure = Dict().from_json(File(structure_file).read()) return self._update_structure_from_config(structure) elif structure_file.endswith(".json"): return Dict().from_json(File(structure_file).read()) return self._update_structure_from_config(Dict().from_json(req.text))
def parse_env_file(cls, env_file_location): """ Parse the environment file into something we understand. :param str env_file_location: The location of the file we have to parse. """ result = {} content = "" if PyFunceble.path.isfile(env_file_location): content = File(env_file_location).read() for line in content.splitlines(): line = line.strip() if line.startswith("#"): continue if "#" in line: line = line[:line.find("#")] if "=" in line: splited = line.split("=") result[splited[0]] = splited[1] return result
def test_retrieve_file_exist(self): """ Test the case that we want to retrieve a file that exist. """ File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) PyFunceble.INTERN["to_test_type"] = "domain" Dict(self.excepted_content).to_json(self.file) Mining()._retrieve() self.assertEqual(self.excepted_content, PyFunceble.INTERN["mined"]) del PyFunceble.INTERN["mined"] del PyFunceble.INTERN["to_test_type"] File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual)
def test_restore_old_system(self): """ This method test AutoContinue().restore() for the case that we run the most recent version but with data from the old system. """ PyFunceble.CONFIGURATION["auto_continue"] = True File(self.file).delete() old_system = { PyFunceble.CONFIGURATION["file_to_test"]: { "number_of_up": 15, "number_of_down": 18, "number_of_invalid": 5, "number_of_tested": 38, } } Dict(old_system).to_json(self.file) AutoContinue().restore() expected = {"up": 15, "down": 18, "invalid": 5, "tested": 38} actual = PyFunceble.CONFIGURATION["counter"]["number"] self.assertEqual(expected, actual) self.set_counter(0) expected = {"up": 0, "down": 0, "invalid": 0, "tested": 0} actual = PyFunceble.CONFIGURATION["counter"]["number"] self.assertEqual(expected, actual) PyFunceble.CONFIGURATION["auto_continue"] = False File(self.file).delete()
def __init__(self): if (PyFunceble.CONFIGURATION["auto_continue"] and not PyFunceble.CONFIGURATION["no_files"]): # * The auto_continue subsystem is activated. # and # * We are authorized to generate files. # We set the log file location. self.autocontinue_log_file = ( PyFunceble.OUTPUT_DIRECTORY + PyFunceble.OUTPUTS["parent_directory"] + PyFunceble.OUTPUTS["logs"]["filenames"]["auto_continue"]) if PyFunceble.path.isfile(self.autocontinue_log_file): # The log file already exist. # We get its content and save it inside backup_content. self.backup_content = Dict().from_json( File(self.autocontinue_log_file).read()) else: # The log file does not exist. # We initiate the backup content. self.backup_content = {} # And we save our empty backup_content to the log file. File(self.autocontinue_log_file).write(str( self.backup_content))
def test_remove(self): """ Test the deletion subsystem. """ File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) PyFunceble.INTERN["mined"] = self.excepted_content PyFunceble.INTERN["to_test_type"] = "domain" PyFunceble.INTERN["to_test"] = "www.google.com" expected = { PyFunceble.INTERN["file_to_test"]: { "myètherwället.com": ["www.facebook.com"] } } Mining().remove() self.assertEqual(expected, PyFunceble.INTERN["mined"]) del PyFunceble.INTERN["mined"] del PyFunceble.INTERN["to_test"] del PyFunceble.INTERN["to_test_type"] File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file)
def test_to_yaml(self): """ Test Dict().to_yaml. """ file_to_read = "this_yaml_is_a_ghost.yaml" expected = False actual = PyFunceble.path.isfile(file_to_read) self.assertEqual(expected, actual) to_write = {"hello": ["This is PyFunceble!", "Uhh!"], "world": "Fun Ilrys"} expected = "{hello: [This is PyFunceble!, Uhh!], world: Fun Ilrys}\n" Dict(to_write).to_yaml(file_to_read, flow_style=True) expected = """hello: - This is PyFunceble! - Uhh! world: Fun Ilrys """ Dict(to_write).to_yaml(file_to_read, flow_style=False) actual = File(file_to_read).read() self.assertEqual(expected, actual) File(file_to_read).delete() expected = False actual = PyFunceble.path.isfile(file_to_read) self.assertEqual(expected, actual)
def test_retrieve_file_not_exist(self): """ Test the case that we want to retrieve a file that does not exist. """ File(self.file).delete() actual = PyFunceble.path.isfile(self.file) expected = False self.assertEqual(expected, actual) Mining()._retrieve() excepted = {} self.assertEqual(excepted, PyFunceble.INTERN["mined"]) PyFunceble.INTERN["mined"] = {} File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual)
def test_write_overwrite_delete(self): """ Test File.write() along with File.write() for the case that we want to overwrite the content of a file. """ expected = "Hello, World! I'm domain2idna" File("hi").write(expected) with open("hi") as file: actual = file.read() self.assertEqual(expected, actual) expected = "Hello, World! Python is great, you should consider learning it!" File("hi").write(expected, overwrite=True) with open("hi") as file: actual = file.read() self.assertEqual(expected, actual) expected = False File("hi").delete() actual = PyFunceble.path.isfile("hi") self.assertEqual(expected, actual)
def before_header(self): """ Print informations about PyFunceble and the date of generation of a file into a given path, if doesn't exist. """ if not PyFunceble.CONFIGURATION[ "no_files"] and self.output and not path.isfile(self.output): link = ("# File generated with %s\n" % PyFunceble.LINKS["repo"]) date_of_generation = ("# Date of generation: %s \n\n" % PyFunceble.CURRENT_TIME) if self.template in [ "Generic_File", PyFunceble.STATUS["official"]["up"], PyFunceble.STATUS["official"]["down"], PyFunceble.STATUS["official"]["invalid"], "Less", ]: header = self._header_constructor(self.currently_used_header, None)[0] + "\n" try: File(self.output).write(link + date_of_generation + header) except UnboundLocalError: File(self.output).write(link + date_of_generation)
def testhash_file(self): """ Test Hash._hash_file(). """ expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) File(self.file).write("\n".join(self.data_to_write)) expected = True actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) for algo, result in self.expected_hashed.items(): self.assertEqual( result, Hash(self.file)._hash_file(algo), msg="%s did not passed the test" % repr(algo), ) File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual)
def test_save(self): """ Test the saving system. """ File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) self.mining.database = self.excepted_content self.mining.save() expected = True actual = PyFunceble.path.isfile(self.file) self.assertEqual( self.excepted_content, Dict().from_json(File(self.file).read()) ) File(self.file).delete() actual = PyFunceble.path.isfile(self.file) expected = False self.assertEqual(expected, actual)
def _load_config_file(self): """ Load .PyFunceble.yaml into the system. """ try: # We try to load the configuration file. PyFunceble.CONFIGURATION.update( Dict.from_yaml(File(self.path_to_config).read())) # We install the latest iana configuration file. self._install_iana_config() # We install the latest public suffix configuration file. self._install_psl_config() # We install the latest directory structure file. self._install_directory_structure_file() except FileNotFoundError as exception: # But if the configuration file is not found. if PyFunceble.path.isfile(self.path_to_default_config): # The `DEFAULT_CONFIGURATION_FILENAME` file exists. # We copy it as the configuration file. File(self.path_to_default_config).copy(self.path_to_config) # And we load the configuration file as it does exist (yet). self._load_config_file() else: # The `DEFAULT_CONFIGURATION_FILENAME` file does not exists. # We raise the exception we were handling. raise exception
def test_list_of_mined(self): """ Test Mining.list_of_mined """ File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) self.mining.database = self.excepted_content expected = [ ("myètherwället.com", "www.google.com"), ("myètherwället.com", "www.facebook.com"), ] self.assertEqual(expected, self.mining.list_of_mined()) File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file)
def test_remove(self): """ Test the deletion subsystem. """ File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) expected = { self.file_to_test: { "myètherwället.com": ["www.facebook.com", "facebook.com"], "example.org": ["facebook.com"], } } self.mining.database = {self.file_to_test: {}} self.mining["myètherwället.com"] = ["www.facebook.com", "facebook.com"] self.mining["example.org"] = ["www.facebook.com", "facebook.com"] self.mining.remove("example.org", "www.facebook.com") self.assertEqual(expected, self.mining.database) File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file)
def test_add(self): """ Test the addition subsystem. """ File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) expected = { self.file_to_test: {"www.google.com": ["facebook.com", "www.facebook.com"]} } self.mining["www.google.com"] = ["facebook.com", "www.facebook.com"] self.assertEqual(expected, self.mining.database) self.mining["www.google.com"] = ["github.com"] expected[self.file_to_test]["www.google.com"].append("github.com") self.assertEqual(expected, self.mining.database) File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual)
def test_get_specific_algo(self): """ Test Hash.get() for the case that we want a specifig algorithm. """ expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) File(self.file).write("\n".join(self.data_to_write)) expected = True actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) expected = self.expected_hashed["sha512"] actual = Hash(self.file, algorithm="sha512", only_hash=True).get() self.assertEqual(expected, actual) expected = self.expected_hashed["sha512"] actual = Hash( data="\n".join(self.data_to_write), algorithm="sha512", only_hash=True ).get() self.assertEqual(expected, actual) File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual)
def test_backup(self): """ This function test AutoContinue().backup(). """ PyFunceble.CONFIGURATION["auto_continue"] = True File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) self.set_counter(to_set=25) AutoContinue().backup() expected = True actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) expected = { PyFunceble.CONFIGURATION["file_to_test"]: { "up": 25, "down": 25, "invalid": 25, "tested": 25 } } actual = Dict().from_json(File(self.file).read()) self.assertEqual(expected, actual) PyFunceble.CONFIGURATION["auto_continue"] = False File(self.file).delete()
def test_get_all(self): """ Test Hash.get() for the case that we want all. """ expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) File(self.file).write("\n".join(self.data_to_write)) expected = True actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) expected = self.expected_hashed actual = Hash(self.file, algorithm="all").get() self.assertEqual(expected, actual) actual = Hash(data="\n".join(self.data_to_write), algorithm="all").get() self.assertEqual(expected, actual) File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual)
def test_backup(self): """ Test the backup system. """ File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) PyFunceble.INTERN["mined"] = self.excepted_content Mining()._backup() expected = True actual = PyFunceble.path.isfile(self.file) self.assertEqual(self.excepted_content, Dict().from_json(File(self.file).read())) del PyFunceble.INTERN["mined"] File(self.file).delete() actual = PyFunceble.path.isfile(self.file) expected = False self.assertEqual(expected, actual)
def test_hash_file(self): """ Tests the method which let us the content of a given file. """ file_instance = File(self.file) expected = False actual = file_instance.exists() self.assertEqual(expected, actual) file_instance.write("\n".join(self.data_to_write)) expected = True actual = file_instance.exists() self.assertEqual(expected, actual) for algo, expected in self.expected_hashed.items(): self.assertEqual( expected, Hash(algo=algo).file(self.file), ) file_instance.delete() expected = False actual = file_instance.exists() self.assertEqual(expected, actual)
def _before_header(self): """ Print informations about PyFunceble and the date of generation of a file into a given path, if doesn't exist. """ if ( not PyFunceble.CONFIGURATION["no_files"] and self.output and not PyFunceble.path.isfile(self.output) ): # * We are allowed to generate files. # and # * And output is given. # and # * The given output does not exist. # We initiate the information about what generated the file. link = "# File generated by %s\n" % PyFunceble.LINKS["repo"] # We initiate the information about the generation date of this file. date_of_generation = ( "# Date of generation: %s \n\n" % PyFunceble.CURRENT_TIME ) # We initiate a variable which will save the list of # templates which have to meet in order to write the before # header informations. authorized_templates = [ "Generic_File", PyFunceble.STATUS["official"]["up"], PyFunceble.STATUS["official"]["down"], PyFunceble.STATUS["official"]["invalid"], PyFunceble.STATUS["official"]["valid"], "Less", ] if self.template in authorized_templates: # The current header is in our list of authorized templated. # We get the header. header = ( self._header_constructor(self.currently_used_header, None)[0] + "\n" ) try: # We try to print the link, the date of generation and the header in the # given file. File(self.output).write(link + date_of_generation + header) except UnboundLocalError: # We don't have any header. # We print the link and the date in the given file. File(self.output).write(link + date_of_generation)
def setUp(self): """ Setups everything needed for the tests. """ self.file = "this_file_is_a_ghost" self.file_instance = File(self.file) self.file_instance_2 = File(self.file + "_2") self.file_instance.delete() self.file_instance_2.delete()
def test_before_header(self, header_constructor_patch): """ Test the functionability of Prints().before_header() """ File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) expected = """# File generated by {0} (v{1}) / {2} # Date of generation: {3} """.format( PyFunceble.NAME, PyFunceble.VERSION.split()[0], PyFunceble.LINKS["repo"], PyFunceble.CURRENT_TIME + " ", ) Prints(None, None, output_file=self.file, only_on_file=False)._before_header() self.assertEqual(expected, File(self.file).read()) # Test of the case that we have a Generic_File template File(self.file).delete() expected = False actual = PyFunceble.path.isfile(self.file) self.assertEqual(expected, actual) expected = """# File generated by {0} (v{1}) / {2} # Date of generation: {3} Hello World! """.format( PyFunceble.NAME, PyFunceble.VERSION.split()[0], PyFunceble.LINKS["repo"], PyFunceble.CURRENT_TIME + " ", ) header_constructor_patch.return_value = ["Hello World!"] Prints( None, "Generic_File", output_file=self.file, only_on_file=False )._before_header() self.assertEqual(expected, File(self.file).read())
def __init__(self): if PyFunceble.CONFIGURATION["auto_continue"]: self.autocontinue_log_file = PyFunceble.CURRENT_DIRECTORY + PyFunceble.OUTPUTS[ "parent_directory"] + PyFunceble.OUTPUTS["logs"]["filenames"][ "auto_continue"] if path.isfile(self.autocontinue_log_file): self.backup_content = Dict().from_json( File(self.autocontinue_log_file).read()) else: self.backup_content = {} File(self.autocontinue_log_file).write(str( self.backup_content))
def restore(self): """ Restore the 'output/' directory structure based on the `dir_structure.json` file. """ structure = self._get_structure() list_of_key = list(structure.keys()) structure = structure[list_of_key[0]] parent_path = list_of_key[0] + directory_separator for directory in structure: base = self.base + parent_path + directory + directory_separator self._create_directory(base) for file in structure[directory]: file_path = base + file content_to_write = structure[directory][file]["content"] online_sha = structure[directory][file]["sha512"] content_to_write = Regex(content_to_write, "@@@", escape=True, replace_with="\\n").replace() git_to_keep = file_path.replace("gitignore", "keep") keep_to_git = file_path.replace("keep", "gitignore") if self._restore_replace(): if path.isfile(file_path) and Hash( file_path, "sha512", True).get() == online_sha: rename(file_path, git_to_keep) write = False else: File(file_path).delete() file_path = git_to_keep write = True else: if path.isfile(keep_to_git) and Hash( file_path, "sha512", True).get() == online_sha: rename(file_path, keep_to_git) write = False else: File(keep_to_git).delete() file_path = keep_to_git write = True if write: File(file_path).write(content_to_write + "\n", True)
def __sort_generated_files(cls): """ Sort the content of all files we generated. """ for root, _, files in PyFunceble.walk( PyFunceble.OUTPUT_DIRECTORY + PyFunceble.OUTPUTS["parent_directory"] ): # We loop through the list of directories of the output directory. for file in files: # We loop through the list of file of the # currently read directory. if file.endswith(".json"): # The currently read filename ends # with .json. # We continue the loop. continue if file in [".keep", ".gitignore"]: # The currently read filename is # into a list of filename that are not relevant # for us. # We continue the loop. continue # We create an instance of our File(). file_instance = File( "{0}{1}{2}".format(root, PyFunceble.directory_separator, file) ) # We get the content of the current file. file_content = file_instance.read().splitlines() if not PyFunceble.CONFIGURATION["hierarchical_sorting"]: # We do not have to sort hierarchicaly. # We sort the lines of the file standarly. formatted = List(file_content[3:]).custom_format(Sort.standard) else: # We do have to sort hierarchicaly. # We sort the lines of the file hierarchicaly. formatted = List(file_content[3:]).custom_format(Sort.hierarchical) # We finally put the formatted data in place. file_instance.write( "\n".join(file_content[:3] + formatted), overwrite=True )
def test_copy(self): """ Test File.copy(). """ file_to_write = "hello_world" copy_destination = "world_hello" expected = False actual = PyFunceble.path.isfile(file_to_write) self.assertEqual(expected, actual) expected = "Hello, World! Python is great, you should consider learning it!" File(file_to_write).write(expected) with open(file_to_write) as file: actual = file.read() self.assertEqual(expected, actual) expected = False actual = PyFunceble.path.isfile(copy_destination) self.assertEqual(expected, actual) File(file_to_write).copy(copy_destination) expected = True actual = PyFunceble.path.isfile(copy_destination) self.assertEqual(expected, actual) expected = "Hello, World! Python is great, you should consider learning it!" with open(copy_destination) as file: actual = file.read() self.assertEqual(expected, actual) File(copy_destination).delete() File(file_to_write).delete() expected = False actual = PyFunceble.path.isfile(copy_destination) self.assertEqual(expected, actual) actual = PyFunceble.path.isfile(file_to_write) self.assertEqual(expected, actual)
def _update_docs(self, file_to_update): """ Update the given documentation file or :code:`README.rst` so that it always gives branch related URL and informations. .. note:: This only apply to :code:`dev` and :code:`master` branch. :param str file_to_update: The file to update. """ if self.is_dev_version(): # The current version is the dev version. # We map what we have to replace. # Format: {match:replacement} regexes = { "/%s/" % "dev": r"\/%s\/" % "master", "=%s" % "dev": "=%s" % "master", } elif self.is_master_version(): # The current version is the master version. # We map what we have to replace. regexes = { "/%s/" % "master": r"\/%s\/" % "dev", "=%s" % "master": "=%s" % "dev", } else: # The current version is not the master nor the dev version. # We raise an exception as the branch we are currently is not meaned # for production. raise Exception("Please switch to `dev` or `master` branch.") file_instance = File(file_to_update) # We get the content of the file to fix. to_update = file_instance.read() for replacement, regex in regexes.items(): # We loop through reach element of the map. # We process the replacement. to_update = Regex(to_update, regex, replace_with=replacement).replace() # We finally overwrite the file to fix with the filtered. # content. file_instance.write(to_update, overwrite=True)