Example #1
0
    def _install_production_config(self):
        """
        Download the production configuration and install it in the
        current directory.
        """

        # We initiate the link to the production configuration.
        # It is not hard coded because this method is called only if we
        # are sure that the configuration file exist.
        production_config_link = "https://raw.githubusercontent.com/funilrys/PyFunceble/master/.PyFunceble_production.yaml"  # pylint: disable=line-too-long

        # We update the link according to our current version.
        production_config_link = Version(True).right_url_from_version(
            production_config_link)

        if not Version(True).is_cloned():
            # The current version is not the cloned one.

            # We download the link content and save it inside the default location.
            #
            # Note: We add this one in order to allow the enduser to always have
            # a copy of our upstream configuration file.
            Download(production_config_link,
                     self.path_to_default_config).text()

        # And we download the link content and return the download status.
        return Download(production_config_link, self.path_to_config).text()
Example #2
0
    def __init__(self, configuration_path):
        config_link = Version(True).right_url_from_version(
            "https://raw.githubusercontent.com/funilrys/PyFunceble/master/.PyFunceble_production.yaml"  # pylint: disable=line-too-long
        )

        self.path_to_config = configuration_path

        if not self.path_to_config.endswith(PyFunceble.directory_separator):
            self.path_to_config += PyFunceble.directory_separator

        self.path_to_config += PyFunceble.CONFIGURATION_FILENAME

        dict_instance = Dict()

        self.local_config = dict_instance.from_yaml(
            File(self.path_to_config).read())
        self.upstream_config = dict_instance.from_yaml(
            Download(config_link, return_data=True).text())

        if self.upstream_config["links"]["config"] != config_link:
            self.upstream_config = dict_instance.from_yaml(
                Download(self.upstream_config["links"]["config"],
                         return_data=True).text())

        self.new_config = {}

        self._load()
Example #3
0
    def _extensions(self):
        """
        Extract the extention from the given block.
        Plus get its referer.
        """

        upstream_lines = (Download(
            self.iana_url,
            return_data=True).text().split('<span class="domain tld">'))

        # We extract the different extension from the currently readed line.
        regex_valid_extension = r"(/domains/root/db/)(.*)(\.html)"

        for block in upstream_lines:
            if "/domains/root/db/" in block:
                # The link is in the line.

                # We try to extract the extension.
                matched = Regex(block,
                                regex_valid_extension,
                                return_data=True,
                                rematch=True).match()[1]

                if matched:
                    # The extraction is not empty or None.

                    # We get the referer.
                    referer = self._referer(matched)

                    # We yield the matched extension and its referer.
                    yield (matched, referer)
Example #4
0
    def _install_directory_structure_file(cls):
        """
        Download the latest version of `dir_structure_production.json`.
        """

        # We initiate the link to the public suffix configuration.
        # It is not hard coded because this method is called only if we
        # are sure that the configuration file exist.
        dir_structure_link = PyFunceble.CONFIGURATION["links"]["dir_structure"]

        # We update the link according to our current version.
        dir_structure_link = Version(True).right_url_from_version(
            dir_structure_link)

        # We set the destination of the downloaded file.
        destination = (PyFunceble.CURRENT_DIRECTORY +
                       PyFunceble.CONFIGURATION["outputs"]["default_files"]
                       ["dir_structure"])

        if not Version(True).is_cloned() or not PyFunceble.path.isfile(
                destination):
            # The current version is not the cloned version.

            # We Download the link content and return the download status.
            data = Download(dir_structure_link, destination,
                            return_data=True).text()

            File(destination).write(data, overwrite=True)
            return True

        # We are in the cloned version.

        # We do not need to download the file, so we are returning None.
        return None
Example #5
0
    def _install_psl_config(cls):
        """
        Download `public-suffix.json` if not present.
        """

        # We initiate the link to the public suffix configuration.
        # It is not hard coded because this method is called only if we
        # are sure that the configuration file exist.
        psl_link = PyFunceble.CONFIGURATION["links"]["psl"]

        # We update the link according to our current version.
        psl_link = Version(True).right_url_from_version(psl_link)

        # We set the destination of the downloaded file.
        destination = (PyFunceble.CURRENT_DIRECTORY +
                       PyFunceble.CONFIGURATION["outputs"]["default_files"]
                       ["public_suffix"])

        if not Version(True).is_cloned() or not PyFunceble.path.isfile(
                destination):
            # The current version is not the cloned version.

            # We Download the link content and return the download status.
            return Download(psl_link, destination).text()

        # We are in the cloned version.

        # We do not need to download the file, so we are returning None.
        return None
Example #6
0
    def _install_iana_config(cls):
        """
        Download `iana-domains-db.json` if not present.
        """

        # We initiate the link to the iana configuration.
        # It is not hard coded because this method is called only if we
        # are sure that the configuration file exist.
        iana_link = PyFunceble.CONFIGURATION["links"]["iana"]

        # We update the link according to our current version.
        iana_link = Version(True).right_url_from_version(iana_link)

        # We set the destination of the downloaded file.
        destination = PyFunceble.CURRENT_DIRECTORY + "iana-domains-db.json"

        if not Version(True).is_cloned() or not PyFunceble.path.isfile(
                destination):
            # The current version is not the cloned version.

            # We Download the link content and return the download status.
            return Download(iana_link, destination).text()

        # We are in the cloned version.

        # We do not need to download the file, so we are returning None.
        return None
Example #7
0
    def update(self):
        """
        Update the content of the `iana-domains-db` file.
        """

        if not PyFunceble.CONFIGURATION["quiet"]:
            # * The quiet mode is not activated.

            # We print on screen what we are doing.
            print("Update of iana-domains-db", end=" ")

        # We loop through the line of the iana website.
        list(
            map(
                self._extensions,
                Download(self.iana_url, return_data=True)
                .text()
                .split('<span class="domain tld">'),
            )
        )

        if not PyFunceble.CONFIGURATION["quiet"]:
            # The quiet mode is not activated.

            # We indicate that the work is done without any issue.
            print(PyFunceble.INTERN["done"])
Example #8
0
    def data(cls):
        """
        Get the database from IANA website.
        """
        iana_url = "https://www.iana.org/domains/root/db"

        return Download(iana_url, return_data=True).text()
Example #9
0
    def __init__(self, configuration_path):
        self.path_to_config = configuration_path

        if not self.path_to_config.endswith(PyFunceble.directory_separator):
            self.path_to_config += PyFunceble.directory_separator

        self.path_to_config += PyFunceble.CONFIGURATION_FILENAME

        self.upstream_config = Dict().from_yaml(
            Download(PyFunceble.LINKS["config"], return_data=True).text())

        if self.upstream_config["links"]["config"] != PyFunceble.LINKS[
                "config"]:
            self.upstream_config = Dict().from_yaml(
                Download(self.upstream_config["links"]["repo"],
                         return_data=True).text())

        self.new_config = {}

        self._load()
Example #10
0
    def _data(cls):
        """
        Get the database from the public suffix repository.
        """

        # We initiate a variable which will save the link to the upstream public suffix file.
        public_suffix_url = (
            "https://raw.githubusercontent.com/publicsuffix/list/%s/public_suffix_list.dat"
            % "master")

        # And we return the content of the previously declared link.
        return Download(public_suffix_url, return_data=True).text()
Example #11
0
    def __init__(self, used=False):
        if not used:
            self.local_splited = self.split_versions(PyFunceble.VERSION)

            upstream_link = "https://raw.githubusercontent.com/funilrys/PyFunceble/master/version.yaml"  # pylint: disable=line-too-long

            if "dev" in PyFunceble.VERSION:
                upstream_link = upstream_link.replace("master", "dev")
            else:
                upstream_link = upstream_link.replace("dev", "master")

            self.upstream_data = Dict().from_yaml(
                Download(upstream_link, return_data=True).text())
Example #12
0
    def _referer(self, extension):
        """
        Return the referer for the given extension.

        :param extension: A valid domain extension.
        :type extension: str

        :return: The whois server to use to get the WHOIS record.
        :rtype: str
        """

        # We get the a copy of the page.
        iana_record = Download(
            self.iana_url + "/" + extension + ".html", return_data=True
        ).text()

        if iana_record:
            # The record is not empty.

            # We initiate a regex which will extract the referer.
            regex_referer = r"(?s)\<b\>(?:WHOIS\sServer:)\<\/b>\s+([a-zA-Z0-9._-]+)"

            # We try to extract the referer.
            matched = Regex(
                iana_record, regex_referer, return_data=True, group=1
            ).match()

            if matched:
                # The referer was extracted successfully.

                # We return the matched referer.
                return matched

        # *The referer was not extracted successfully.
        # or
        # * The iana record is empty.

        if extension in self.manual_server:
            # The extension is in the list of manual entries.

            # We return the server which we set manually.
            return self.manual_server[extension]

        # We return None because we weren't able to get the server to call for
        # the given extension.
        return None
Example #13
0
    def install_iana_config(cls):
        """
        This method download `iana-domains-db.json` if not present.
        """

        iana_link = "https://raw.githubusercontent.com/funilrys/PyFunceble/master/iana-domains-db.json"  # pylint: disable=line-too-long
        destination = PyFunceble.CURRENT_DIRECTORY + "iana-domains-db.json"

        if "dev" in PyFunceble.VERSION:
            iana_link = iana_link.replace("master", "dev")
        else:
            iana_link = iana_link.replace("dev", "master")

        if not path.isfile(destination):
            return Download(iana_link, destination).text()

        return True
Example #14
0
    def _entry_management_url_download(cls, passed):
        """
        Check if the given information is a URL.
        If it is the case, it download and update the location of file to test.

        :param passed: The url passed to the system.
        :type passed: str

        :return: The state of the check.
        :rtype: bool
        """

        if passed and Check().is_url_valid(passed):
            # The passed string is an URL.

            # We get the file name based on the URL.
            # We actually just get the  string after the last `/` in the URL.
            file_to_test = passed.split("/")[-1]

            if (
                not PyFunceble.path.isfile(file_to_test)
                or PyFunceble.CONFIGURATION["counter"]["number"]["tested"] == 0
            ):
                # The filename does not exist in the current directory
                # or the currently number of tested is equal to 0.

                # We download the content of the link.
                Download(passed, file_to_test).text()

            # The files does exist or the currently number of tested is greater than
            # 0.

            # We initiate the file we have to test.
            PyFunceble.INTERN["file_to_test"] = file_to_test

            # We return true to say that everything goes right.
            return True

        # The passed string is not an URL.

        # We do not need to do anything else.
        return False
Example #15
0
    def update(self):
        """
        Update the content of `iana-domains-db.json` file.
        """

        if not PyFunceble.CONFIGURATION["quiet"]:
            # The quiet mode is not activated.

            # We print on screen what we are doing.
            print(
                "Update of {0}".format(
                    PyFunceble.OUTPUTS["default_files"]["iana"]),
                end=" ",
            )

        upstream_lines = (Download(
            self.iana_url,
            return_data=True).text().split('<span class="domain tld">'))

        with Pool(PyFunceble.CONFIGURATION["maximal_processes"]) as pool:
            already_checked = []
            for extension, referer, referer_checked in pool.map(
                    self._get_extension_and_referer_from_block,
                    upstream_lines):
                if (extension is not None
                        and referer) and (referer_checked
                                          or referer in already_checked):
                    if (extension not in self.iana_db
                            or self.iana_db[extension] != referer):
                        # We add the extension to the databae.
                        self.iana_db[extension] = referer

                already_checked.append(referer)

        # We save the content of the constructed database.
        Dict(self.iana_db).to_json(self.destination)

        if not PyFunceble.CONFIGURATION["quiet"]:
            # The quiet mode is not activated.

            # We indicate that the work is done without any issue.
            print(PyFunceble.INTERN["done"] + "\n")
Example #16
0
    def install_production_config(self):
        """
        This method download the production configuration and install it in the
        current directory.

        Argument:
            - path_to_config: str
                The path were we have to install the configuration file.
        """

        production_config_link = "https://raw.githubusercontent.com/funilrys/PyFunceble/master/.PyFunceble_production.yaml"  # pylint: disable=line-too-long

        if "dev" in PyFunceble.VERSION:
            production_config_link = production_config_link.replace(
                "master", "dev")
        else:
            production_config_link = production_config_link.replace(
                "dev", "master")

        return Download(production_config_link, self.path_to_config).text()
Example #17
0
    def __init__(self, used=False):
        if not used:
            # A method of this class is not called directly.

            # We split the local version.
            self.local_splited = self.split_versions(PyFunceble.VERSION)

            # We initiate the link to the upstream version file.
            # It is hard coded because we may not have the chance to have the
            # configuration file everytime we need it.
            upstream_link = (
                "https://raw.githubusercontent.com/funilrys/PyFunceble/master/version.yaml"
            )  # pylint: disable=line-too-long

            # We update the link according to our current version.
            upstream_link = Version(True).right_url_from_version(upstream_link)

            # We get the link content and convert it to a dict which is more
            # usable.
            self.upstream_data = Dict().from_yaml(
                Download(upstream_link, return_data=True).text())
Example #18
0
    def download_link(self):  # pragma: no cover
        """
        Download the file if it is an URL.
        """

        if PyFunceble.Check(self.file).is_url():
            # We get the destination.
            destination = self.file.split("/")[-1]

            if self.file and self.autocontinue.is_empty():
                # The given file is an URL.

                if (not PyFunceble.path.isfile(destination) or
                        PyFunceble.INTERN["counter"]["number"]["tested"] == 0):
                    # The filename does not exist in the current directory
                    # or the currently number of tested is equal to 0.

                    # We download the content of the link.
                    Download(self.file, destination).text()

            # We update the global file with the destination.
            self.file = destination
Example #19
0
    def _install_db_type_files(self):
        """
        Create the .db_type directory if it does not exists and update
        its content.
        """

        if not self.version.is_cloned():
            # * The current version is not the cloned version.
            # and
            # * The database type is not JSON.

            destination_dir = (
                PyFunceble.CONFIG_DIRECTORY +
                PyFunceble.CONFIGURATION["outputs"]["db_type"]["directory"] +
                PyFunceble.directory_separator)

            if not PyFunceble.path.isdir(destination_dir):
                PyFunceble.mkdir(destination_dir)

            # We set the list of index to download.
            index_to_download = ["sqlite", "mariadb", "mysql"]

            for index in index_to_download:
                # We loop through the list of indexes.

                # We create the right link.
                link_to_download = self.version.right_url_from_version(
                    PyFunceble.CONFIGURATION["links"][index])

                # We create the destination.
                destination = (destination_dir +
                               PyFunceble.CONFIGURATION["outputs"]["db_type"]
                               ["files"][index])

                # We finally download the file.
                Download(link_to_download, destination).text()
Example #20
0
    def __init__(self, domain=None, file_path=None, **args):

        optional_arguments = {
            "url_to_test": None,
            "file_urls": None,
            "modulo_test": False,
            "link_to_test": None,
        }

        # We initiate our optional_arguments in order to be usable all over the
        # class
        for (arg, default) in optional_arguments.items():
            setattr(self, arg, args.get(arg, default))

        if not self.modulo_test:  # pylint: disable=no-member

            PyFunceble.CONFIGURATION["file_to_test"] = file_path  # pylint: disable=no-member

            if self.file_urls:  # pylint: disable=no-member
                PyFunceble.CONFIGURATION["file_to_test"] = self.file_urls  # pylint: disable=no-member

            if PyFunceble.CONFIGURATION["travis"]:
                AutoSave().travis_permissions()

            self.bypass()
            ExecutionTime("start")

            if domain:
                PyFunceble.CONFIGURATION["show_percentage"] = False
                PyFunceble.CONFIGURATION["domain"] = domain.lower()
                self.domain()
            elif self.url_to_test and not file_path:  # pylint: disable=no-member
                PyFunceble.CONFIGURATION["show_percentage"] = False
                PyFunceble.CONFIGURATION["URL"] = self.url_to_test  # pylint: disable=no-member
                self.url()
            elif self.file_urls:  # pylint: disable=no-member
                PyFunceble.CONFIGURATION[
                    "no_whois"] = PyFunceble.CONFIGURATION[
                        "plain_list_domain"] = PyFunceble.CONFIGURATION[
                            "split"] = True
                PyFunceble.CONFIGURATION["generate_hosts"] = False

                self.url_file()
            elif file_path:
                self.file()
            elif self.link_to_test and self.link_to_test.startswith(  # pylint: disable=no-member
                    "http"):
                file_to_test = self.link_to_test.split(  # pylint: disable=no-member
                    "/")[-1]
                Download(
                    self.link_to_test,
                    file_to_test  # pylint: disable=no-member
                ).text()

                PyFunceble.CONFIGURATION["file_to_test"] = file_to_test

                self.file()

            ExecutionTime("stop")
            Percentage().log()

            if domain:
                self.colored_logo()
        else:
            PyFunceble.CONFIGURATION["simple"] = True
            PyFunceble.CONFIGURATION["quiet"] = True
            PyFunceble.CONFIGURATION["no_files"] = True

            if domain:
                PyFunceble.CONFIGURATION["domain"] = domain.lower()