def _install_production_config(self):
"""
Download the production configuration and install it in the
current directory.
"""
# We initiate the link to the production configuration.
# It is not hard coded because this method is called only if we
# are sure that the configuration file exist.
production_config_link = "https://raw.githubusercontent.com/funilrys/PyFunceble/master/.PyFunceble_production.yaml" # pylint: disable=line-too-long
# We update the link according to our current version.
production_config_link = Version(True).right_url_from_version(
production_config_link)
if not Version(True).is_cloned():
# The current version is not the cloned one.
# We download the link content and save it inside the default location.
#
# Note: We add this one in order to allow the enduser to always have
# a copy of our upstream configuration file.
Download(production_config_link,
self.path_to_default_config).text()
# And we download the link content and return the download status.
return Download(production_config_link, self.path_to_config).text()
def __init__(self, configuration_path):
config_link = Version(True).right_url_from_version(
"https://raw.githubusercontent.com/funilrys/PyFunceble/master/.PyFunceble_production.yaml" # pylint: disable=line-too-long
)
self.path_to_config = configuration_path
if not self.path_to_config.endswith(PyFunceble.directory_separator):
self.path_to_config += PyFunceble.directory_separator
self.path_to_config += PyFunceble.CONFIGURATION_FILENAME
dict_instance = Dict()
self.local_config = dict_instance.from_yaml(
File(self.path_to_config).read())
self.upstream_config = dict_instance.from_yaml(
Download(config_link, return_data=True).text())
if self.upstream_config["links"]["config"] != config_link:
self.upstream_config = dict_instance.from_yaml(
Download(self.upstream_config["links"]["config"],
return_data=True).text())
self.new_config = {}
self._load()
def _extensions(self):
"""
Extract the extention from the given block.
Plus get its referer.
"""
upstream_lines = (Download(
self.iana_url,
return_data=True).text().split('<span class="domain tld">'))
# We extract the different extension from the currently readed line.
regex_valid_extension = r"(/domains/root/db/)(.*)(\.html)"
for block in upstream_lines:
if "/domains/root/db/" in block:
# The link is in the line.
# We try to extract the extension.
matched = Regex(block,
regex_valid_extension,
return_data=True,
rematch=True).match()[1]
if matched:
# The extraction is not empty or None.
# We get the referer.
referer = self._referer(matched)
# We yield the matched extension and its referer.
yield (matched, referer)
def _install_directory_structure_file(cls):
"""
Download the latest version of `dir_structure_production.json`.
"""
# We initiate the link to the public suffix configuration.
# It is not hard coded because this method is called only if we
# are sure that the configuration file exist.
dir_structure_link = PyFunceble.CONFIGURATION["links"]["dir_structure"]
# We update the link according to our current version.
dir_structure_link = Version(True).right_url_from_version(
dir_structure_link)
# We set the destination of the downloaded file.
destination = (PyFunceble.CURRENT_DIRECTORY +
PyFunceble.CONFIGURATION["outputs"]["default_files"]
["dir_structure"])
if not Version(True).is_cloned() or not PyFunceble.path.isfile(
destination):
# The current version is not the cloned version.
# We Download the link content and return the download status.
data = Download(dir_structure_link, destination,
return_data=True).text()
File(destination).write(data, overwrite=True)
return True
# We are in the cloned version.
# We do not need to download the file, so we are returning None.
return None
def _install_psl_config(cls):
"""
Download `public-suffix.json` if not present.
"""
# We initiate the link to the public suffix configuration.
# It is not hard coded because this method is called only if we
# are sure that the configuration file exist.
psl_link = PyFunceble.CONFIGURATION["links"]["psl"]
# We update the link according to our current version.
psl_link = Version(True).right_url_from_version(psl_link)
# We set the destination of the downloaded file.
destination = (PyFunceble.CURRENT_DIRECTORY +
PyFunceble.CONFIGURATION["outputs"]["default_files"]
["public_suffix"])
if not Version(True).is_cloned() or not PyFunceble.path.isfile(
destination):
# The current version is not the cloned version.
# We Download the link content and return the download status.
return Download(psl_link, destination).text()
# We are in the cloned version.
# We do not need to download the file, so we are returning None.
return None
def _install_iana_config(cls):
"""
Download `iana-domains-db.json` if not present.
"""
# We initiate the link to the iana configuration.
# It is not hard coded because this method is called only if we
# are sure that the configuration file exist.
iana_link = PyFunceble.CONFIGURATION["links"]["iana"]
# We update the link according to our current version.
iana_link = Version(True).right_url_from_version(iana_link)
# We set the destination of the downloaded file.
destination = PyFunceble.CURRENT_DIRECTORY + "iana-domains-db.json"
if not Version(True).is_cloned() or not PyFunceble.path.isfile(
destination):
# The current version is not the cloned version.
# We Download the link content and return the download status.
return Download(iana_link, destination).text()
# We are in the cloned version.
# We do not need to download the file, so we are returning None.
return None
def update(self):
"""
Update the content of the `iana-domains-db` file.
"""
if not PyFunceble.CONFIGURATION["quiet"]:
# * The quiet mode is not activated.
# We print on screen what we are doing.
print("Update of iana-domains-db", end=" ")
# We loop through the line of the iana website.
list(
map(
self._extensions,
Download(self.iana_url, return_data=True)
.text()
.split('<span class="domain tld">'),
)
)
if not PyFunceble.CONFIGURATION["quiet"]:
# The quiet mode is not activated.
# We indicate that the work is done without any issue.
print(PyFunceble.INTERN["done"])
def data(cls):
"""
Get the database from IANA website.
"""
iana_url = "https://www.iana.org/domains/root/db"
return Download(iana_url, return_data=True).text()
def __init__(self, configuration_path):
self.path_to_config = configuration_path
if not self.path_to_config.endswith(PyFunceble.directory_separator):
self.path_to_config += PyFunceble.directory_separator
self.path_to_config += PyFunceble.CONFIGURATION_FILENAME
self.upstream_config = Dict().from_yaml(
Download(PyFunceble.LINKS["config"], return_data=True).text())
if self.upstream_config["links"]["config"] != PyFunceble.LINKS[
"config"]:
self.upstream_config = Dict().from_yaml(
Download(self.upstream_config["links"]["repo"],
return_data=True).text())
self.new_config = {}
self._load()
def _data(cls):
"""
Get the database from the public suffix repository.
"""
# We initiate a variable which will save the link to the upstream public suffix file.
public_suffix_url = (
"https://raw.githubusercontent.com/publicsuffix/list/%s/public_suffix_list.dat"
% "master")
# And we return the content of the previously declared link.
return Download(public_suffix_url, return_data=True).text()
def __init__(self, used=False):
if not used:
self.local_splited = self.split_versions(PyFunceble.VERSION)
upstream_link = "https://raw.githubusercontent.com/funilrys/PyFunceble/master/version.yaml" # pylint: disable=line-too-long
if "dev" in PyFunceble.VERSION:
upstream_link = upstream_link.replace("master", "dev")
else:
upstream_link = upstream_link.replace("dev", "master")
self.upstream_data = Dict().from_yaml(
Download(upstream_link, return_data=True).text())
def _referer(self, extension):
"""
Return the referer for the given extension.
:param extension: A valid domain extension.
:type extension: str
:return: The whois server to use to get the WHOIS record.
:rtype: str
"""
# We get the a copy of the page.
iana_record = Download(
self.iana_url + "/" + extension + ".html", return_data=True
).text()
if iana_record:
# The record is not empty.
# We initiate a regex which will extract the referer.
regex_referer = r"(?s)\<b\>(?:WHOIS\sServer:)\<\/b>\s+([a-zA-Z0-9._-]+)"
# We try to extract the referer.
matched = Regex(
iana_record, regex_referer, return_data=True, group=1
).match()
if matched:
# The referer was extracted successfully.
# We return the matched referer.
return matched
# *The referer was not extracted successfully.
# or
# * The iana record is empty.
if extension in self.manual_server:
# The extension is in the list of manual entries.
# We return the server which we set manually.
return self.manual_server[extension]
# We return None because we weren't able to get the server to call for
# the given extension.
return None
def install_iana_config(cls):
"""
This method download `iana-domains-db.json` if not present.
"""
iana_link = "https://raw.githubusercontent.com/funilrys/PyFunceble/master/iana-domains-db.json" # pylint: disable=line-too-long
destination = PyFunceble.CURRENT_DIRECTORY + "iana-domains-db.json"
if "dev" in PyFunceble.VERSION:
iana_link = iana_link.replace("master", "dev")
else:
iana_link = iana_link.replace("dev", "master")
if not path.isfile(destination):
return Download(iana_link, destination).text()
return True
def _entry_management_url_download(cls, passed):
"""
Check if the given information is a URL.
If it is the case, it download and update the location of file to test.
:param passed: The url passed to the system.
:type passed: str
:return: The state of the check.
:rtype: bool
"""
if passed and Check().is_url_valid(passed):
# The passed string is an URL.
# We get the file name based on the URL.
# We actually just get the string after the last `/` in the URL.
file_to_test = passed.split("/")[-1]
if (
not PyFunceble.path.isfile(file_to_test)
or PyFunceble.CONFIGURATION["counter"]["number"]["tested"] == 0
):
# The filename does not exist in the current directory
# or the currently number of tested is equal to 0.
# We download the content of the link.
Download(passed, file_to_test).text()
# The files does exist or the currently number of tested is greater than
# 0.
# We initiate the file we have to test.
PyFunceble.INTERN["file_to_test"] = file_to_test
# We return true to say that everything goes right.
return True
# The passed string is not an URL.
# We do not need to do anything else.
return False
def update(self):
"""
Update the content of `iana-domains-db.json` file.
"""
if not PyFunceble.CONFIGURATION["quiet"]:
# The quiet mode is not activated.
# We print on screen what we are doing.
print(
"Update of {0}".format(
PyFunceble.OUTPUTS["default_files"]["iana"]),
end=" ",
)
upstream_lines = (Download(
self.iana_url,
return_data=True).text().split('<span class="domain tld">'))
with Pool(PyFunceble.CONFIGURATION["maximal_processes"]) as pool:
already_checked = []
for extension, referer, referer_checked in pool.map(
self._get_extension_and_referer_from_block,
upstream_lines):
if (extension is not None
and referer) and (referer_checked
or referer in already_checked):
if (extension not in self.iana_db
or self.iana_db[extension] != referer):
# We add the extension to the databae.
self.iana_db[extension] = referer
already_checked.append(referer)
# We save the content of the constructed database.
Dict(self.iana_db).to_json(self.destination)
if not PyFunceble.CONFIGURATION["quiet"]:
# The quiet mode is not activated.
# We indicate that the work is done without any issue.
print(PyFunceble.INTERN["done"] + "\n")
def install_production_config(self):
"""
This method download the production configuration and install it in the
current directory.
Argument:
- path_to_config: str
The path were we have to install the configuration file.
"""
production_config_link = "https://raw.githubusercontent.com/funilrys/PyFunceble/master/.PyFunceble_production.yaml" # pylint: disable=line-too-long
if "dev" in PyFunceble.VERSION:
production_config_link = production_config_link.replace(
"master", "dev")
else:
production_config_link = production_config_link.replace(
"dev", "master")
return Download(production_config_link, self.path_to_config).text()
def __init__(self, used=False):
if not used:
# A method of this class is not called directly.
# We split the local version.
self.local_splited = self.split_versions(PyFunceble.VERSION)
# We initiate the link to the upstream version file.
# It is hard coded because we may not have the chance to have the
# configuration file everytime we need it.
upstream_link = (
"https://raw.githubusercontent.com/funilrys/PyFunceble/master/version.yaml"
) # pylint: disable=line-too-long
# We update the link according to our current version.
upstream_link = Version(True).right_url_from_version(upstream_link)
# We get the link content and convert it to a dict which is more
# usable.
self.upstream_data = Dict().from_yaml(
Download(upstream_link, return_data=True).text())
def download_link(self): # pragma: no cover
"""
Download the file if it is an URL.
"""
if PyFunceble.Check(self.file).is_url():
# We get the destination.
destination = self.file.split("/")[-1]
if self.file and self.autocontinue.is_empty():
# The given file is an URL.
if (not PyFunceble.path.isfile(destination) or
PyFunceble.INTERN["counter"]["number"]["tested"] == 0):
# The filename does not exist in the current directory
# or the currently number of tested is equal to 0.
# We download the content of the link.
Download(self.file, destination).text()
# We update the global file with the destination.
self.file = destination
def _install_db_type_files(self):
"""
Create the .db_type directory if it does not exists and update
its content.
"""
if not self.version.is_cloned():
# * The current version is not the cloned version.
# and
# * The database type is not JSON.
destination_dir = (
PyFunceble.CONFIG_DIRECTORY +
PyFunceble.CONFIGURATION["outputs"]["db_type"]["directory"] +
PyFunceble.directory_separator)
if not PyFunceble.path.isdir(destination_dir):
PyFunceble.mkdir(destination_dir)
# We set the list of index to download.
index_to_download = ["sqlite", "mariadb", "mysql"]
for index in index_to_download:
# We loop through the list of indexes.
# We create the right link.
link_to_download = self.version.right_url_from_version(
PyFunceble.CONFIGURATION["links"][index])
# We create the destination.
destination = (destination_dir +
PyFunceble.CONFIGURATION["outputs"]["db_type"]
["files"][index])
# We finally download the file.
Download(link_to_download, destination).text()
def __init__(self, domain=None, file_path=None, **args):
optional_arguments = {
"url_to_test": None,
"file_urls": None,
"modulo_test": False,
"link_to_test": None,
}
# We initiate our optional_arguments in order to be usable all over the
# class
for (arg, default) in optional_arguments.items():
setattr(self, arg, args.get(arg, default))
if not self.modulo_test: # pylint: disable=no-member
PyFunceble.CONFIGURATION["file_to_test"] = file_path # pylint: disable=no-member
if self.file_urls: # pylint: disable=no-member
PyFunceble.CONFIGURATION["file_to_test"] = self.file_urls # pylint: disable=no-member
if PyFunceble.CONFIGURATION["travis"]:
AutoSave().travis_permissions()
self.bypass()
ExecutionTime("start")
if domain:
PyFunceble.CONFIGURATION["show_percentage"] = False
PyFunceble.CONFIGURATION["domain"] = domain.lower()
self.domain()
elif self.url_to_test and not file_path: # pylint: disable=no-member
PyFunceble.CONFIGURATION["show_percentage"] = False
PyFunceble.CONFIGURATION["URL"] = self.url_to_test # pylint: disable=no-member
self.url()
elif self.file_urls: # pylint: disable=no-member
PyFunceble.CONFIGURATION[
"no_whois"] = PyFunceble.CONFIGURATION[
"plain_list_domain"] = PyFunceble.CONFIGURATION[
"split"] = True
PyFunceble.CONFIGURATION["generate_hosts"] = False
self.url_file()
elif file_path:
self.file()
elif self.link_to_test and self.link_to_test.startswith( # pylint: disable=no-member
"http"):
file_to_test = self.link_to_test.split( # pylint: disable=no-member
"/")[-1]
Download(
self.link_to_test,
file_to_test # pylint: disable=no-member
).text()
PyFunceble.CONFIGURATION["file_to_test"] = file_to_test
self.file()
ExecutionTime("stop")
Percentage().log()
if domain:
self.colored_logo()
else:
PyFunceble.CONFIGURATION["simple"] = True
PyFunceble.CONFIGURATION["quiet"] = True
PyFunceble.CONFIGURATION["no_files"] = True
if domain:
PyFunceble.CONFIGURATION["domain"] = domain.lower()