def almost_everything(self, clean_all=False):
"""
Delete almost all discovered files.
:param bool clean_all:
Tell the subsystem if we have to clean everything instesd
of almost everything.
"""
# We get the list of file to delete.
to_delete = self.file_to_delete()
version = Version(True)
if not version.is_cloned() and clean_all: # pragma: no cover
to_delete.extend(self.databases_to_delete())
for file in to_delete:
# We loop through the list of file to delete.
# And we delete the currently read file.
File(file).delete()
if clean_all:
if PyFunceble.CONFIGURATION["db_type"] == "sqlite":
from PyFunceble.sqlite import SQLite
sqlite_db = SQLite()
for database_name in [
y for x, y in sqlite_db.tables.items() if x != "whois"
]:
query = "DELETE FROM {0}".format(database_name)
sqlite_db.cursor.execute(query)
sqlite_db.connection.commit()
elif PyFunceble.CONFIGURATION["db_type"] in ["mariadb", "mysql"]:
from PyFunceble.mysql import MySQL
mysql_db = MySQL()
for database_name in [
y for x, y in mysql_db.tables.items() if x != "whois"
]:
query = "DELETE FROM {0}".format(database_name)
with mysql_db.get_connection() as cursor:
cursor.execute(query)
if not version.is_cloned() and clean_all: # pragma: no cover
PyFunceble.Load(PyFunceble.CONFIG_DIRECTORY)
def __init__(self, production=False):
# We set the base directory where we are going to replicate
# the directory structure.
self.base = PyFunceble.OUTPUT_DIRECTORY
if not self.base.endswith(PyFunceble.directory_separator):
# The base path does not ends wiith a directory separator.
# We append the directory separator to the ends.
self.base += PyFunceble.directory_separator
# We set the structure base.
self.structure = (self.base +
PyFunceble.OUTPUTS["default_files"]["dir_structure"])
if production and Version(True).is_cloned():
# We are preparing the repository for production.
# We backup the directory structure.
self.backup()
else:
# We are not preparing the repository for production.
# We restore the directory structure.
self.restore()
def __init__(self):
self.data_version_yaml = self._get_current_version_yaml()
self.version_yaml = Version(True).split_versions(
self.data_version_yaml["current_version"])
self.current_version = Version(True).split_versions(
PyFunceble.VERSION, True)
if self._is_version_greater():
DirectoryStructure(production=True)
if self._does_require_deprecation():
to_deprecate = ".".join(self.version_yaml)
self.data_version_yaml["deprecated"].append(to_deprecate)
if self._does_require_force_update():
to_force_update = ".".join(self.version_yaml)
self.data_version_yaml["force_update"][
"minimal_version"].append(to_force_update)
if self.current_version[-1]:
self.current_version[0].append(self.current_version[-1])
self.data_version_yaml["current_version"] = ".".join(
self.current_version[0])
Dict(self.data_version_yaml).to_yaml(PyFunceble.CURRENT_DIRECTORY +
"version.yaml")
self._update_readme_md()
message = Fore.GREEN + Style.BRIGHT + "We are ready to ship!! \n"
message += Fore.CYAN + "Please do not touch version.yaml nor setup.py (version update)"
print(message)
exit(0)
else:
print(
Fore.YELLOW + Style.BRIGHT +
"Are you sure that you did some changes ? Please update PyFunceble.VERSION if it is the case." # pylint: disable=line-too-long
)
exit(1)
def _is_version_greater(self):
"""
This method check if the current version is greater as the older older one.
"""
checked = Version(True).check_versions(self.current_version[0],
self.version_yaml)
if checked != None and not checked:
return True
return False
def _is_version_greater(self):
"""
Check if the current version is greater as the older older one.
"""
# we compare the 2 versions.
checked = Version(True).check_versions(self.current_version[0],
self.version_yaml)
if checked is not None and not checked:
# The current version is greater as the older one.
# We return True.
return True
# We return False
return False
def test_clean_all(self):
"""
Test the clean_all process.
"""
if not Version(True).is_cloned(): # pragma: no cover
file = "whois_db.json"
File(file).write("Hello, World!")
expected = True
actual = PyFunceble.path.isfile(file)
self.assertEqual(expected, actual)
Clean(None, clean_all=True)
expected = False
actual = PyFunceble.path.isfile(file)
self.assertEqual(expected, actual)
def command_line(): # pragma: no cover # pylint: disable=too-many-branches,too-many-statements
"""
This function provide the command line arguments of PyFunceble.
"""
if __name__ == "PyFunceble":
load_config()
initiate(autoreset=True)
PARSER = argparse.ArgumentParser(
description='A tool to check domains or IP availability \
(ACTIVE, INACTIVE, INVALID). Also described as "[an] excellent \
script for checking ACTIVE and INACTIVE domain names"',
epilog="Crafted with %s by %s" % (
Fore.RED + "♥" + Fore.RESET,
Style.BRIGHT + Fore.CYAN + "Nissar Chababy (Funilrys) " +
Style.RESET_ALL + "with the help of " + Style.BRIGHT +
Fore.GREEN + "https://git.io/vND4m " + Style.RESET_ALL +
"&& " + Style.BRIGHT + Fore.GREEN + "https://git.io/vND4a",
),
add_help=False,
)
CURRENT_VALUE_FORMAT = Fore.YELLOW + Style.BRIGHT + "Installed value: " + Fore.BLUE
PARSER.add_argument(
"-ad",
"--adblock",
action="store_true",
help="Switch the decoding of the adblock format. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["adblock"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-a",
"--all",
action="store_false",
help="Output all available informations on screen. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["less"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--cmd-before-end",
type=str,
help="Pass a command before the results (final) commit of travis \
mode. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["command_before_end"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-c",
"--auto-continue",
"--continue",
action="store_true",
help="Switch the value of the auto continue mode. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["auto_continue"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--autosave-minutes",
type=int,
help="Update the minimum of minutes before we start commiting \
to upstream under Travis CI. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["travis_autosave_minutes"]) + Style.RESET_ALL),
)
PARSER.add_argument("--clean",
action="store_true",
help="Clean all files under output.")
PARSER.add_argument(
"--commit-autosave-message",
type=str,
help="Replace the default autosave commit message. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["travis_autosave_commit"]) + Style.RESET_ALL),
)
PARSER.add_argument(
"--commit-results-message",
type=str,
help="Replace the default results (final) commit message. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["travis_autosave_final_commit"]) +
Style.RESET_ALL),
)
PARSER.add_argument("-d",
"--domain",
type=str,
help="Analyze the given domain.")
PARSER.add_argument(
"-db",
"--database",
action="store_true",
help="Switch the value of the usage of a database to store \
inactive domains of the currently tested list. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["inactive_database"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-dbr",
"--days-between-db-retest",
type=int,
help=
"Set the numbers of day(s) between each retest of domains present \
into inactive-db.json. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["days_between_db_retest"]) + Style.RESET_ALL),
)
PARSER.add_argument(
"--debug",
action="store_true",
help="Switch the value of the debug mode. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["debug"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--directory-structure",
action="store_true",
help=
"Generate the directory and files that are needed and which does \
not exist in the current directory.",
)
PARSER.add_argument("-f",
"--file",
type=str,
help="Test a file with a list of domains.")
PARSER.add_argument("--filter", type=str, help="Domain to filter.")
PARSER.add_argument(
"-ex",
"--execution",
action="store_true",
help="Switch the dafault value of the execution time showing. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["show_execution_time"]) + Style.RESET_ALL),
)
PARSER.add_argument(
"--help",
action="help",
default=argparse.SUPPRESS,
help="Show this help message and exit.",
)
PARSER.add_argument(
"-h",
"--host",
action="store_true",
help="Switch the value of the generation of hosts file. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["generate_hosts"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--http",
action="store_true",
help="Switch the value of the usage of HTTP code. %s" %
(CURRENT_VALUE_FORMAT + repr(HTTP_CODE["active"]) +
Style.RESET_ALL),
)
PARSER.add_argument("--iana",
action="store_true",
help="Update `iana-domains-db.json`.")
PARSER.add_argument(
"-ip",
type=str,
help="Change the ip to print in host file. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["custom_ip"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--less",
action="store_true",
help="Output less informations on screen. %s" %
(CURRENT_VALUE_FORMAT + repr(Core.switch("less")) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-n",
"--no-files",
action="store_true",
help="Switch the value the production of output files. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["no_files"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-nl",
"--no-logs",
action="store_true",
help="Switch the value of the production of logs files in case we \
encounter some errors. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["logs"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-nu",
"--no-unified",
action="store_true",
help=
"Switch the value of the production of result.txt as unified result \
under the output directory. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["unified"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-nw",
"--no-whois",
action="store_true",
help=
"Switch the value the usage of whois to test domain's status. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["no_whois"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-p",
"--percentage",
action="store_true",
help="Switch the value of the percentage output mode. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["show_percentage"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--plain",
action="store_true",
help="Switch the value of the generation \
of the plain list of domain. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["plain_list_domain"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--production",
action="store_true",
help="Prepare the repository for production.",
)
PARSER.add_argument(
"-q",
"--quiet",
action="store_true",
help="Run the script in quiet mode. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["quiet"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--share-logs",
action="store_true",
help=
"Activate the sharing of logs to an API which helps manage logs in \
order to make PyFunceble a better script. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["share_logs"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-s",
"--simple",
action="store_true",
help="Switch the value of the simple output mode. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["simple"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--split",
action="store_true",
help=
"Switch the valur of the split of the generated output files. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["inactive_database"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-t",
"--timeout",
type=int,
default=3,
help="Switch the value of the timeout. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["seconds_before_http_timeout"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--travis",
action="store_true",
help="Activate the travis mode. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["travis"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--travis-branch",
type=str,
default="master",
help="Switch the branch name where we are going to push. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["travis_branch"]) +
Style.RESET_ALL),
)
PARSER.add_argument("-u",
"--url",
type=str,
help="Analyze the given url.")
PARSER.add_argument("-uf",
"--url-file",
type=str,
help="Test a file with a list of URL.")
PARSER.add_argument("-v",
"--version",
action="version",
version="%(prog)s " + VERSION)
ARGS = PARSER.parse_args()
if ARGS.less:
CONFIGURATION.update({"less": ARGS.less})
if ARGS.adblock:
CONFIGURATION.update({"adblock": Core.switch("adblock")})
if ARGS.auto_continue:
CONFIGURATION.update(
{"auto_continue": Core.switch("auto_continue")})
if ARGS.autosave_minutes:
CONFIGURATION.update(
{"travis_autosave_minutes": ARGS.autosave_minutes})
if ARGS.clean:
Clean(None)
if ARGS.cmd_before_end:
CONFIGURATION.update({"command_before_end": ARGS.cmd_before_end})
if ARGS.commit_autosave_message:
CONFIGURATION.update(
{"travis_autosave_commit": ARGS.commit_autosave_message})
if ARGS.commit_results_message:
CONFIGURATION.update(
{"travis_autosave_final_commit": ARGS.commit_results_message})
if ARGS.database:
CONFIGURATION.update(
{"inactive_database": Core.switch("inactive_database")})
if ARGS.days_between_db_retest:
CONFIGURATION.update(
{"days_between_db_retest": ARGS.days_between_db_retest})
if ARGS.debug:
CONFIGURATION.update({"debug": Core.switch("debug")})
if ARGS.directory_structure:
DirectoryStructure()
if ARGS.execution:
CONFIGURATION.update(
{"show_execution_time": Core.switch("show_execution_time")})
if ARGS.filter:
CONFIGURATION.update({"to_filter": ARGS.filter})
if ARGS.host:
CONFIGURATION.update(
{"generate_hosts": Core.switch("generate_hosts")})
if ARGS.http:
HTTP_CODE.update(
{"active": Core.switch(HTTP_CODE["active"], True)})
if ARGS.iana:
IANA()
if ARGS.ip:
CONFIGURATION.update({"custom_ip": ARGS.ip})
if ARGS.no_files:
CONFIGURATION.update({"no_files": Core.switch("no_files")})
if ARGS.no_logs:
CONFIGURATION.update({"logs": Core.switch("logs")})
if ARGS.no_unified:
CONFIGURATION.update({"unified": Core.switch("unified")})
if ARGS.no_whois:
CONFIGURATION.update({"no_whois": Core.switch("no_whois")})
if ARGS.percentage:
CONFIGURATION.update(
{"show_percentage": Core.switch("show_percentage")})
if ARGS.plain:
CONFIGURATION.update(
{"plain_list_domain": Core.switch("plain_list_domain")})
if ARGS.production:
Production()
if ARGS.quiet:
CONFIGURATION.update({"quiet": Core.switch("quiet")})
if ARGS.share_logs:
CONFIGURATION.update({"share_logs": Core.switch("share_logs")})
if ARGS.simple:
CONFIGURATION.update({
"simple": Core.switch("simple"),
"quiet": Core.switch("quiet")
})
if ARGS.split:
CONFIGURATION.update({"split": Core.switch("split")})
if ARGS.timeout:
if ARGS.timeout % 3 == 0:
CONFIGURATION.update(
{"seconds_before_http_timeout": ARGS.timeout})
if ARGS.travis:
CONFIGURATION.update({"travis": Core.switch("travis")})
if ARGS.travis_branch:
CONFIGURATION.update({"travis_branch": ARGS.travis_branch})
if not CONFIGURATION["quiet"]:
print(Fore.YELLOW + ASCII_PYFUNCEBLE + Fore.RESET)
Version().compare()
Core(
domain=ARGS.domain,
file_path=ARGS.file,
url_to_test=ARGS.url,
url_file=ARGS.url_file,
)
def __init__(self, extern=False):
if not extern:
# A method of this class is not called.
if not self.is_dev_version() and not self.is_master_version():
# The version is not the `dev` version nor the `master` version.
# We raise an exception telling the user that there is a
# problem somewhere around the versioning.
raise Exception("Please switch to `dev` or `master` branch.")
# We read and get the current content of `version.yaml`.
self.data_version_yaml = self._get_current_version_yaml()
# We split the version in oder to get only the list of digits from
# the local version.
self.version_yaml = Version(True).split_versions(
self.data_version_yaml["current_version"])
# We we get the full version with the non-digits and the digits.
self.current_version = Version(True).split_versions(
PyFunceble.VERSION, True)
if self._is_version_greater(
) or not Version(True).check_versions_literally(
PyFunceble.VERSION,
self.data_version_yaml["current_version"]):
# * The local version is greater than the older one.
# or
# * The local version is literally different than the
# upstream one.
# We clean the output directory.
Clean(None)
# We generate the productive directory structure file.
DirectoryStructure(production=True)
if self._does_require_deprecation():
# We have to put the previous version into the list of deprecated list.
# We reconstruct the version.
to_deprecate = ".".join(self.version_yaml)
# And we append it into the list of deprecated version.
self.data_version_yaml["deprecated"].append(to_deprecate)
if self._does_require_force_update():
# We have to put the previous version into the list of forced for update list.
# We reconstruct the version.
to_force_update = ".".join(self.version_yaml)
# And we append it into the list of minimal version.
self.data_version_yaml["force_update"][
"minimal_version"].append(to_force_update)
if self.current_version[-1]:
# The non digit part of the version is not empty.
# We append it to the digit part.
self.current_version[0].append(self.current_version[-1])
# We update the current version.
self.data_version_yaml["current_version"] = ".".join(
self.current_version[0])
# We fix the urls in the README file.
self._update_docs(PyFunceble.CURRENT_DIRECTORY + "README.rst")
# We fix the urls in the configuration file.
self._update_docs(PyFunceble.CURRENT_DIRECTORY +
".PyFunceble_production.yaml")
# We fix the urls in the setup.py file.
self._update_docs(PyFunceble.CURRENT_DIRECTORY + "setup.py")
# We fix the urls in the documentation index.
self._update_docs(PyFunceble.CURRENT_DIRECTORY +
PyFunceble.directory_separator + "docs" +
PyFunceble.directory_separator + "index.rst")
# We fix the urls in the documentation logic representation.
self._update_docs(PyFunceble.CURRENT_DIRECTORY +
PyFunceble.directory_separator + "docs" +
PyFunceble.directory_separator +
"logic-representation.rst")
# We fix the urls in the usage documentation.
self._update_docs(PyFunceble.CURRENT_DIRECTORY +
PyFunceble.directory_separator + "docs" +
PyFunceble.directory_separator + "usage" +
PyFunceble.directory_separator +
"from-a-terminal.rst")
# We fix the urls in the links configuration documentation.
self._update_docs(PyFunceble.CURRENT_DIRECTORY +
PyFunceble.directory_separator + "docs" +
PyFunceble.directory_separator +
"configuration" +
PyFunceble.directory_separator + "links.rst")
# We fix the urls in the code.
self._update_code_urls()
# We fix the setup.py.
self._update_setup_py()
# We fix the .travis.yml file.
self._update_travis_yml()
# We save our version data into our `version.yaml` file.
Dict(self.data_version_yaml).to_yaml(
PyFunceble.CURRENT_DIRECTORY + "version.yaml",
flow_style=None)
# We prepare the message we are going to print on screen.
message = (PyFunceble.Fore.GREEN + PyFunceble.Style.BRIGHT +
"We are ready to ship!! \n")
message += (
PyFunceble.Fore.CYAN
+ "Please do not touch version.yaml nor setup.py (version update)"
) # pylint: disable=line-too-long
# We print the message.
print(message)
# We exit the process.
exit(0)
else:
# The local version is less or equal to the older one.
# We print a message on screen.
print(
PyFunceble.Fore.YELLOW + PyFunceble.Style.BRIGHT +
"Are you sure that you did some changes ? Please update PyFunceble.VERSION if it is the case." # pylint: disable=line-too-long
)
# We exit the process.
exit(1)
def _command_line(): # pragma: no cover pylint: disable=too-many-branches,too-many-statements
"""
Provide the command line interface.
"""
if __name__ == "PyFunceble":
# We initiate the end of the coloration at the end of each line.
initiate(autoreset=True)
# We load the configuration and the directory structure.
load_config(True)
try:
# The following handle the command line argument.
try:
PARSER = argparse.ArgumentParser(
epilog="Crafted with %s by %s" % (
Fore.RED + "♥" + Fore.RESET,
Style.BRIGHT + Fore.CYAN +
"Nissar Chababy (Funilrys) " + Style.RESET_ALL +
"with the help of " + Style.BRIGHT + Fore.GREEN +
"https://pyfunceble.rtfd.io/en/master/contributors.html "
+ Style.RESET_ALL + "&& " + Style.BRIGHT + Fore.GREEN +
"https://pyfunceble.rtfd.io/en/master/special-thanks.html",
),
add_help=False,
)
CURRENT_VALUE_FORMAT = (Fore.YELLOW + Style.BRIGHT +
"Configured value: " + Fore.BLUE)
PARSER.add_argument(
"-ad",
"--adblock",
action="store_true",
help="Switch the decoding of the adblock format. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["adblock"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-a",
"--all",
action="store_false",
help="Output all available information on the screen. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["less"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
""
"-c",
"--auto-continue",
"--continue",
action="store_true",
help="Switch the value of the auto continue mode. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["auto_continue"]) + Style.RESET_ALL),
)
PARSER.add_argument(
"--autosave-minutes",
type=int,
help="Update the minimum of minutes before we start "
"committing to upstream under Travis CI. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["travis_autosave_minutes"]) +
Style.RESET_ALL),
)
PARSER.add_argument("--clean",
action="store_true",
help="Clean all files under output.")
PARSER.add_argument(
"--clean-all",
action="store_true",
help=
"Clean all files under output and all file generated by PyFunceble.",
)
PARSER.add_argument(
"--cmd",
type=str,
help="Pass a command to run before each commit "
"(except the final one) under the Travis mode. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["command_before_end"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--cmd-before-end",
type=str,
help="Pass a command to run before the results "
"(final) commit under the Travis mode. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["command_before_end"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--commit-autosave-message",
type=str,
help="Replace the default autosave commit message. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["travis_autosave_commit"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--commit-results-message",
type=str,
help=
"Replace the default results (final) commit message. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["travis_autosave_final_commit"]) +
Style.RESET_ALL),
)
PARSER.add_argument("-d",
"--domain",
type=str,
help="Set and test the given domain.")
PARSER.add_argument(
"-db",
"--database",
action="store_true",
help="Switch the value of the usage of a database to store "
"inactive domains of the currently tested list. %s" %
(CURRENT_VALUE_FORMAT + repr(
CONFIGURATION["inactive_database"]) + Style.RESET_ALL),
)
PARSER.add_argument(
"-dbr",
"--days-between-db-retest",
type=int,
help=
"Set the numbers of days between each retest of domains present "
"into inactive-db.json. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["days_between_db_retest"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--debug",
action="store_true",
help="Switch the value of the debug mode. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["debug"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--directory-structure",
action="store_true",
help=
"Generate the directory and files that are needed and which does "
"not exist in the current directory.",
)
PARSER.add_argument(
"-ex",
"--execution",
action="store_true",
help=
"Switch the default value of the execution time showing. %s"
% (CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["show_execution_time"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-f",
"--file",
type=str,
help="Read the given file and test all domains inside it. "
"If a URL is given we download and test the content of the given URL.", # pylint: disable=line-too-long
)
PARSER.add_argument("--filter",
type=str,
help="Domain to filter (regex).")
PARSER.add_argument(
"--help",
action="help",
default=argparse.SUPPRESS,
help="Show this help message and exit.",
)
PARSER.add_argument(
"--hierarchical",
action="store_true",
help=
"Switch the value of the hierarchical sorting of the tested file. %s"
% (CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["hierarchical_sorting"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-h",
"--host",
action="store_true",
help="Switch the value of the generation of hosts file. %s"
%
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["generate_hosts"]) + Style.RESET_ALL),
)
PARSER.add_argument(
"--http",
action="store_true",
help="Switch the value of the usage of HTTP code. %s" %
(CURRENT_VALUE_FORMAT + repr(HTTP_CODE["active"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--iana",
action="store_true",
help="Update/Generate `iana-domains-db.json`.",
)
PARSER.add_argument(
"--idna",
action="store_true",
help="Switch the value of the IDNA conversion. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["idna_conversion"]) + Style.RESET_ALL),
)
PARSER.add_argument(
"-ip",
type=str,
help=
"Change the IP to print in the hosts files with the given one. %s"
% (CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["custom_ip"]) + Style.RESET_ALL),
)
PARSER.add_argument(
"--json",
action="store_true",
help="Switch the value of the generation "
"of the JSON formatted list of domains. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["generate_json"]) + Style.RESET_ALL),
)
PARSER.add_argument(
"--less",
action="store_true",
help="Output less informations on screen. %s" %
(CURRENT_VALUE_FORMAT + repr(Core.switch("less")) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--local",
action="store_true",
help="Switch the value of the local network testing. %s" %
(CURRENT_VALUE_FORMAT + repr(Core.switch("local")) +
Style.RESET_ALL),
)
PARSER.add_argument("--link",
type=str,
help="Download and test the given file.")
PARSER.add_argument(
"-m",
"--mining",
action="store_true",
help="Switch the value of the mining subsystem usage. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["mining"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-n",
"--no-files",
action="store_true",
help=
"Switch the value of the production of output files. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["no_files"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-nl",
"--no-logs",
action="store_true",
help="Switch the value of the production of logs files "
"in the case we encounter some errors. %s" %
(CURRENT_VALUE_FORMAT + repr(not CONFIGURATION["logs"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-nu",
"--no-unified",
action="store_true",
help="Switch the value of the production unified logs "
"under the output directory. %s" %
(CURRENT_VALUE_FORMAT +
repr(not CONFIGURATION["unified"]) + Style.RESET_ALL),
)
PARSER.add_argument(
"-nw",
"--no-whois",
action="store_true",
help=
"Switch the value the usage of whois to test domain's status. %s"
% (CURRENT_VALUE_FORMAT + repr(CONFIGURATION["no_whois"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-p",
"--percentage",
action="store_true",
help="Switch the value of the percentage output mode. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["show_percentage"]) + Style.RESET_ALL),
)
PARSER.add_argument(
"--plain",
action="store_true",
help="Switch the value of the generation "
"of the plain list of domains. %s" %
(CURRENT_VALUE_FORMAT + repr(
CONFIGURATION["plain_list_domain"]) + Style.RESET_ALL),
)
PARSER.add_argument(
"--production",
action="store_true",
help="Prepare the repository for production.",
)
PARSER.add_argument(
"-psl",
"--public-suffix",
action="store_true",
help="Update/Generate `public-suffix.json`.",
)
PARSER.add_argument(
"-q",
"--quiet",
action="store_true",
help="Run the script in quiet mode. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["quiet"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--share-logs",
action="store_true",
help="Switch the value of the sharing of logs. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["share_logs"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-s",
"--simple",
action="store_true",
help="Switch the value of the simple output mode. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["simple"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--split",
action="store_true",
help=
"Switch the value of the split of the generated output files. %s"
% (CURRENT_VALUE_FORMAT + repr(
CONFIGURATION["inactive_database"]) + Style.RESET_ALL),
)
PARSER.add_argument(
"--syntax",
action="store_true",
help="Switch the value of the syntax test mode. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["syntax"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-t",
"--timeout",
type=int,
default=3,
help="Switch the value of the timeout. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["seconds_before_http_timeout"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--travis",
action="store_true",
help="Switch the value of the Travis mode. %s" %
(CURRENT_VALUE_FORMAT + repr(CONFIGURATION["travis"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"--travis-branch",
type=str,
default="master",
help="Switch the branch name where we are going to push. %s"
% (CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["travis_branch"]) + Style.RESET_ALL),
)
PARSER.add_argument("-u",
"--url",
type=str,
help="Analyze the given URL.")
PARSER.add_argument(
"-uf",
"--url-file",
type=str,
help="Read and test the list of URL of the given file. "
"If a URL is given we download and test the content of the given URL.", # pylint: disable=line-too-long
)
PARSER.add_argument(
"-ua",
"--user-agent",
type=str,
help="Set the user-agent to use and set every time we "
"interact with everything which is not our logs sharing system.", # pylint: disable=line-too-long
)
PARSER.add_argument(
"-v",
"--version",
help="Show the version of PyFunceble and exit.",
action="version",
version="%(prog)s " + VERSION,
)
PARSER.add_argument(
"-vsc",
"--verify-ssl-certificate",
action="store_true",
help="Switch the value of the verification of the "
"SSL/TLS certificate when testing for URL. %s" %
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["verify_ssl_certificate"]) +
Style.RESET_ALL),
)
PARSER.add_argument(
"-wdb",
"--whois-database",
action="store_true",
help="Switch the value of the usage of a database to store "
"whois data in order to avoid whois servers rate limit. %s"
%
(CURRENT_VALUE_FORMAT +
repr(CONFIGURATION["whois_database"]) + Style.RESET_ALL),
)
ARGS = PARSER.parse_args()
if ARGS.less:
CONFIGURATION.update({"less": ARGS.less})
elif not ARGS.all:
CONFIGURATION.update({"less": ARGS.all})
if ARGS.adblock:
CONFIGURATION.update({"adblock": Core.switch("adblock")})
if ARGS.auto_continue:
CONFIGURATION.update(
{"auto_continue": Core.switch("auto_continue")})
if ARGS.autosave_minutes:
CONFIGURATION.update(
{"travis_autosave_minutes": ARGS.autosave_minutes})
if ARGS.clean:
Clean(None)
if ARGS.clean_all:
Clean(None, ARGS.clean_all)
if ARGS.cmd:
CONFIGURATION.update({"command": ARGS.cmd})
if ARGS.cmd_before_end:
CONFIGURATION.update(
{"command_before_end": ARGS.cmd_before_end})
if ARGS.commit_autosave_message:
CONFIGURATION.update({
"travis_autosave_commit":
ARGS.commit_autosave_message
})
if ARGS.commit_results_message:
CONFIGURATION.update({
"travis_autosave_final_commit":
ARGS.commit_results_message
})
if ARGS.database:
CONFIGURATION.update({
"inactive_database":
Core.switch("inactive_database")
})
if ARGS.days_between_db_retest:
CONFIGURATION.update({
"days_between_db_retest":
ARGS.days_between_db_retest
})
if ARGS.debug:
CONFIGURATION.update({"debug": Core.switch("debug")})
if ARGS.directory_structure:
DirectoryStructure()
if ARGS.execution:
CONFIGURATION.update({
"show_execution_time":
Core.switch("show_execution_time")
})
if ARGS.filter:
CONFIGURATION.update({"filter": ARGS.filter})
if ARGS.hierarchical:
CONFIGURATION.update({
"hierarchical_sorting":
Core.switch("hierarchical_sorting")
})
if ARGS.host:
CONFIGURATION.update(
{"generate_hosts": Core.switch("generate_hosts")})
if ARGS.http:
HTTP_CODE.update(
{"active": Core.switch(HTTP_CODE["active"], True)})
if ARGS.iana:
IANA().update()
if ARGS.idna:
CONFIGURATION.update(
{"idna_conversion": Core.switch("idna_conversion")})
if ARGS.ip:
CONFIGURATION.update({"custom_ip": ARGS.ip})
if ARGS.json:
CONFIGURATION.update(
{"generate_json": Core.switch("generate_json")})
if ARGS.local:
CONFIGURATION.update({"local": Core.switch("local")})
if ARGS.mining:
CONFIGURATION.update({"mining": Core.switch("mining")})
if ARGS.no_files:
CONFIGURATION.update({"no_files": Core.switch("no_files")})
if ARGS.no_logs:
CONFIGURATION.update({"logs": Core.switch("logs")})
if ARGS.no_unified:
CONFIGURATION.update({"unified": Core.switch("unified")})
if ARGS.no_whois:
CONFIGURATION.update({"no_whois": Core.switch("no_whois")})
if ARGS.percentage:
CONFIGURATION.update(
{"show_percentage": Core.switch("show_percentage")})
if ARGS.plain:
CONFIGURATION.update({
"plain_list_domain":
Core.switch("plain_list_domain")
})
if ARGS.production:
Production()
if ARGS.public_suffix:
PublicSuffix().update()
if ARGS.quiet:
CONFIGURATION.update({"quiet": Core.switch("quiet")})
if ARGS.share_logs:
CONFIGURATION.update(
{"share_logs": Core.switch("share_logs")})
if ARGS.simple:
CONFIGURATION.update({
"simple": Core.switch("simple"),
"quiet": Core.switch("quiet")
})
if ARGS.split:
CONFIGURATION.update({"split": Core.switch("split")})
if ARGS.syntax:
CONFIGURATION.update({"syntax": Core.switch("syntax")})
if ARGS.timeout and ARGS.timeout % 3 == 0:
CONFIGURATION.update(
{"seconds_before_http_timeout": ARGS.timeout})
if ARGS.travis:
CONFIGURATION.update({"travis": Core.switch("travis")})
if ARGS.travis_branch:
CONFIGURATION.update({"travis_branch": ARGS.travis_branch})
if ARGS.user_agent:
CONFIGURATION.update({"user_agent": ARGS.user_agent})
if ARGS.verify_ssl_certificate:
CONFIGURATION.update({
"verify_ssl_certificate":
ARGS.verify_ssl_certificate
})
if ARGS.whois_database:
CONFIGURATION.update(
{"whois_database": Core.switch("whois_database")})
if not CONFIGURATION["quiet"]:
Core.colorify_logo(home=True)
# We compare the versions (upstream and local) and in between.
Version().compare()
# We call our Core which will handle all case depending of the configuration or
# the used command line arguments.
Core(
domain_or_ip_to_test=ARGS.domain,
file_path=ARGS.file,
url_to_test=ARGS.url,
url_file=ARGS.url_file,
link_to_test=ARGS.link,
)
except KeyError as e:
if not Version(True).is_cloned():
# We are not into the cloned version.
# We merge the local with the upstream configuration.
Merge(CURRENT_DIRECTORY)
else:
# We are in the cloned version.
# We raise the exception.
#
# Note: The purpose of this is to avoid having
# to search for a mistake while developing.
raise e
except KeyboardInterrupt:
stay_safe()
from PyFunceble.config import Load, Merge, Version
from PyFunceble.core import Core
from PyFunceble.directory_structure import DirectoryStructure
from PyFunceble.iana import IANA
from PyFunceble.production import Production
from PyFunceble.publicsuffix import PublicSuffix
# We set our project name.
NAME = "PyFunceble"
# We set out project version.
VERSION = "1.7.0. (Blue Bontebok: Moth)"
if "PYFUNCEBLE_OUTPUT_DIR" in environ: # pragma: no cover
# We handle the case that the `PYFUNCEBLE_OUTPUT_DIR` environnement variable is set.
CURRENT_DIRECTORY = environ["PYFUNCEBLE_OUTPUT_DIR"]
elif Version(True).is_cloned(): # pragma: no cover
# We handle the case that we are in a cloned.
CURRENT_DIRECTORY = getcwd() + directory_separator
elif "TRAVIS_BUILD_DIR" in environ: # pragma: no cover
# We handle the case that we are under Travis CI.
CURRENT_DIRECTORY = getcwd() + directory_separator
else: # pragma: no cover
# We handle all other case and distributions specific cases.
if system().lower() == "linux" or system().lower() == "darwin":
# We are under a Linux distribution.
# We set the default configuration location path.
config_dir_path = (
path.expanduser("~" + directory_separator + ".config") +
directory_separator)