def standard(element: Any) -> List[Union[int, Any]]:
"""
Provides the key to use for the standard sorting.
:param element:
The element to format.
"""
element = element.strip()
if not element:
return []
regex_helper = RegexHelper()
element = Url2Netloc(element).get_converted().strip()
if PyFunceble.facility.ConfigLoader.is_already_loaded():
element = regex_helper.set_regex(
r"^%s\s+" % PyFunceble.storage.CONFIGURATION.cli_testing.hosts_ip
).replace_match(element, "")
cleaned = regex_helper.set_regex(r"[^a-zA-Z0-9\.]").replace_match(
element, "")
return [
int(x) if x.isdigit() else x
for x in regex_helper.set_regex(r"(\d+)").split(cleaned)
]
def get_remote_destination():
"""
Provides the remote destination to use.
:raise PyFunceble.cli.continuous_integration.exceptions.RemoteURLNotFound:
When we could not determine the remote destination.
"""
regex = r"(?:[a-z]+(?:\s+|\t+))(.*)(?:(?:\s+|\t+)\([a-z]+\))"
remote_of_interest = [
x for x in CommandHelper("git remote -v").execute().splitlines()
if "(fetch)" in x
][0]
filtered = RegexHelper(regex).match(remote_of_interest,
return_match=True,
group=1)
if filtered:
if "@" in filtered:
return filtered[filtered.find("@") + 1:]
if "//" in filtered:
return filtered[filtered.find("//") + 2:]
raise PyFunceble.cli.continuous_integration.exceptions.RemoteURLNotFound(
)
def is_valid(self) -> bool:
"""
Validate the given subject if exists.
"""
extension = self.get_extension()
if (not extension or extension not in self.iana_dataset
and extension not in self.SPECIAL_USE_DOMAIN_NAMES_EXTENSIONS):
return False
subject_without_extension = self.idna_subject[:self.last_point_index]
subject_without_suffix, suffix = self.get_subject_without_suffix(
self.idna_subject, extension)
if subject_without_suffix:
if suffix.count(".") >= 2:
return RegexHelper(self.REGEX_VALID_SUBDOMAIN).match(
subject_without_extension, return_match=False)
if "." in subject_without_suffix:
return RegexHelper(self.REGEX_VALID_SUBDOMAIN).match(
subject_without_suffix, return_match=False)
return False
if "." in subject_without_extension:
return RegexHelper(self.REGEX_VALID_SUBDOMAIN).match(
subject_without_extension, return_match=False)
return False
def get_extension_and_referrer_from_block(
self, block: str
) -> Tuple[Optional[str], Optional[str]]:
"""
Given an HTML block, we try to extract an extension and it's underlying
referrer (WHOIS server).
The referrer is extracted from the official IANA page, and guessed if
missing.
:param block:
The block to parse.
"""
regex_valid_extension = r"(/domains/root/db/)(.*)(\.html)"
regex_helper = RegexHelper(regex_valid_extension)
if regex_helper.match(block, return_match=False):
extension = regex_helper.match(block, return_match=True, group=2)
if extension:
return extension, self.get_referrer_from_extension(extension)
return None, None
def is_valid(self) -> bool:
"""
Validate the given subject.
.. warning::
A valid domain may also be a valid subdomain.
If you precisely want to check a subdomain please refer to the
right checker (not this one :-) )!
"""
# pylint: disable=too-many-return-statements
extension = self.get_extension()
if not extension or (extension not in self.iana_dataset and extension
not in self.SPECIAL_USE_DOMAIN_NAMES_EXTENSIONS):
return False
subject_without_extension = self.idna_subject[:self.last_point_index]
subject_without_suffix, _ = self.get_subject_without_suffix(
self.idna_subject, extension)
if subject_without_suffix:
if "." in subject_without_suffix:
return False
return RegexHelper(self.REGEX_VALID_DOMAIN).match(
self.idna_subject, return_match=False)
if "." in subject_without_extension:
return False
return RegexHelper(self.REGEX_VALID_DOMAIN).match(self.idna_subject,
return_match=False)
def __init__(self,
data_to_convert: Optional[Any] = None,
aggressive: bool = False) -> None:
if aggressive is not None:
self.aggressive = aggressive
self.__regex_helper = RegexHelper()
super().__init__(data_to_convert=data_to_convert)
def test_set_regex_escape(self) -> None:
"""
Tests the method which let us set the regex to work with for the case
that it's not a string.
"""
regex_helper = RegexHelper()
regex_helper.escape_regex = True
regex_helper.set_regex("[a-z]")
expected = r"\[a\-z\]"
actual = regex_helper.regex
self.assertEqual(expected, actual)
def launch_marker_in_last_commit() -> bool:
"""
Check if the launch marker is into the last commit message.
"""
return RegexHelper(infrastructure.MARKERS["launch"]).match(
CommandHelper("git log -1").execute(), return_match=False)
def __get_actual_expiration_date(
self, extracted: str) -> Optional[Tuple[str, str, str]]:
"""
Tries to extract the actual expiration date.
"""
for index, date_regex in self.MARKER2DATE_REGEX.items():
matched = RegexHelper(date_regex).match(extracted,
return_match=True,
rematch=True)
date_parts = tuple()
if not matched:
continue
for parse_case in self.REGEX_PARSE_MAP:
if int(index) not in parse_case["regex_keys"]:
continue
date_parts = (
Digit2Digits(matched[parse_case["positions"]["day"]],
).get_converted(),
Month2Unified(matched[parse_case["positions"]
["month"]]).get_converted(),
str(matched[parse_case["positions"]["year"]]),
)
if date_parts:
return "-".join(date_parts)
return None
def __web_regex_handler(
self,
url: str,
regex_list: List[str],
method: Callable[..., "ExtraRulesHandler"],
) -> "ExtraRulesHandler":
"""
Handles a web request along with a regex filter.
"""
try:
req = PyFunceble.factory.Requester.get(url, allow_redirects=True)
for regex in regex_list:
if regex in req.text or RegexHelper(regex).match(
req.text, return_match=False):
method()
break
except (
PyFunceble.factory.Requester.exceptions.InvalidURL,
PyFunceble.factory.Requester.exceptions.Timeout,
PyFunceble.factory.Requester.exceptions.ConnectionError,
PyFunceble.factory.Requester.urllib3_exceptions.InvalidHeader,
socket.timeout,
):
pass
return self
def __regex_registry_handler(self,
regex_registry: dict) -> "ExtraRulesHandler":
"""
Handles the standard regex lookup case.
"""
for (
regex,
data,
) in regex_registry.items():
broken = False
for element in data:
if RegexHelper(regex).match(self.status.subject,
return_match=False):
if isinstance(element, tuple):
element[0](*element[1:])
else:
element()
broken = True
break
if broken:
break
return self
def setUp(self) -> None:
"""
Setups everything needed for the test.
"""
self.helper = RegexHelper()
self.test_regex = "[a-z]"
self.testing_list_subject = [
"hello",
"world",
"funilrys",
"funceble",
"PyFunceble",
"pyfunceble",
]
self.testing_subject = "Hello, this is Fun Ilrys. I just wanted to know how things goes around the tests." # pylint: disable=line-too-long
def is_launch_flag_given_by_commit_message() -> bool:
"""
Checks if the launch flag per commit message is given.
"""
return RegexHelper(dead_hosts.launcher.defaults.markers.LAUNCH_TEST).match(
Command("git log -1").execute(), return_match=False
)
def start(self) -> "ReadmeUpdater":
logging.info(
"Started to update the `About PyFunceble` section of %r",
self.destination_instance.path,
)
with importlib.resources.path("dead_hosts.launcher.data.docs",
"about_pyfunceble.md") as file_path:
updated_version = RegexHelper(
dead_hosts.launcher.defaults.markers.ABOUT_FUNCEBLE_REGEX
).replace_match(
self.destination_instance.read(),
FileHelper(str(file_path)).read() + "\n\n",
)
logging.info(
"Finished to update the `About PyFunceble` section of %r",
self.destination_instance.path,
)
logging.info(
"Started to update the `About Dead-Hosts` section of %r",
self.destination_instance.path,
)
with importlib.resources.path("dead_hosts.launcher.data.docs",
"about_dead_hosts.md") as file_path:
updated_version = RegexHelper(
dead_hosts.launcher.defaults.markers.ABOUT_DEAD_HOSTS_REGEX
).replace_match(
self.destination_instance.read(),
FileHelper(str(file_path)).read() + "\n\n",
)
logging.info(
"Finished to update the `About Dead-Hosts` section of %s",
self.destination_instance.path,
)
self.destination_instance.write(updated_version, overwrite=True)
return self
def get_destination_from_origin(origin: str) -> str:
"""
Given the origin, we provides the destination.
"""
if "/" in origin:
origin = origin.rsplit("/", 1)[-1]
if os.sep in origin:
origin = origin.rsplit(os.sep, 1)[-1]
return RegexHelper("[^a-zA-Z0-9._-]").replace_match(origin, "_")
def try_to_query_status_from_http_status_code(
self) -> "AvailabilityCheckerBase":
"""
Tries to query the status from the HTTP status code.
"""
PyFunceble.facility.Logger.info(
"Started to try to query the status of %r from: HTTP Status code Lookup",
self.status.idna_subject,
)
if not self.status.url_syntax and not RegexHelper("[^a-z0-9._]").match(
self.idna_subject, return_match=False):
# The regex is there because while testing for domain, sometime we
# may see something like mailto:[email protected]
self.http_status_code_query_tool.set_subject(
f"http://{self.idna_subject}:80")
lookup_result = self.http_status_code_query_tool.get_status_code()
if (lookup_result and lookup_result !=
self.http_status_code_query_tool.STD_UNKNOWN_STATUS_CODE):
self.status.http_status_code = lookup_result
if (PyFunceble.facility.ConfigLoader.is_already_loaded()
): # pragma: no cover ## Special behavior.
dataset = PyFunceble.storage.HTTP_CODES
else:
dataset = PyFunceble.storage.STD_HTTP_CODES
if (not self.status.status
or self.status.status == PyFunceble.storage.STATUS.down
) and (self.status.http_status_code in dataset.list.up
or self.status.http_status_code
in dataset.list.potentially_up):
self.status.status = PyFunceble.storage.STATUS.up
self.status.status_source = "HTTP CODE"
PyFunceble.facility.Logger.info(
"Could define the status of %r from: HTTP Status code Lookup",
self.status.idna_subject,
)
else:
self.status.http_status_code = None
PyFunceble.facility.Logger.info(
"Finished to try to query the status of %r from: HTTP Status code Lookup",
self.status.idna_subject,
)
return self
def test_set_regex_through_init(self) -> None:
"""
Tests the overwritting of the regex to work through the class
constructor.
"""
given = self.test_regex
expected = given
helper = RegexHelper(given)
actual = helper.regex
self.assertEqual(expected, actual)
def should_be_ignored(subject: str) -> bool:
"""
Checks if the given subject should be ignored.
"""
# pylint: disable=line-too-long
regex_ignore = r"localhost$|localdomain$|local$|broadcasthost$|0\.0\.0\.0$|allhosts$|allnodes$|allrouters$|localnet$|loopback$|mcastprefix$|ip6-mcastprefix$|ip6-localhost$|ip6-loopback$|ip6-allnodes$|ip6-allrouters$|ip6-localnet$"
if RegexHelper(regex_ignore).match(subject, return_match=False):
PyFunceble.facility.Logger.info(
"Ignoring %r because it is in our default regex.", subject)
return True
if (not PyFunceble.storage.CONFIGURATION.cli_testing.local_network
and IPSyntaxChecker(subject).is_reserved()):
PyFunceble.facility.Logger.info(
"Ignoring %r because it is a reserved IP and we are not testing "
"for/in a local network.",
subject,
)
return True
if bool(
PyFunceble.storage.CONFIGURATION.cli_testing.file_filter
) and not RegexHelper(PyFunceble.storage.CONFIGURATION.cli_testing.
file_filter).match(subject, return_match=False):
PyFunceble.facility.Logger.info(
"Ignoring %r because it does not match the filter to look for.",
subject,
)
return True
PyFunceble.facility.Logger.info(
"Allowed to test %r.",
subject,
)
return False
def __get_line(self) -> Optional[str]:
"""
Tries to get the expiration date line from the given record.
"""
for regex in self.PATTERNS:
expiration_date_line = RegexHelper(regex).match(
self.data_to_convert, return_match=True, rematch=True, group=0)
if not expiration_date_line:
continue
return expiration_date_line
return None
def get_converted(self) -> Optional[str]:
"""
Provides the expiration date of the record (if found).
"""
expiration_date_line = self.__get_line()
if expiration_date_line:
expiration_date = expiration_date_line[0].strip()
if RegexHelper(self.REGEX_DIGITS).match(expiration_date,
return_match=False):
return self.__get_actual_expiration_date(expiration_date)
return None
def hierarchical(element: Any) -> List[Union[int, Any]]:
"""
Provides the key to use for the hierarchical sorting.
:param element:
The element to format.
"""
element = element.strip()
if not element:
return []
element = Url2Netloc(element).get_converted().strip()
if PyFunceble.facility.ConfigLoader.is_already_loaded():
element = RegexHelper(
r"^%s\s+" % PyFunceble.storage.CONFIGURATION.cli_testing.hosts_ip
).replace_match(element, "")
return standard(".".join(reversed(element.split("."))))
def get_referrer_from_extension(self, extension: str) -> Optional[str]:
"""
Given an extension, tries to get or guess its extension.
"""
whois_query_tool = WhoisQueryTool()
dummy_domain = f"hello.{extension}"
iana_record = (
whois_query_tool.set_server(self.IANA_WHOIS_SERVER)
.set_subject(dummy_domain)
.get_record()
)
if iana_record and "refer" in iana_record:
regex_referrer = r"(?s)refer\:\s+([a-zA-Z0-9._-]+)\n"
matched = RegexHelper(regex_referrer).match(
iana_record, return_match=True, group=1
)
if matched:
return matched
possible_server = f"whois.nic.{extension}"
response = whois_query_tool.set_server(possible_server).get_record()
if response:
return possible_server
if extension in self.MANUAL_SERVER:
possible_server = self.MANUAL_SERVER[extension]
response = whois_query_tool.set_server(possible_server).get_record()
if response:
return possible_server
return None
def is_reserved(self) -> bool:
"""
Checks if the given subject is a reserved IPv4.
"""
if self.is_valid():
try:
address = ipaddress.IPv4Address(self.idna_subject)
return (
address.is_multicast
or address.is_private
or address.is_unspecified
or address.is_reserved
or address.is_loopback
or address.is_link_local
or not address.is_global
or RegexHelper(self._get_regex_reserved_ip()).match(
self.idna_subject, return_match=False
)
)
except ValueError:
pass
return False
class AdblockInputLine2Subject(ConverterBase):
"""
Provides an interface for the conversion or extraction of valuable subjects
from an inputted AdBlock line.
"""
_aggressive: bool = False
__regex_helper: Optional[RegexHelper] = None
def __init__(self,
data_to_convert: Optional[Any] = None,
aggressive: bool = False) -> None:
if aggressive is not None:
self.aggressive = aggressive
self.__regex_helper = RegexHelper()
super().__init__(data_to_convert=data_to_convert)
@ConverterBase.data_to_convert.setter
def data_to_convert(self, value: Any) -> None:
"""
Overrites the default behavior.
:raise TypeError:
When the given data to convert is not :py:class:`str`
"""
if not isinstance(value, str):
raise TypeError(f"<value> should be {str}, {type(value)} given.")
# pylint: disable=no-member
super(AdblockInputLine2Subject,
self.__class__).data_to_convert.fset(self, value)
@property
def aggressive(self) -> bool:
"""
Provides the state of the :code:`_aggressive` attribute.
"""
return self._aggressive
@aggressive.setter
def aggressive(self, value: bool) -> None:
"""
Provides a way to activate/deactivate the aggressive decoding.
:raise TypeError:
When the given data to convert is not :py:class:`str`
"""
if not isinstance(value, bool):
raise TypeError(f"<value> should be {bool}, {type(value)} given.")
self._aggressive = value
def set_aggressive(self, value: bool) -> "AdblockInputLine2Subject":
"""
Provides a way to activate/deactivate the aggressive decoding.
"""
self.aggressive = value
return self
@staticmethod
def should_be_ignored(line: str) -> bool:
"""
Checks if we should ignore the given line.
"""
starting_chars = ["!", "@@", "/", "[", ".", "-", "_", "?", "&"]
return any(line.startswith(x) for x in starting_chars)
@staticmethod
def extract_base(subject: Union[str, List[str]]) -> Union[str, List[str]]:
"""
Extracts the base of the given subject (supposely URL).
:param subject:
The subject to work with.
Example:
Giving :code:`"hello.world/?is=beautiful"` returns :code:`"hello.world"`
"""
subject = subject.replace("*", "").replace("~", "")
try:
return Url2Netloc(subject).get_converted()
except ValueError:
return subject
def _decode_multiple_subject(self, decoded: str) -> Set[str]:
"""
Implementation of the decoding of the case that multiple
subjects are possible in the given :py:class:`str`.
:param decoded:
The decoded part to split.
"""
result = set()
rematch = self.__regex_helper.set_regex(r"((?:[^~\*,]+))").match(
decoded, rematch=True, return_match=True)
if rematch:
result.update({self.extract_base(x) for x in rematch})
return result
def _decode_options(self, decoded_options: List[str]) -> Set[str]:
"""
Handle the decoding of the options.
What it does:
- It extracts all :code:`domain=` component - when found.
- It extracts all :code:`href` URL base - when found.
:param decoded_options:
The splitted list of options.
"""
result = set()
for rule in decoded_options:
if "domain=" in rule:
rule = rule.replace("domain=", "").replace("|", ",")
result.update(self._decode_multiple_subject(rule))
continue
if "href" in rule:
matched = self.__regex_helper.set_regex(
r"((?:\"|\')(.*)(?:\"|\'))").match(rule,
return_match=True,
rematch=True,
group=1)
if matched:
result.add(self.extract_base(matched))
continue
return result
def _decode_v1(self, line: str) -> Set[str]:
"""
Implementation of our first decoding mode.
In this mode we try to decode the simple:
||ads.example.com^
rule.
:param line:
The line to decode.
"""
result = set()
local_line = line.strip()
if local_line.startswith("||") and (local_line.endswith("^")
or local_line.endswith("$")):
local_line = local_line.replace("||", "", 1)
if local_line.endswith("^"):
local_line = "".join(local_line.rsplit("^", 1))
elif local_line.endswith("$"):
local_line = "".join(local_line.rsplit("$", 1))
result.update(self._decode_multiple_subject(local_line))
return {x for x in result if "." in x}
def _decode_v2(self, line: str) -> Set[str]:
"""
Implementation of our second decoding mode.
In this mode, we try to decode the simple:
|https://ads.example.com|
rule.
:param line:
The line to decode.
"""
result = set()
local_line = line.strip()
if local_line.startswith("|") and local_line.endswith("|"):
local_line = local_line.replace("|", "", 1)
local_line = "".join(local_line.rsplit("|", 1))
result.add(self.extract_base(local_line))
return {x for x in result if "." in x}
def _decode_v3(self, line: str) -> Set[str]:
"""
Implementation of our third decoding mode.
In this mode, we try to decode the simple:
||ads.example.com^$script,image,domain=example.com|~foo.example.info
||ads.example.com$script,image,domain=example.com|~foo.example.info
rule.
:param line:
The line to decode.
"""
result = set()
local_line = line.strip()
if not local_line.startswith("||"):
return result
if "$" in local_line:
v1_mode, options = local_line.split("$", 1)
if not v1_mode.endswith("^"):
v1_mode += "^"
result.update(self._decode_v1(v1_mode))
if self.aggressive:
result.update(self._decode_options(options.split(",")))
elif "^" not in local_line:
result.update(self._decode_v1(f"{local_line}^"))
else:
result.update(
self._decode_v1(local_line[:local_line.find("^") + 1]))
return {x for x in result if "." in x}
def _decode_v4(self, line: str) -> Set[str]:
"""
Implementation of our fourth decoding mode.
In this mode, we try to decode the simple:
@@||ads.example.com/notbanner^$~script
rule.
:param line:
The line to decode.
"""
result = set()
local_line = line.strip()
if (not self.aggressive or not local_line.startswith("@@||")
or "^$" not in local_line):
return result
v1_mode, options = local_line.split("$", 1)
result.update({
self.extract_base(x)
for x in self._decode_v1(v1_mode.replace("@@", ""))
})
result.update(self._decode_options(options.split(",")))
return {x for x in result if "." in x}
def _decode_v5(self, line: str) -> Set[str]:
"""
Implementation of our fifth decoding mode.
In this mode, we try to decode the simple:
example.com,example.net##.advert
exception.example.com#@#.advert
example.com,example.net#?#div:-abp-has(> div > img.advert)
exception.example.com#@#div:-abp-has(> div > img.advert)
rule.
:param line:
The line to decode.
"""
local_line = line.strip()
result = set()
if not self.aggressive:
return result
separators = ["##", "#@#", "#?#"]
obj_of_interest, options = "", ""
for separator in separators:
if separator in local_line:
obj_of_interest, options = local_line.split(separator, 1)
break
result.update(self._decode_multiple_subject(obj_of_interest))
result.update(self._decode_options(options.split(",")))
return {x for x in result if "." in x}
def _decode_v6(self, line: str) -> Set[str]:
"""
Implementation of our sixth decoding mode.
In this mode we try to decode the simple:
$domain=exam.pl|elpmaxe.pl|example.pl
^hello^$domain=example.com
rule.
:param line:
The line to decode.
"""
local_line = line.strip()
result = set()
if not self.aggressive:
return result
separators = ["$"]
for separator in separators:
if separator not in line:
continue
options = local_line[local_line.find(separator) + 1:]
result.update(self._decode_options(options.split(",")))
return {x for x in result if "." in x}
def get_converted(self) -> List[str]:
"""
Provides the converted data.
"""
result = set()
if not self.should_be_ignored(self.data_to_convert.strip()):
result.update(self._decode_v1(self.data_to_convert))
result.update(self._decode_v2(self.data_to_convert))
result.update(self._decode_v3(self.data_to_convert))
result.update(self._decode_v5(self.data_to_convert))
result.update(self._decode_v6(self.data_to_convert))
result.update(self._decode_v4(self.data_to_convert))
return ListHelper(list(result)).sort().subject
class ProductionPrep:
"""
Provides an interface for the production file modification.
The idea is that we always have 2 branches: the `dev` and the `master`
branch.
We want to fix all the URL to point to the right one, so this interface
just provides everything needed for that job.
Another important part is the cleanup of the production environment.
What is meant is the cleanup of the `output/` directory and the
construction of the dir_structure file.
.. warning::
This class assumes that you know what you are doing. Meaning that
you should run this only if your are developing PyFunceble.
"""
VERSION_FILE_PATH: str = os.path.join(
PyFunceble.storage.CONFIG_DIRECTORY,
PyFunceble.cli.storage.DISTRIBUTED_VERSION_FILENAME,
)
AVAILABLE_BRANCHES: List[str] = ["dev", "master"]
regex_helper: RegexHelper = RegexHelper()
file_helper: FileHelper = FileHelper()
dict_helper: DictHelper = DictHelper()
version_utility: VersionUtility = VersionUtility(PyFunceble.storage.PROJECT_VERSION)
version_file_content: Optional[dict] = None
"""
A copy of the local version file.
"""
_branch: Optional[str] = None
previous_version: Optional[str] = None
"""
Provides the previous version (from :code:`version_file_content`)
"""
def __init__(self, branch: Optional[str] = None) -> None:
self.version_file_content = self.dict_helper.from_yaml_file(
self.VERSION_FILE_PATH
)
self.previous_version = copy.deepcopy(
self.version_file_content["current_version"]
)
if branch is not None:
self.branch = branch
def ensure_branch_is_given(func): # pylint: disable=no-self-argument
"""
Ensures that the branch is given before running the decorated method.
:raise TypeError:
When the :code:`self.branch` is not set.
"""
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
if not isinstance(self.branch, str):
raise TypeError(
f"<self.branch> should be {str}, " f"{type(self.branch)} given."
)
return func(self, *args, **kwargs) # pylint: disable=not-callable
return wrapper
@property
def branch(self) -> Optional[str]:
"""
Provides the current state of the :code:`_branch` attribute.
"""
return self._branch
@branch.setter
def branch(self, value: str) -> None:
"""
Sets the branch to act with.
:param value:
The value to set.
:raise TypeError:
When the given :code:`value` is not a :py:class:`str`.
:raise ValueError:
When the given :code:`value` is empty.
"""
if not isinstance(value, str):
raise TypeError(f"<value> should be {str}, {type(value)} given.")
if not value:
raise ValueError("<value> should not be empty.")
self._branch = value
def set_branch(self, value: str) -> "ProductionPrep":
"""
Sets the branch to act with.
:param value:
The value to set.
"""
self.branch = value
return self
def should_be_deprecated(self, previous_version: str) -> bool:
"""
Checks if we should deprecates the current version.
"""
splitted = self.version_utility.get_splitted(previous_version)[0]
local_splitted = self.version_utility.get_splitted(
self.version_utility.local_version
)[0]
for index, version_part in enumerate(splitted[:2]):
if int(version_part) < int(local_splitted[index]):
return True
return False
@ensure_branch_is_given
def update_urls(self, file: str) -> "ProductionPrep":
"""
Updates the common URLS which are in the given file.
:param file:
The file to work with.
:raise FileNotFoundError:
When the given :code:`file` is not found.
"""
if self.branch == "dev":
regexes = [
(r"PyFunceble\/%s\/" % "master", "PyFunceble/%s/" % "dev"),
("=%s" % "master", "=%s" % "dev"),
(r"/{1,}en\/%s" % "latest", "/en/%s" % "dev"),
(r"\/pyfunceble-dev.png", "/pyfunceble-%s.png" % "dev"),
(r"\/project\/pyfunceble$", "/project/pyfunceble-%s" % "dev"),
(
r"\/badge\/pyfunceble(/month|/week|)$",
"/badge/pyfunceble-%s\\1" % "dev",
),
(r"\/blob\/%s\/" % "master", "/blob/%s/" % "dev"),
(r"\/pypi\/v\/pyfunceble\.png$", "/pypi/v/pyfunceble-%s.png" % "dev"),
(r"\/(logo|graphmls|gifs\/raw)\/%s\/" % "master", "/\\1/%s/" % "dev"),
(r"\/(PyFunceble\/tree)\/%s" % "master", "/\\1/%s" % "dev"),
]
elif self.branch == "master":
regexes = [
(r"PyFunceble\/%s\/" % "dev", "PyFunceble/%s/" % "master"),
("=%s" % "dev", "=%s" % "master"),
(r"/{1,}en\/%s" % "dev", "/en/%s" % "latest"),
(r"\/pyfunceble-dev.png", "/pyfunceble-dev.png"),
(r"\/project\/pyfunceble-%s$" % "dev", "/project/pyfunceble"),
(
r"\/badge\/pyfunceble-%s(/month|/week|)$" % "dev",
"/badge/pyfunceble\\1",
),
(r"\/blob\/%s\/" % "dev", "/blob/%s/" % "master"),
(
r"\/pypi\/v\/pyfunceble-%s\.png$" % "dev",
"/pypi/v/pyfunceble.png",
),
(r"\/(logo|graphmls|gifs\/raw)\/%s\/" % "dev", "/\\1/%s/" % "master"),
(r"\/(PyFunceble\/tree)\/%s" % "dev", "/\\1/%s" % "master"),
]
else:
regexes = {}
self.file_helper.set_path(file)
PyFunceble.facility.Logger.info(
"Started to update our URL into %r", self.file_helper.path
)
if not self.file_helper.exists():
raise FileNotFoundError(self.file_helper.path)
to_update = self.file_helper.read()
for regex, replacement in regexes:
to_update = self.regex_helper.set_regex(regex).replace_match(
to_update, replacement, multiline=True
)
self.file_helper.write(to_update, overwrite=True)
PyFunceble.facility.Logger.info(
"Finished to update our URL into %r", self.file_helper.path
)
return self
def update_docs_urls(self) -> "ProductionPrep":
"""
Updates all URL in the documentation files.
"""
to_ignore = ["they-use-d-it.rst"]
self.update_urls(
os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "README.rst")
)
for root, _, files in os.walk(
os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "docs")
):
for file in files:
if not file.endswith(".rst"):
continue
full_path = os.path.join(root, file)
if any(x in full_path for x in to_ignore):
continue
self.update_urls(os.path.join(root, file))
@staticmethod
def update_code_format() -> "ProductionPrep":
"""
Updates the format of the source code using black.
"""
# pylint: disable=import-outside-toplevel, import-error
import black
import isort
def format_file(file: str, isortconfig: isort.settings.Config) -> None:
"""
Formats the given file using black.
:param file:
The file to format.
:parm isortconfig:
The configuration to apply while sorting the imports.
"""
isort.api.sort_file(pathlib.Path(file), config=isortconfig)
black.format_file_in_place(
pathlib.Path(file),
fast=False,
mode=black.Mode(),
write_back=black.WriteBack.YES,
)
PyFunceble.facility.Logger.info("Update format of %r", file)
isort_config = isort.settings.Config(settings_file="setup.cfg")
files = [
os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "setup.py"),
]
for file in files:
format_file(file, isort_config)
for root, _, files in os.walk(
os.path.join(
PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.storage.PROJECT_NAME
)
):
if "__pycache__" in root:
continue
for file in files:
if not file.endswith(".py"):
continue
format_file(os.path.join(root, file), isort_config)
for root, _, files in os.walk(
os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "tests")
):
if "__pycache__" in root:
continue
for file in files:
if not file.endswith(".py"):
continue
format_file(os.path.join(root, file), isort_config)
@staticmethod
def update_documentation() -> "ProductionPrep":
"""
Updates the code documentation.
:raise RuntimeError:
When one of the wanted directory is not found.
"""
PyFunceble.facility.Logger.info(
"Started to update and generate the documentation.",
)
docs_dir_helper = DirectoryHelper("docs")
source_code_dir_helper = DirectoryHelper("PyFunceble")
if not docs_dir_helper.exists():
raise RuntimeError(f"{docs_dir_helper.realpath!r} not found.")
if not source_code_dir_helper.exists():
raise RuntimeError(f"{source_code_dir_helper.realpath!r} not found.")
header = "Code Documentation"
source_code_destination = os.path.join(docs_dir_helper.realpath, "code")
CommandHelper(
f"sphinx-apidoc -d 5 -f -H {header!r} -o "
f"{source_code_destination!r} {source_code_dir_helper.realpath}"
).execute(raise_on_error=True)
docs_destination = os.path.join(docs_dir_helper.realpath, "_build", "html")
CommandHelper(
f"sphinx-build -a -Q {docs_dir_helper.realpath!r} {docs_destination!r}"
).execute(raise_on_error=False)
PyFunceble.facility.Logger.info(
"Finished to update and generate the documentation.",
)
def update_code_urls(self) -> "ProductionPrep":
"""
Updates all URL in the source code.
"""
to_ignore = [
".gitignore",
".keep",
]
self.update_urls(os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "setup.py"))
for root, _, files in os.walk(
os.path.join(
PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.storage.PROJECT_NAME
)
):
if "__pycache__" in root:
continue
for file in files:
if file in to_ignore:
continue
self.update_urls(os.path.join(root, file))
for root, _, files in os.walk(
os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "tests")
):
if "__pycache__" in root:
continue
for file in files:
if file in to_ignore:
continue
self.update_urls(os.path.join(root, file))
@ensure_branch_is_given
def update_setup_py(self) -> "ProductionPrep":
"""
Updates content of :code:`setup.py`.
:raise FileNotFoundError:
When the :code:`setup.py` file does not exists.
"""
PyFunceble.facility.Logger.info(
"Started to update setup.py.",
)
if self.branch == "dev":
regexes = [
(r'name=".*"', 'name="PyFunceble-dev"'),
(r'"Development\sStatus\s::.*"', '"Development Status :: 4 - Beta"'),
]
elif self.branch == "master":
regexes = [
(r'name=".*"', 'name="PyFunceble-dev"'),
(
r'"Development\sStatus\s::.*"',
'"Development Status :: 5 - Production/Stable"',
),
]
else:
regexes = [
(r'name=".*"', 'name="PyFunceble-dev"'),
(
r'"Development\sStatus\s::.*"',
'"Development Status :: 3 - Alpha"',
),
]
self.file_helper.set_path(
os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "setup.py")
)
if not self.file_helper.exists():
raise FileNotFoundError(self.file_helper.path)
to_update = self.file_helper.read()
for regex, replacement in regexes:
to_update = self.regex_helper.set_regex(regex).replace_match(
to_update, replacement, multiline=True
)
self.file_helper.write(to_update, overwrite=True)
PyFunceble.facility.Logger.info(
"Started to update setup.py.",
)
return self
def update_version_file(self) -> "ProductionPrep":
"""
Updates the version file.
"""
PyFunceble.facility.Logger.info(
"Started to update version file.",
)
if self.should_be_deprecated(self.previous_version):
to_append = ".".join(
self.version_utility.get_splitted(self.version_utility.local_version)[0]
)
if to_append not in self.version_file_content["deprecated"]:
self.version_file_content["deprecated"].append(to_append)
self.version_file_content[
"current_version"
] = PyFunceble.storage.PROJECT_VERSION
self.dict_helper.set_subject(self.version_file_content).to_yaml_file(
self.VERSION_FILE_PATH
)
PyFunceble.facility.Logger.info(
"Finished to update version file.",
)
return self
def update_dir_structure_file(self) -> "ProductionPrep":
"""
Updates the directory structure.
"""
DirectoryStructureBackup().start()
return self
def start(self) -> "ProductionPrep":
"""
Starts the production process.
"""
return (
self.update_setup_py()
.update_code_urls()
.update_code_format()
.update_version_file()
)
def tool() -> None:
"""
Provides the CLI of PyFunceble.
"""
# pylint: disable=too-many-locals
# We start with loading the configuration. That way, we don't have to
# think of this anymore as soon as the CLI is called.
# As the merging is now done on demand and not on first hand, this will
# give us a bit of agility.
PyFunceble.facility.ConfigLoader.start()
colorama.init(autoreset=True)
description = (
f"{colorama.Style.BRIGHT}{colorama.Fore.GREEN}PyFunceble"
f"{colorama.Style.RESET_ALL} - "
"The tool to check the availability or syntax of domain, IP or URL.")
our_epilog = (
f"{colorama.Style.BRIGHT}{colorama.Fore.YELLOW}For an in-depth usage, "
"explanation and examples of the arguments,\n"
f"you should read the documentation at{colorama.Fore.GREEN} "
"https://pyfunceble.readthedocs.io/en/latest/"
f"{colorama.Style.RESET_ALL}\n\n")
parser = OurArgumentParser(
description=description,
epilog=our_epilog + PyFunceble.cli.storage.STD_EPILOG,
add_help=False,
formatter_class=argparse.RawTextHelpFormatter,
)
# pylint: disable=possibly-unused-variable
shtab.add_argument_to(
parser,
option_string=["--show-completion"],
help="Show Shell completion script and exit.",
)
source_group = parser.add_argument_group("Test sources")
filtering_group = parser.add_argument_group(
"Source filtering, decoding, conversion and expansion")
test_control_group = parser.add_argument_group("Test control")
dns_control_group = parser.add_argument_group("DNS control")
database_control_group = parser.add_argument_group("Databases")
output_control_group = parser.add_argument_group("Output control")
multiprocessing_group = parser.add_argument_group("Multiprocessing")
ci_group = parser.add_argument_group("CI / CD")
funcs = [
get_source_group_data,
get_filtering_group_data,
get_test_control_group_data,
get_dns_control_group_data,
get_database_control_group_data,
get_output_control_group_data,
get_multiprocessing_group_data,
get_ci_group_data,
]
for func in funcs:
parser_name = func.__name__.replace("get_", "").replace("_data", "")
try:
add_arguments_to_parser(locals()[parser_name], func())
except ValueError as exception:
exception_message = str(exception)
if "configuration" not in exception_message:
raise exception
missing_key = RegexHelper(r"<entry>\s\(\'(.*)\'\)").match(
exception_message, return_match=True, group=1)
if ask_authorization_to_merge_config(missing_key):
PyFunceble.facility.ConfigLoader.set_merge_upstream(
True).start()
add_arguments_to_parser(locals()[parser_name], func())
else:
print(
f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Could not find "
f"the {missing_key!r} in your configuration.\n"
f"{colorama.Fore.MAGENTA}Please fix your "
"configuration file manually or fill a new issue if you "
"don't understand this error.")
sys.exit(1)
add_arguments_to_parser(parser, get_default_group_data())
args = parser.parse_args()
if any(
getattr(args, x)
for x in ["domains", "urls", "files", "url_files"]):
SystemIntegrator(args).start()
SystemLauncher(args).start()
class TestRegexHelper(unittest.TestCase):
"""
Tests our regex helper.
"""
def setUp(self) -> None:
"""
Setups everything needed for the test.
"""
self.helper = RegexHelper()
self.test_regex = "[a-z]"
self.testing_list_subject = [
"hello",
"world",
"funilrys",
"funceble",
"PyFunceble",
"pyfunceble",
]
self.testing_subject = "Hello, this is Fun Ilrys. I just wanted to know how things goes around the tests." # pylint: disable=line-too-long
def tearDown(self) -> None:
"""
Destroys everything previously initialized for the tests.
"""
del self.testing_list_subject
del self.testing_subject
def test_set_regex_return(self) -> None:
"""
Tests the response from the method which let us set the regex to work
with.
"""
actual = self.helper.set_regex(self.test_regex)
self.assertIsInstance(actual, RegexHelper)
def test_set_regex_method(self) -> None:
"""
Tests the method which let us set the regex to work with.
"""
given = self.test_regex
expected = given
self.helper.set_regex(given)
actual = self.helper.regex
self.assertEqual(expected, actual)
def test_set_regex_attribute(self) -> None:
"""
Tests overwritting of the :code:`regex` attribute.
"""
given = self.test_regex
expected = given
self.helper.regex = given
actual = self.helper.regex
self.assertEqual(expected, actual)
def test_set_regex_through_init(self) -> None:
"""
Tests the overwritting of the regex to work through the class
constructor.
"""
given = self.test_regex
expected = given
helper = RegexHelper(given)
actual = helper.regex
self.assertEqual(expected, actual)
def test_set_regex_not_str(self) -> None:
"""
Tests the method which let us set the regex to work with for the case
that it's not a string.
"""
given = ["Hello", "World"]
self.assertRaises(TypeError, lambda: self.helper.set_regex(given))
def test_set_regex_escape(self) -> None:
"""
Tests the method which let us set the regex to work with for the case
that it's not a string.
"""
regex_helper = RegexHelper()
regex_helper.escape_regex = True
regex_helper.set_regex("[a-z]")
expected = r"\[a\-z\]"
actual = regex_helper.regex
self.assertEqual(expected, actual)
def test_not_matching_list(self) -> None:
"""
Tests the method which let us get a list of non
matching strin from a given list of string.
"""
regex = "fun"
expected = ["hello", "world", "PyFunceble"]
actual = self.helper.set_regex(regex).get_not_matching_list(
self.testing_list_subject)
self.assertEqual(expected, actual)
def test_matching_list(self) -> None:
"""
Tests the method which let us get a list of
matchint string from a given list of string.
"""
regex = "fun"
expected = ["funilrys", "funceble", "pyfunceble"]
actual = self.helper.set_regex(regex).get_matching_list(
self.testing_list_subject)
self.assertEqual(expected, actual)
def test_match_check(self) -> None:
"""
Tests the matching method for the case that we want to just check.
"""
regex = r"([a-z]{1,})\s([a-z]{1,})\s"
expected = True
actual = self.helper.set_regex(regex).match(self.testing_subject,
return_match=False)
self.assertEqual(expected, actual)
def test_match_not_check(self) -> None:
"""
Tests the matching method for the case that we want to just check.
"""
regex = r"@funilrys"
expected = False
actual = self.helper.set_regex(regex).match(self.testing_subject,
return_match=False)
self.assertEqual(expected, actual)
def test_match_rematch(self) -> None:
"""
Tests the matching method for the case that we want to rematch
the different groups.
"""
regex = r"([a-z]{1,})\s([a-z]{1,})\s"
expected = "is"
actual = self.helper.set_regex(regex).match(self.testing_subject,
rematch=True,
group=1)
self.assertEqual(expected, actual)
def test_match_get_group(self) -> None:
"""
Tests the matching method for the case that we want
a specific group.
"""
regex = "e"
expected = "e"
actual = self.helper.set_regex(regex).match(self.testing_subject,
group=0)
self.assertEqual(expected, actual)
regex = r"([a-z]{1,})\s([a-z]{1,})\s"
expected = "this"
actual = self.helper.set_regex(regex).match(self.testing_subject,
group=1)
self.assertEqual(expected, actual)
def test_replace_no_replacement(self) -> None:
"""
Tests the replacement method for the case that no replacement
is not given.
"""
regex = "th"
expected = self.testing_subject
actual = self.helper.set_regex(regex).replace_match(
self.testing_subject, None)
self.assertEqual(expected, actual)
def test_replace(self) -> None:
"""
Tests the replacement method.
"""
regex = "th"
expected = ("Hello, htis is Fun Ilrys. I just wanted to know how "
"htings goes around hte tests.")
actual = self.helper.set_regex(regex).replace_match(
self.testing_subject, "ht")
self.assertEqual(expected, actual)
def test_split(self) -> None:
"""
Tests the method which le us split occurences of a given regex.
"""
regex = "th"
expected = [
"Hello, ",
"is is Fun Ilrys. I just wanted to know how ",
"ings goes around ",
"e tests.",
]
actual = self.helper.set_regex(regex).split(self.testing_subject)
self.assertEqual(expected, actual)