def query_a_record(self) -> Optional[List[str]]:
url_base = Url2Netloc(self.status.subject).get_converted()
ip_syntax_checker = IPSyntaxChecker(url_base)
if ip_syntax_checker.is_valid_v4():
return [url_base]
if ip_syntax_checker.is_valid_v6() or (url_base.startswith("[")
and url_base.endswith("]")):
url_base = url_base.replace("[", "").replace("]", "")
result = set()
for subject in (self.dns_query_tool.set_query_record_type(
"PTR").set_subject(url_base).query()):
result.update(
self.dns_query_tool.set_subject(
subject).set_query_record_type("A").query())
self.dns_query_tool.subject = self.idna_subject
return result
result = (self.dns_query_tool.set_query_record_type("A").set_subject(
url_base).query())
self.dns_query_tool.subject = self.idna_subject
return result
def __init__(
self,
subject: Optional[str] = None,
do_syntax_check_first: Optional[bool] = None,
db_session: Optional[Session] = None,
use_collection: Optional[bool] = None,
) -> None:
self.dns_query_tool = DNSQueryTool()
self.ipv4_reputation_query_tool = IPV4ReputationDataset()
self.domain_syntax_checker = DomainSyntaxChecker()
self.ip_syntax_checker = IPSyntaxChecker()
self.url_syntax_checker = URLSyntaxChecker()
self.params = ReputationCheckerParams()
self.status = ReputationCheckerStatus()
self.status.params = self.params
self.status.dns_lookup_record = self.dns_query_tool.lookup_record
super().__init__(
subject,
do_syntax_check_first=do_syntax_check_first,
db_session=db_session,
use_collection=use_collection,
)
class CIDR2Subject(ConverterBase):
"""
Converts/Extracts the subjects of from the given CIDR.
"""
ip_syntax_checker: Optional[IPSyntaxChecker] = None
def __init__(self, data_to_convert: Optional[Any] = None) -> None:
super().__init__(data_to_convert=data_to_convert)
self.ip_syntax_checker = IPSyntaxChecker()
@ConverterBase.data_to_convert.setter
def data_to_convert(self, value: Any) -> None:
"""
Overrites the default behavior.
:raise TypeError:
When the given data to convert is not :py:class:`str`
"""
if not isinstance(value, str):
raise TypeError(f"<value> should be {str}, {type(value)} given.")
# pylint: disable=no-member
super(CIDR2Subject, self.__class__).data_to_convert.fset(self, value)
def get_converted(self) -> List[str]:
"""
Provides the subject-s to test.
"""
result = set()
subject = self.data_to_convert.strip()
if subject:
try:
self.ip_syntax_checker.set_subject(subject)
if self.ip_syntax_checker.is_valid_v4_range():
result.update(
str(x) for x in IPv4Network(self.ip_syntax_checker.subject)
)
elif self.ip_syntax_checker.is_valid_v6_range():
...
# Not Implemented yet.
result.add(subject)
else:
result.add(subject)
except ValueError:
result.add(subject)
return list(result)
def test_is_valid_v4(self) -> None:
"""
Tests the method which let us check if the given subject is valid.
"""
ip_checker = IPSyntaxChecker()
expected = True
for subject in pyf_test_dataset.VALID_IPV4:
actual = ip_checker.set_subject(subject).is_valid()
self.assertEqual(expected, actual, subject)
def test_is_reserved_v6(self) -> None:
"""
Tests the method which let us check if the given subject is reserved.
"""
ip_checker = IPSyntaxChecker()
expected = True
for subject in pyf_test_dataset.RESERVED_IPV6:
ip_checker.subject = subject
actual = ip_checker.is_reserved()
self.assertEqual(expected, actual, subject)
def test_is_valid_range_v6(self) -> None:
"""
Tests the method which let us check if the given subject is valid range.
"""
ip_checker = IPSyntaxChecker()
expected = True
for subject in pyf_test_dataset.VALID_IPV6_RANGES:
ip_checker.subject = subject
actual = ip_checker.is_valid_range()
self.assertEqual(expected, actual, subject)
def test_is_not_valid_v4(self) -> None:
"""
Tests the method which let us check if the given subject is valid for
the case that the given subject is not valid.
"""
ip_checker = IPSyntaxChecker()
expected = False
for subject in pyf_test_dataset.NOT_VALID_IPV4:
ip_checker.subject = subject
actual = ip_checker.is_valid()
self.assertEqual(expected, actual, subject)
def test_is_valid(self) -> None:
"""
Tests the method which let us check if the given subject is valid
IPv4 or IPv6.
"""
ip_checker = IPSyntaxChecker()
expected = True
for subject in pyf_test_dataset.VALID_IPV4 + pyf_test_dataset.VALID_IPV6:
ip_checker.subject = subject
actual = ip_checker.is_valid()
self.assertEqual(expected, actual, subject)
def get_converted(self) -> List[str]:
"""
Provides the subject to test.
"""
result = []
subject = self.data_to_convert.strip()
if subject and (not subject.startswith(self.COMMENT)
and any(not subject.startswith(x)
for x in self.PARTICULAR_COMMENT)):
if self.COMMENT in subject:
subject = subject[:subject.find(self.COMMENT)].strip()
if self.NSLOOKUP_SPACE in subject:
# Comply with RFC 6367:
# Note that nslookup escapes spaces as "\032" for display
# purposes, but a graphical DNS-SD browser should not.
subject = subject.replace(self.NSLOOKUP_SPACE, self.SPACE)
if self.SPACE in subject or self.TAB in subject:
splitted = subject.split()
if IPSyntaxChecker(splitted[0]).is_valid():
# This is for the hosts format.
# If the first entry is an IP, we will only extract
# the entries after the first one.
result.extend(splitted[1:])
else:
# All other cases, we extract every entries.
result.extend(splitted)
else:
result.append(subject)
return result
def resolve_without_cache(self, hostname: str) -> Optional[str]:
"""
Resolves the IP of the given hostname.
:param hostname:
The hostname to get resolve.
"""
def get_last_cname(subject: str) -> Optional[str]:
"""
Given a subject, this function tries to query the CNAME until there
is none.
:param subject:
The first subject.
"""
last_cname_result = []
last_cname_new_subject = subject
while True:
local_last_cname_result = (
self.dns_query_tool.set_query_record_type(
"CNAME").set_subject(last_cname_new_subject).query())
if any(x in last_cname_result
for x in local_last_cname_result):
break
last_cname_result.extend(local_last_cname_result)
if local_last_cname_result:
last_cname_new_subject = local_last_cname_result[0]
else:
break
try:
return last_cname_result[-1]
except IndexError:
return None
result = set()
if not IPSyntaxChecker(hostname).is_valid():
last_cname = get_last_cname(hostname)
if last_cname:
result.update(
self.dns_query_tool.set_query_record_type("A").set_subject(
last_cname).query())
else:
result.update(
self.dns_query_tool.set_query_record_type("A").set_subject(
hostname).query())
else:
result.add(hostname)
if result:
return result.pop()
return None
def cidr(
*,
data: str = Body(
..., embed=True, summary="Data", description="The data to convert."
),
) -> List[str]:
"""
Provides the conversion of an IPv4 range to a list of IP.
"""
if IPSyntaxChecker(data).is_valid_v4_range():
return CIDR2Subject(data_to_convert=data).get_converted()
return []
def is_valid(self) -> bool:
"""
Validate the given subject.
"""
parsed = urllib.parse.urlparse(self.idna_subject)
if not parsed.scheme or not parsed.netloc:
return False
if (DomainSyntaxChecker(parsed.netloc).is_valid()
or IPSyntaxChecker(parsed.netloc).is_valid()):
return True
return False
def is_ipv6_range(subject: str, **kwargs) -> bool:
"""
Checks if the given subject is a syntactically valid IPv6 range.
.. warning::
This method may be removed in the future.
It is still available for convenience.
Please consider the following alternative example:
::
from PyFunceble import IPSyntaxChecker
my_subject = "::1"
the_status = IPSyntaxChecker(
my_subject
).get_status()
# Get the status in dict format.
print(the_status.to_dict())
# Get the status in json format.
print(the_status.to_json())
# Check if it is IPv6 range.
print(f"{my_subject} is IPv6 range ? {the_status.is_valid_v6_range()}")
:param subject:
The subject to work with.
"""
warnings.warn(
"PyFunceble.is_ipv6_range may be removed in the future."
"Please consider using PyFunceble.IPSyntaxChecker explicitly.",
DeprecationWarning,
)
return IPSyntaxChecker(subject, **kwargs).is_valid_v6_range()
def should_be_ignored(subject: str) -> bool:
"""
Checks if the given subject should be ignored.
"""
# pylint: disable=line-too-long
regex_ignore = r"localhost$|localdomain$|local$|broadcasthost$|0\.0\.0\.0$|allhosts$|allnodes$|allrouters$|localnet$|loopback$|mcastprefix$|ip6-mcastprefix$|ip6-localhost$|ip6-loopback$|ip6-allnodes$|ip6-allrouters$|ip6-localnet$"
if RegexHelper(regex_ignore).match(subject, return_match=False):
PyFunceble.facility.Logger.info(
"Ignoring %r because it is in our default regex.", subject)
return True
if (not PyFunceble.storage.CONFIGURATION.cli_testing.local_network
and IPSyntaxChecker(subject).is_reserved()):
PyFunceble.facility.Logger.info(
"Ignoring %r because it is a reserved IP and we are not testing "
"for/in a local network.",
subject,
)
return True
if bool(
PyFunceble.storage.CONFIGURATION.cli_testing.file_filter
) and not RegexHelper(PyFunceble.storage.CONFIGURATION.cli_testing.
file_filter).match(subject, return_match=False):
PyFunceble.facility.Logger.info(
"Ignoring %r because it does not match the filter to look for.",
subject,
)
return True
PyFunceble.facility.Logger.info(
"Allowed to test %r.",
subject,
)
return False
def __init__(
self,
subject: Optional[str] = None,
*,
use_extra_rules: Optional[bool] = None,
use_whois_lookup: Optional[bool] = None,
use_dns_lookup: Optional[bool] = None,
use_netinfo_lookup: Optional[bool] = None,
use_http_code_lookup: Optional[bool] = None,
use_reputation_lookup: Optional[bool] = None,
do_syntax_check_first: Optional[bool] = None,
db_session: Optional[Session] = None,
use_whois_db: Optional[bool] = None,
use_collection: Optional[bool] = None,
) -> None:
self.dns_query_tool = DNSQueryTool()
self.whois_query_tool = WhoisQueryTool()
self.addressinfo_query_tool = AddressInfo()
self.hostbyaddr_query_tool = HostByAddrInfo()
self.http_status_code_query_tool = HTTPStatusCode()
self.domain_syntax_checker = DomainSyntaxChecker()
self.ip_syntax_checker = IPSyntaxChecker()
self.url_syntax_checker = URLSyntaxChecker()
self.extra_rules_handler = ExtraRulesHandler()
self.db_session = db_session
self.params = AvailabilityCheckerParams()
self.status = AvailabilityCheckerStatus()
self.status.params = self.params
self.status.dns_lookup_record = self.dns_query_tool.lookup_record
self.status.whois_lookup_record = self.whois_query_tool.lookup_record
if use_extra_rules is not None:
self.use_extra_rules = use_extra_rules
else:
self.guess_and_set_use_extra_rules()
if use_whois_lookup is not None:
self.use_whois_lookup = use_whois_lookup
else:
self.guess_and_set_use_whois_lookup()
if use_dns_lookup is not None:
self.use_dns_lookup = use_dns_lookup
else:
self.guess_and_set_dns_lookup()
if use_netinfo_lookup is not None:
self.use_netinfo_lookup = use_netinfo_lookup
else:
self.guess_and_set_use_netinfo_lookup()
if use_http_code_lookup is not None:
self.use_http_code_lookup = use_http_code_lookup
else:
self.guess_and_set_use_http_code_lookup()
if use_reputation_lookup is not None:
self.use_reputation_lookup = use_reputation_lookup
else:
self.guess_and_set_use_reputation_lookup()
if use_whois_db is not None:
self.use_whois_db = use_whois_db
else:
self.guess_and_set_use_whois_db()
super().__init__(
subject,
do_syntax_check_first=do_syntax_check_first,
db_session=db_session,
use_collection=use_collection,
)
def __init__(self, data_to_convert: Optional[Any] = None) -> None:
super().__init__(data_to_convert=data_to_convert)
self.ip_syntax_checker = IPSyntaxChecker()
class ReputationCheckerBase(CheckerBase):
"""
Provides the base of all our reputation checker classes.
:param str subject:
Optional, The subject to work with.
:param bool do_syntax_check_first:
Optional, Activates/Disables the check of the status before the actual
status gathering.
"""
dns_query_tool: Optional[DNSQueryTool] = None
ipv4_reputation_query_tool: Optional[IPV4ReputationDataset] = None
domain_syntax_checker: Optional[DomainSyntaxChecker] = None
ip_syntax_checker: Optional[IPSyntaxChecker] = None
url_syntax_checker: Optional[URLSyntaxChecker] = None
status: Optional[ReputationCheckerStatus] = None
params: Optional[ReputationCheckerParams] = None
def __init__(
self,
subject: Optional[str] = None,
do_syntax_check_first: Optional[bool] = None,
db_session: Optional[Session] = None,
use_collection: Optional[bool] = None,
) -> None:
self.dns_query_tool = DNSQueryTool()
self.ipv4_reputation_query_tool = IPV4ReputationDataset()
self.domain_syntax_checker = DomainSyntaxChecker()
self.ip_syntax_checker = IPSyntaxChecker()
self.url_syntax_checker = URLSyntaxChecker()
self.params = ReputationCheckerParams()
self.status = ReputationCheckerStatus()
self.status.params = self.params
self.status.dns_lookup_record = self.dns_query_tool.lookup_record
super().__init__(
subject,
do_syntax_check_first=do_syntax_check_first,
db_session=db_session,
use_collection=use_collection,
)
@staticmethod
def is_valid() -> bool:
raise NotImplementedError()
def subject_propagator(self) -> "CheckerBase":
"""
Propagate the currently set subject.
.. warning::
You are not invited to run this method directly.
"""
self.dns_query_tool.set_subject(self.idna_subject)
self.domain_syntax_checker.subject = self.idna_subject
self.ip_syntax_checker.subject = self.idna_subject
self.url_syntax_checker.subject = self.idna_subject
self.status = ReputationCheckerStatus()
self.status.params = self.params
self.status.dns_lookup_record = self.dns_query_tool.lookup_record
self.status.subject = self.subject
self.status.idna_subject = self.idna_subject
self.query_syntax_checker()
return self
def should_we_continue_test(self, status_post_syntax_checker: str) -> bool:
"""
Checks if we are allowed to continue a standard testing.
"""
return bool(
not self.status.status
or status_post_syntax_checker == PyFunceble.storage.STATUS.invalid)
def query_syntax_checker(self) -> "ReputationCheckerBase":
"""
Queries the syntax checker.
"""
self.status.second_level_domain_syntax = (
self.domain_syntax_checker.is_valid_second_level())
self.status.subdomain_syntax = self.domain_syntax_checker.is_valid_subdomain(
)
self.status.domain_syntax = bool(
self.status.subdomain_syntax
or self.status.second_level_domain_syntax)
self.status.ipv4_syntax = self.ip_syntax_checker.is_valid_v4()
self.status.ipv6_syntax = self.ip_syntax_checker.is_valid_v6()
self.status.ipv4_range_syntax = self.ip_syntax_checker.is_valid_v4_range(
)
self.status.ipv6_range_syntax = self.ip_syntax_checker.is_valid_v6_range(
)
self.status.ip_syntax = bool(self.status.ipv4_syntax
or self.status.ipv6_syntax)
self.status.url_syntax = self.url_syntax_checker.is_valid()
return self
def query_a_record(self) -> Optional[List[str]]:
"""
Queries all the A record.
"""
raise NotImplementedError()
def try_to_query_status_from_dns_lookup(self) -> "ReputationCheckerBase":
"""
Tries to query the status from the DNS lookup.
"""
PyFunceble.facility.Logger.info(
"Started to try to query the status of %r from: DNS Lookup",
self.status.idna_subject,
)
if not self.status.ipv4_syntax:
lookup_result = self.query_a_record()
self.status.dns_lookup = lookup_result
else:
lookup_result = [self.status.subject]
if lookup_result:
for subject in lookup_result:
if subject in self.ipv4_reputation_query_tool:
self.status.status = PyFunceble.storage.STATUS.malicious
self.status.status_source = "REPUTATION"
PyFunceble.facility.Logger.info(
"Could define the status of %r from: DNS Lookup",
self.status.idna_subject,
)
break
PyFunceble.facility.Logger.info(
"Finished to try to query the status of %r from: DNS Lookup",
self.status.idna_subject,
)
return self
def try_to_query_status_from_syntax_lookup(
self) -> "ReputationCheckerBase":
"""
Tries to query the status from the syntax.
"""
PyFunceble.facility.Logger.info(
"Started to try to query the status of %r from: Syntax Lookup",
self.status.idna_subject,
)
if (not self.status.domain_syntax and not self.status.ip_syntax
and not self.status.url_syntax):
self.status.status = PyFunceble.storage.STATUS.invalid
self.status.status_source = "SYNTAX"
PyFunceble.facility.Logger.info(
"Could define the status of %r from: Syntax Lookup",
self.status.idna_subject,
)
PyFunceble.facility.Logger.info(
"Finished to try to query the status of %r from: Syntax Lookup",
self.status.idna_subject,
)
return self
def try_to_query_status_from_collection(self) -> "ReputationCheckerBase":
"""
Tries to get and set the status from the Collection API.
"""
PyFunceble.facility.Logger.info(
"Started to try to query the status of %r from: Collection Lookup",
self.status.idna_subject,
)
data = self.collection_query_tool[self.idna_subject]
if data and "status" in data:
if (self.collection_query_tool.preferred_status_origin
== "frequent"
and data["status"]["reputation"]["frequent"]):
self.status.status = data["status"]["reputation"]["frequent"]
self.status.status_source = "COLLECTION"
elif (self.collection_query_tool.preferred_status_origin
== "latest" and data["status"]["reputation"]["latest"]):
self.status.status = data["status"]["reputation"]["latest"][
"status"]
self.status.status_source = "COLLECTION"
elif (self.collection_query_tool.preferred_status_origin
== "recommended"
and data["status"]["reputation"]["recommended"]):
self.status.status = data["status"]["reputation"][
"recommended"]
self.status.status_source = "COLLECTION"
PyFunceble.facility.Logger.info(
"Could define the status of %r from: Collection Lookup",
self.status.idna_subject,
)
PyFunceble.facility.Logger.info(
"Finished to try to query the status of %r from: Collection Lookup",
self.status.idna_subject,
)
@CheckerBase.ensure_subject_is_given
@CheckerBase.update_status_date_after_query
def query_status(self) -> "ReputationCheckerBase":
"""
Queries the status and for for more action.
"""
status_post_syntax_checker = None
if (not self.status.status and
self.use_collection): # pragma: no cover ## Underlying tested
self.try_to_query_status_from_collection()
if not self.status.status and self.do_syntax_check_first:
self.try_to_query_status_from_syntax_lookup()
if self.status.status:
status_post_syntax_checker = self.status.status
if self.should_we_continue_test(status_post_syntax_checker):
self.try_to_query_status_from_dns_lookup()
if not self.status.status:
self.status.status = PyFunceble.storage.STATUS.sane
self.status.status_source = "REPUTATION"
PyFunceble.facility.Logger.info(
"Could not define the status of %r. Setting to %r",
self.status.idna_subject,
self.status.status,
)
return self
# pylint: disable=useless-super-delegation
def get_status(self) -> Optional[ReputationCheckerStatus]:
return super().get_status()
class AvailabilityCheckerBase(CheckerBase):
"""
Provides the base of all our availability checker classes.
:param str subject:
Optional, The subject to work with.
:param bool use_extra_rules:
Optional, Activates/Disables the usage of our own set of extra rules.
:param bool use_whois_lookup:
Optional, Activates/Disables the usage of the WHOIS lookup to gather
the status of the given :code:`subject`.
:param bool use_dns_lookup:
Optional, Activates/Disables the usage of the DNS lookup to gather the
status of the given :code:`subject`.
:param bool use_netinfo_lookup:
Optional, Activates/Disables the usage of the network information
lookup module to gather the status of the given :code:`subject`.
:param bool use_http_code_lookup:
Optional, Activates/Disables the usage of the HTTP status code lookup
to gather the status of the given :code:`subject`.
:param bool use_reputation_lookup:
Optional, Activates/Disables the usage of the reputation dataset
lookup to gather the status of the given :code:`subject`.
:param bool do_syntax_check_first:
Optional, Activates/Disables the check of the status before the actual
status gathering.
:param bool use_whois_db:
Optional, Activates/Disable the usage of a local database to store the
WHOIS datasets.
"""
# pylint: disable=too-many-public-methods, too-many-instance-attributes
STD_USE_EXTRA_RULES: bool = True
STD_USE_WHOIS_LOOKUP: bool = True
STD_USE_DNS_LOOKUP: bool = True
STD_USE_NETINFO_LOOKUP: bool = True
STD_USE_HTTP_CODE_LOOKUP: bool = True
STD_USE_REPUTATION_LOOKUP: bool = False
STD_USE_WHOIS_DB: bool = True
dns_query_tool: Optional[DNSQueryTool] = None
whois_query_tool: Optional[WhoisQueryTool] = None
addressinfo_query_tool: Optional[AddressInfo] = None
hostbyaddr_query_tool: Optional[HostByAddrInfo] = None
http_status_code_query_tool: Optional[HTTPStatusCode] = None
domain_syntax_checker: Optional[DomainSyntaxChecker] = None
ip_syntax_checker: Optional[IPSyntaxChecker] = None
url_syntax_checker: Optional[URLSyntaxChecker] = None
extra_rules_handler: Optional[ExtraRulesHandler] = None
_use_extra_rules: bool = False
_use_whois_lookup: bool = False
_use_dns_lookup: bool = False
_use_netinfo_lookup: bool = False
_use_http_code_lookup: bool = False
_use_reputation_lookup: bool = False
_use_whois_db: bool = False
status: Optional[AvailabilityCheckerStatus] = None
params: Optional[AvailabilityCheckerParams] = None
def __init__(
self,
subject: Optional[str] = None,
*,
use_extra_rules: Optional[bool] = None,
use_whois_lookup: Optional[bool] = None,
use_dns_lookup: Optional[bool] = None,
use_netinfo_lookup: Optional[bool] = None,
use_http_code_lookup: Optional[bool] = None,
use_reputation_lookup: Optional[bool] = None,
do_syntax_check_first: Optional[bool] = None,
db_session: Optional[Session] = None,
use_whois_db: Optional[bool] = None,
use_collection: Optional[bool] = None,
) -> None:
self.dns_query_tool = DNSQueryTool()
self.whois_query_tool = WhoisQueryTool()
self.addressinfo_query_tool = AddressInfo()
self.hostbyaddr_query_tool = HostByAddrInfo()
self.http_status_code_query_tool = HTTPStatusCode()
self.domain_syntax_checker = DomainSyntaxChecker()
self.ip_syntax_checker = IPSyntaxChecker()
self.url_syntax_checker = URLSyntaxChecker()
self.extra_rules_handler = ExtraRulesHandler()
self.db_session = db_session
self.params = AvailabilityCheckerParams()
self.status = AvailabilityCheckerStatus()
self.status.params = self.params
self.status.dns_lookup_record = self.dns_query_tool.lookup_record
self.status.whois_lookup_record = self.whois_query_tool.lookup_record
if use_extra_rules is not None:
self.use_extra_rules = use_extra_rules
else:
self.guess_and_set_use_extra_rules()
if use_whois_lookup is not None:
self.use_whois_lookup = use_whois_lookup
else:
self.guess_and_set_use_whois_lookup()
if use_dns_lookup is not None:
self.use_dns_lookup = use_dns_lookup
else:
self.guess_and_set_dns_lookup()
if use_netinfo_lookup is not None:
self.use_netinfo_lookup = use_netinfo_lookup
else:
self.guess_and_set_use_netinfo_lookup()
if use_http_code_lookup is not None:
self.use_http_code_lookup = use_http_code_lookup
else:
self.guess_and_set_use_http_code_lookup()
if use_reputation_lookup is not None:
self.use_reputation_lookup = use_reputation_lookup
else:
self.guess_and_set_use_reputation_lookup()
if use_whois_db is not None:
self.use_whois_db = use_whois_db
else:
self.guess_and_set_use_whois_db()
super().__init__(
subject,
do_syntax_check_first=do_syntax_check_first,
db_session=db_session,
use_collection=use_collection,
)
@property
def use_extra_rules(self) -> bool:
"""
Provides the current value of the :code:`_use_extra_rules` attribute.
"""
return self._use_extra_rules
@use_extra_rules.setter
def use_extra_rules(self, value: bool) -> None:
"""
Sets the value which authorizes the usage of the special rule.
:param value:
The value to set.
:raise TypeError:
When the given :code:`value` is not a :py:class:`bool`.
"""
if not isinstance(value, bool):
raise TypeError(f"<value> should be {bool}, {type(value)} given.")
self._use_extra_rules = self.params.use_extra_rules = value
def set_use_extra_rules(self, value: bool) -> "AvailabilityCheckerBase":
"""
Sets the value which authorizes the usage of the special rule.
:param value:
The value to set.
"""
self.use_extra_rules = value
return self
@property
def use_whois_lookup(self) -> bool:
"""
Provides the current value of the :code:`_use_whois_lookup` attribute.
"""
return self._use_whois_lookup
@use_whois_lookup.setter
def use_whois_lookup(self, value: bool) -> None:
"""
Sets the value which authorizes the usage of the WHOIS lookup.
:param value:
The value to set.
:raise TypeError:
When the given :code:`value` is not a :py:class:`bool`.
"""
if not isinstance(value, bool):
raise TypeError(f"<value> should be {bool}, {type(value)} given.")
self._use_whois_lookup = self.params.use_whois_lookup = value
def set_use_whois_lookup(self, value: bool) -> "AvailabilityCheckerBase":
"""
Sets the value which authorizes the usage of the WHOIS lookup.
:param value:
The value to set.
"""
self.use_whois_lookup = value
return self
@property
def use_dns_lookup(self) -> bool:
"""
Provides the current value of the :code:`_use_dns_lookup` attribute.
"""
return self._use_dns_lookup
@use_dns_lookup.setter
def use_dns_lookup(self, value: bool) -> None:
"""
Sets the value which authorizes the usage of the DNS Lookup.
:param value:
The value to set.
:raise TypeError:
When the given :code:`value` is not a :py:class:`bool`.
"""
if not isinstance(value, bool):
raise TypeError(f"<value> should be {bool}, {type(value)} given.")
self._use_dns_lookup = self.params.use_dns_lookup = value
def set_use_dns_lookup(self, value: bool) -> "AvailabilityCheckerBase":
"""
Sets the value which authorizes the usage of the DNS Lookup.
:param value:
The value to set.
"""
self.use_dns_lookup = value
return self
@property
def use_netinfo_lookup(self) -> bool:
"""
Provides the current value of the :code:`_use_netinfo_lookup` attribute.
"""
return self._use_netinfo_lookup
@use_netinfo_lookup.setter
def use_netinfo_lookup(self, value: bool) -> None:
"""
Sets the value which authorizes the usage of the network information
lookup.
:param value:
The value to set.
:raise TypeError:
When the given :code:`value` is not a :py:class:`bool`.
"""
if not isinstance(value, bool):
raise TypeError(f"<value> should be {bool}, {type(value)} given.")
self._use_netinfo_lookup = self.params.use_netinfo_lookup = value
def set_use_netinfo_lookup(self, value: bool) -> "AvailabilityCheckerBase":
"""
Sets the value which authorizes the usage of the network information
lookup.
:param value:
The value to set.
"""
self.use_netinfo_lookup = value
return self
@property
def use_http_code_lookup(self) -> None:
"""
Provides the current value of the :code:`_use_http_code_lookup` attribute.
"""
return self._use_http_code_lookup
@use_http_code_lookup.setter
def use_http_code_lookup(self, value: bool) -> None:
"""
Sets the value which authorizes the usage of the HTTP status code
lookup.
:param value:
The value to set.
:raise TypeError:
When the given :code:`value` is not a :py:class:`bool`.
"""
if not isinstance(value, bool):
raise TypeError(f"<value> should be {bool}, {type(value)} given.")
self._use_http_code_lookup = self.params.use_http_code_lookup = value
def set_use_http_code_lookup(self,
value: bool) -> "AvailabilityCheckerBase":
"""
Sets the value which authorizes the usage of the HTTP status code
lookup.
:param value:
The value to set.
"""
self.use_http_code_lookup = value
return self
@property
def use_reputation_lookup(self) -> bool:
"""
Provides the current value of the :code:`_use_reputation_lookup` attribute.
"""
return self._use_reputation_lookup
@use_reputation_lookup.setter
def use_reputation_lookup(self, value: bool) -> None:
"""
Sets the value which authorizes the usage of the reputation
lookup.
:param value:
The value to set.
:raise TypeError:
When the given :code:`value` is not a :py:class:`bool`.
"""
if not isinstance(value, bool):
raise TypeError(f"<value> should be {bool}, {type(value)} given.")
self._use_reputation_lookup = self.params.use_reputation_lookup = value
def set_use_reputation_lookup(self,
value: bool) -> "AvailabilityCheckerBase":
"""
Sets the value which authorizes the usage of the reputation
lookup.
:param value:
The value to set.
"""
self.use_reputation_lookup = value
return self
@property
def use_whois_db(self) -> bool:
"""
Provides the current value of the :code:`_use_whois_db` attribute.
"""
return self._use_whois_db
@use_whois_db.setter
def use_whois_db(self, value: bool) -> None:
"""
Sets the value which authorizes the usage of the WHOIS DB.
:param value:
The value to set.
:param TypeError:
When the given :code:`use_whois_db` is not a :py:class:`bool`.
"""
if not isinstance(value, bool):
raise TypeError(f"<value> should be {bool}, {type(value)} given.")
self._use_whois_db = self.params.use_whois_db = value
def set_use_whois_db(self, value: bool) -> "AvailabilityCheckerBase":
"""
Sets the value which authorizes the usage of the WHOIS DB.
:param value:
The value to set.
"""
self.use_whois_db = value
return self
def subject_propagator(self) -> "CheckerBase":
"""
Propagate the currently set subject.
.. warning::
You are not invited to run this method directly.
"""
self.dns_query_tool.set_subject(self.idna_subject)
self.whois_query_tool.set_subject(self.idna_subject)
self.addressinfo_query_tool.set_subject(self.idna_subject)
self.hostbyaddr_query_tool.set_subject(self.idna_subject)
self.http_status_code_query_tool.set_subject(self.idna_subject)
self.domain_syntax_checker.subject = self.idna_subject
self.ip_syntax_checker.subject = self.idna_subject
self.url_syntax_checker.subject = self.idna_subject
self.status = AvailabilityCheckerStatus()
self.status.params = self.params
self.status.dns_lookup_record = self.dns_query_tool.lookup_record
self.status.whois_lookup_record = self.whois_query_tool.lookup_record
self.status.subject = self.subject
self.status.idna_subject = self.idna_subject
self.status.status = None
self.query_syntax_checker()
return self
def should_we_continue_test(self, status_post_syntax_checker: str) -> bool:
"""
Checks if we are allowed to continue a standard testing.
"""
return bool(
not self.status.status
or status_post_syntax_checker == PyFunceble.storage.STATUS.invalid)
def guess_and_set_use_extra_rules(self) -> "AvailabilityCheckerBase":
"""
Try to guess and set the value of the :code:`use_extra_rules` attribute
from the configuration file.
"""
if PyFunceble.facility.ConfigLoader.is_already_loaded():
self.use_extra_rules = PyFunceble.storage.CONFIGURATION.lookup.special
else:
self.use_extra_rules = self.STD_USE_EXTRA_RULES
return self
def guess_and_set_use_whois_lookup(self) -> "AvailabilityCheckerBase":
"""
Try to guess and set the value of the :code:`use_whois` attribute
from the configuration file.
"""
if PyFunceble.facility.ConfigLoader.is_already_loaded():
self.use_whois_lookup = PyFunceble.storage.CONFIGURATION.lookup.whois
else:
self.use_whois_lookup = self.STD_USE_WHOIS_LOOKUP
return self
def guess_and_set_dns_lookup(self) -> "AvailabilityCheckerBase":
"""
Try to guess and set the value of the :code:`use_dns_lookup` attribute
from the configuration file.
"""
if PyFunceble.facility.ConfigLoader.is_already_loaded():
self.use_dns_lookup = PyFunceble.storage.CONFIGURATION.lookup.dns
else:
self.use_dns_lookup = self.STD_USE_DNS_LOOKUP
return self
def guess_and_set_use_netinfo_lookup(self) -> "AvailabilityCheckerBase":
"""
Try to guess and set the value of the :code:`use_netinfo_lookup` attribute
from the configuration file.
"""
if PyFunceble.facility.ConfigLoader.is_already_loaded():
self.use_netinfo_lookup = PyFunceble.storage.CONFIGURATION.lookup.netinfo
else:
self.use_netinfo_lookup = self.STD_USE_NETINFO_LOOKUP
return self
def guess_and_set_use_http_code_lookup(self) -> "AvailabilityCheckerBase":
"""
Try to guess and set the value of the :code:`use_http_code_lookup` attribute
from the configuration file.
"""
if PyFunceble.facility.ConfigLoader.is_already_loaded():
self.use_http_code_lookup = (
PyFunceble.storage.CONFIGURATION.lookup.http_status_code)
else:
self.use_http_code_lookup = self.STD_USE_HTTP_CODE_LOOKUP
return self
def guess_and_set_use_reputation_lookup(self) -> "AvailabilityCheckerBase":
"""
Try to guess and set the value of the :code:`use_reputation_lookup` attribute
from the configuration file.
"""
if PyFunceble.facility.ConfigLoader.is_already_loaded():
self.use_reputation_lookup = (
PyFunceble.storage.CONFIGURATION.lookup.reputation)
else:
self.use_reputation_lookup = self.STD_USE_REPUTATION_LOOKUP
return self
def guess_and_set_use_whois_db(self) -> "AvailabilityCheckerBase":
"""
Try to guess and set the value of the :code:`use_whois_db` attribute.
"""
if PyFunceble.facility.ConfigLoader.is_already_loaded():
self.use_whois_db = PyFunceble.storage.CONFIGURATION.cli_testing.whois_db
else:
self.use_whois_db = self.STD_USE_WHOIS_DB
def guess_all_settings(
self,
) -> "AvailabilityCheckerBase": # pragma: no cover ## Method are more important
"""
Try to guess all settings.
"""
to_ignore = ["guess_all_settings"]
for method in dir(self):
if method in to_ignore or not method.startswith("guess_"):
continue
getattr(self, method)()
return self
def query_syntax_checker(self) -> "AvailabilityCheckerBase":
"""
Queries the syntax checker.
"""
PyFunceble.facility.Logger.info("Started to check the syntax of %r",
self.status.idna_subject)
self.status.second_level_domain_syntax = (
self.domain_syntax_checker.is_valid_second_level())
self.status.subdomain_syntax = self.domain_syntax_checker.is_valid_subdomain(
)
self.status.domain_syntax = bool(self.status.subdomain_syntax) or bool(
self.status.second_level_domain_syntax)
self.status.ipv4_syntax = self.ip_syntax_checker.is_valid_v4()
self.status.ipv6_syntax = self.ip_syntax_checker.is_valid_v6()
self.status.ipv4_range_syntax = self.ip_syntax_checker.is_valid_v4_range(
)
self.status.ipv6_range_syntax = self.ip_syntax_checker.is_valid_v6_range(
)
self.status.ip_syntax = bool(self.status.ipv4_syntax
or self.status.ipv6_syntax)
self.status.url_syntax = self.url_syntax_checker.is_valid()
PyFunceble.facility.Logger.info("Finished to check the syntax of %r",
self.status.idna_subject)
return self
@CheckerBase.ensure_subject_is_given
def query_dns_record(self) -> Optional[Dict[str, Optional[List[str]]]]:
"""
Tries to query the DNS record(s) of the given subject.
"""
PyFunceble.facility.Logger.info(
"Started to try to query the DNS record of %r.",
self.status.idna_subject,
)
result = {}
if self.status.subdomain_syntax:
lookup_order = ["NS", "A", "AAAA", "CNAME", "DNAME"]
elif self.status.domain_syntax:
lookup_order = ["NS", "CNAME", "A", "AAAA", "DNAME"]
elif self.status.ip_syntax:
lookup_order = ["PTR"]
else:
lookup_order = []
if lookup_order:
for record_type in lookup_order:
local_result = self.dns_query_tool.set_query_record_type(
record_type).query()
if local_result:
result[record_type] = local_result
break
PyFunceble.facility.Logger.debug("DNS Record:\n%r", result)
PyFunceble.facility.Logger.info(
"Finished to try to query the DNS record of %r",
self.status.idna_subject,
)
return result
def try_to_query_status_from_whois(self, ) -> "AvailabilityCheckerBase":
"""
Tries to get and the status from the WHOIS record.
.. warning::
If the configuration is loaded, this method try to query from the
best database first.
If it's not found it will try to query it from the best WHOIS server
then add it into the database (if the expiration date
extraction is successful).
.. note::
The addition into the WHOIS database is only done if this method is
running in a process with a name that does not starts with
:code:`PyFunceble` (case sensitive).
"""
PyFunceble.facility.Logger.info(
"Started to try to query the status of %r from: WHOIS Lookup",
self.status.idna_subject,
)
if (
PyFunceble.facility.ConfigLoader.is_already_loaded()
and self.use_whois_db
): # pragma: no cover ## Not interesting enough to spend time on it.
whois_object = PyFunceble.checker.utils.whois.get_whois_dataset_object(
db_session=self.db_session)
known_record = whois_object[self.subject]
if known_record and not isinstance(known_record, dict):
# Comes from DB engine.
known_record = known_record.to_dict()
if not known_record:
# We assume that expired dataset are never saved into the
# dataset.
self.status.expiration_date = (
self.whois_query_tool.get_expiration_date())
self.status.whois_record = self.whois_query_tool.lookup_record.record
if (self.status.expiration_date and
not multiprocessing.current_process().name.startswith(
PyFunceble.storage.PROJECT_NAME.lower())):
whois_object.update({
"subject":
self.subject,
"idna_subject":
self.idna_subject,
"expiration_date":
self.status.expiration_date,
"epoch":
str(
datetime.strptime(self.status.expiration_date,
"%d-%b-%Y").timestamp()),
})
else:
self.status.expiration_date = known_record["expiration_date"]
self.status.whois_record = None
else:
self.status.expiration_date = self.whois_query_tool.get_expiration_date(
)
self.status.whois_record = self.whois_query_tool.lookup_record.record
if self.status.expiration_date:
self.status.status = PyFunceble.storage.STATUS.up
self.status.status_source = "WHOIS"
PyFunceble.facility.Logger.info(
"Could define the status of %r from: WHOIS Lookup",
self.status.idna_subject,
)
PyFunceble.facility.Logger.info(
"Finished to try to query the status of %r from: WHOIS Lookup",
self.status.idna_subject,
)
return self
def try_to_query_status_from_dns(self) -> "AvailabilityCheckerBase":
"""
Tries to query the status from the DNS lookup.
"""
PyFunceble.facility.Logger.info(
"Started to try to query the status of %r from: DNS Lookup",
self.status.idna_subject,
)
lookup_result = self.query_dns_record()
if lookup_result:
self.status.dns_lookup = lookup_result
self.status.status = PyFunceble.storage.STATUS.up
self.status.status_source = "DNSLOOKUP"
PyFunceble.facility.Logger.info(
"Could define the status of %r from: DNS Lookup",
self.status.idna_subject,
)
PyFunceble.facility.Logger.info(
"Finished to try to query the status of %r from: DNS Lookup",
self.status.idna_subject,
)
return self
def try_to_query_status_from_netinfo(self) -> "AvailabilityCheckerBase":
"""
Tries to query the status from the network information.
"""
PyFunceble.facility.Logger.info(
"Started to try to query the status of %r from: NETINFO Lookup",
self.status.idna_subject,
)
if self.status.domain_syntax:
lookup_result = self.addressinfo_query_tool.get_info()
elif self.status.ip_syntax:
lookup_result = self.hostbyaddr_query_tool.get_info()
elif self.status.idna_subject.isdigit():
lookup_result = None
else:
lookup_result = self.addressinfo_query_tool.get_info()
if lookup_result:
self.status.netinfo = lookup_result
self.status.status = PyFunceble.storage.STATUS.up
self.status.status_source = "NETINFO"
PyFunceble.facility.Logger.info(
"Could define the status of %r from: NETINFO Lookup",
self.status.idna_subject,
)
PyFunceble.facility.Logger.info(
"Finished to try to query the status of %r from: NETINFO Lookup",
self.status.idna_subject,
)
return self
def try_to_query_status_from_http_status_code(
self) -> "AvailabilityCheckerBase":
"""
Tries to query the status from the HTTP status code.
"""
PyFunceble.facility.Logger.info(
"Started to try to query the status of %r from: HTTP Status code Lookup",
self.status.idna_subject,
)
if not self.status.url_syntax and not RegexHelper("[^a-z0-9._]").match(
self.idna_subject, return_match=False):
# The regex is there because while testing for domain, sometime we
# may see something like mailto:[email protected]
self.http_status_code_query_tool.set_subject(
f"http://{self.idna_subject}:80")
lookup_result = self.http_status_code_query_tool.get_status_code()
if (lookup_result and lookup_result !=
self.http_status_code_query_tool.STD_UNKNOWN_STATUS_CODE):
self.status.http_status_code = lookup_result
if (PyFunceble.facility.ConfigLoader.is_already_loaded()
): # pragma: no cover ## Special behavior.
dataset = PyFunceble.storage.HTTP_CODES
else:
dataset = PyFunceble.storage.STD_HTTP_CODES
if (not self.status.status
or self.status.status == PyFunceble.storage.STATUS.down
) and (self.status.http_status_code in dataset.list.up
or self.status.http_status_code
in dataset.list.potentially_up):
self.status.status = PyFunceble.storage.STATUS.up
self.status.status_source = "HTTP CODE"
PyFunceble.facility.Logger.info(
"Could define the status of %r from: HTTP Status code Lookup",
self.status.idna_subject,
)
else:
self.status.http_status_code = None
PyFunceble.facility.Logger.info(
"Finished to try to query the status of %r from: HTTP Status code Lookup",
self.status.idna_subject,
)
return self
def try_to_query_status_from_syntax_lookup(
self) -> "AvailabilityCheckerBase":
"""
Tries to query the status from the syntax.
"""
PyFunceble.facility.Logger.info(
"Started to try to query the status of %r from: Syntax Lookup",
self.status.idna_subject,
)
if (not self.status.domain_syntax and not self.status.ip_syntax
and not self.status.url_syntax):
self.status.status = PyFunceble.storage.STATUS.invalid
self.status.status_source = "SYNTAX"
PyFunceble.facility.Logger.info(
"Could define the status of %r from: Syntax Lookup",
self.status.idna_subject,
)
PyFunceble.facility.Logger.info(
"Finished to try to query the status of %r from: Syntax Lookup",
self.status.idna_subject,
)
return self
def try_to_query_status_from_reputation(self) -> "AvailabilityCheckerBase":
"""
Tries to query the status from the reputation lookup.
"""
raise NotImplementedError()
def try_to_query_status_from_collection(self) -> "AvailabilityCheckerBase":
"""
Tries to get and set the status from the Collection API.
"""
PyFunceble.facility.Logger.info(
"Started to try to query the status of %r from: Collection Lookup",
self.status.idna_subject,
)
data = self.collection_query_tool.pull(self.idna_subject)
if data and "status" in data:
if (self.collection_query_tool.preferred_status_origin
== "frequent"
and data["status"]["availability"]["frequent"]):
self.status.status = data["status"]["availability"]["frequent"]
self.status.status_source = "COLLECTION"
elif (self.collection_query_tool.preferred_status_origin
== "latest" and data["status"]["availability"]["latest"]):
self.status.status = data["status"]["availability"]["latest"][
"status"]
self.status.status_source = "COLLECTION"
elif (self.collection_query_tool.preferred_status_origin
== "recommended"
and data["status"]["availability"]["recommended"]):
self.status.status = data["status"]["availability"][
"recommended"]
self.status.status_source = "COLLECTION"
PyFunceble.facility.Logger.info(
"Could define the status of %r from: Collection Lookup",
self.status.idna_subject,
)
PyFunceble.facility.Logger.info(
"Finished to try to query the status of %r from: Collection Lookup",
self.status.idna_subject,
)
@CheckerBase.ensure_subject_is_given
@CheckerBase.update_status_date_after_query
def query_status(self) -> "AvailabilityCheckerBase":
"""
Queries the status and for for more action.
"""
raise NotImplementedError()
# pylint: disable=useless-super-delegation
def get_status(self) -> Optional[AvailabilityCheckerStatus]:
return super().get_status()
def fetch_data(repo_name: str, info_dir: str) -> Tuple[str]:
"""
Fetches the data of the given input source.
"""
logging.info("Let's fetch the data behind %r", repo_name)
url_base = hubgit.PARTIAL_RAW_URL % repo_name
info_url = url_base + "info.json"
domain_url = url_base + "domains.list"
clean_url = url_base + "clean.list"
ip_url = url_base + "ip.list"
whitelisted_url = url_base + "whitelisted.list"
domain_found = False
clean_found = False
ip_found = False
whitelisted_found = False
ip_file_to_deliver = None
domain_file_to_deliver = None
download_info_file = os.path.join(info_dir, secrets.token_hex(8))
downloaded_ip_file = tempfile.NamedTemporaryFile("r", delete=False)
downloaded_domain_file = tempfile.NamedTemporaryFile("r", delete=False)
downloaded_clean_file = tempfile.NamedTemporaryFile("r", delete=False)
downloaded_whitelisted_file = tempfile.NamedTemporaryFile("r", delete=False)
output_ip_file = tempfile.NamedTemporaryFile("w", delete=False)
output_domain_file = tempfile.NamedTemporaryFile("w", delete=False)
try:
logging.info(
"[%r] Started to download %r into %r",
repo_name,
info_url,
download_info_file,
)
DownloadHelper(info_url).download_text(destination=download_info_file)
logging.info(
"[%r] Finished to download %r into %r",
repo_name,
info_url,
download_info_file,
)
except UnableToDownload:
logging.critical(
"[%r] Could not download %r into %r. Reason: Not found.",
repo_name,
info_url,
download_info_file,
)
try:
logging.info(
"[%r] Started to download %r into %r",
repo_name,
domain_url,
downloaded_domain_file.name,
)
DownloadHelper(domain_url).download_text(
destination=downloaded_domain_file.name
)
logging.info(
"[%r] Finished to download %r into %r",
repo_name,
domain_url,
downloaded_domain_file.name,
)
domain_found = True
except UnableToDownload:
logging.critical(
"[%r] Could not download %r into %r. Reason: Not found.",
repo_name,
domain_url,
downloaded_domain_file.name,
)
try:
logging.info(
"[%r] Started to download %r into %r",
repo_name,
clean_url,
downloaded_clean_file.name,
)
DownloadHelper(clean_url).download_text(
destination=downloaded_clean_file.name
)
logging.info(
"[%r] Finished to download %r into %r",
repo_name,
clean_url,
downloaded_clean_file.name,
)
clean_found = True
except UnableToDownload:
logging.critical(
"[%r] Could not download %r into %r. Reason: Not found.",
repo_name,
clean_url,
downloaded_clean_file.name,
)
try:
logging.info(
"[%r] Started to download %r into %r",
repo_name,
ip_url,
downloaded_ip_file.name,
)
DownloadHelper(ip_url).download_text(destination=downloaded_ip_file.name)
logging.info(
"[%r] Finished to download %r into %r",
repo_name,
ip_url,
downloaded_ip_file.name,
)
ip_found = True
except UnableToDownload:
logging.critical(
"[%r] Could not download %r into %r. Reason: Not found.",
repo_name,
ip_url,
downloaded_ip_file.name,
)
try:
logging.info(
"[%r] Started to download %r into %r",
repo_name,
whitelisted_url,
downloaded_whitelisted_file.name,
)
DownloadHelper(whitelisted_url).download_text(
destination=downloaded_whitelisted_file.name
)
logging.info(
"[%r] Finished to download %r into %r",
repo_name,
whitelisted_url,
downloaded_whitelisted_file.name,
)
whitelisted_found = True
except UnableToDownload:
logging.critical(
"[%r] Could not download %r into %r. Reason: Not found.",
repo_name,
whitelisted_url,
downloaded_whitelisted_file.name,
)
downloaded_domain_file.seek(0)
downloaded_clean_file.seek(0)
downloaded_ip_file.seek(0)
downloaded_whitelisted_file.seek(0)
if whitelisted_found:
domain_file_to_read = (
domain_file_to_deliver
) = downloaded_whitelisted_file.name
elif clean_found:
domain_file_to_read = domain_file_to_deliver = downloaded_clean_file.name
elif domain_found:
domain_file_to_read = domain_file_to_deliver = downloaded_domain_file.name
else:
domain_file_to_read = domain_file_to_deliver = None
if ip_found:
ip_file_to_read = ip_file_to_deliver = downloaded_ip_file.name
else:
ip_file_to_read = ip_file_to_deliver = None
logging.info(
"[%r] Using %r as (domain) file to read and deliver.",
repo_name,
domain_file_to_read,
)
logging.info(
"[%r] Using %r as (ip) file to read and deliver.",
repo_name,
domain_file_to_read,
)
if domain_file_to_read:
logging.info(
"[%r] Starting to whitelist content of %r",
repo_name,
domain_file_to_read,
)
WhitelistCore(
output_file=domain_file_to_read,
use_official=True,
).filter(file=domain_file_to_read, already_formatted=True)
logging.info(
"[%r] Finished to whitelist content of %r",
repo_name,
domain_file_to_read,
)
logging.info(
"[%r] Starting to filter content of %r", repo_name, domain_file_to_read
)
with open(domain_file_to_read, "r", encoding="utf-8") as file_stream:
for line in file_stream:
if not line.strip():
continue
if DomainSyntaxChecker(line.strip()).is_valid():
output_domain_file.write(line)
elif IPSyntaxChecker(line.strip()).is_valid():
output_ip_file.write(line)
logging.info(
"[%r] Finished to filter content of %r", repo_name, domain_file_to_read
)
if ip_file_to_read:
logging.info(
"[%r] Starting to whitelist content of %r", repo_name, ip_file_to_read
)
WhitelistCore(
output_file=ip_file_to_read,
use_official=True,
).filter(file=ip_file_to_read, already_formatted=True)
logging.info(
"[%r] Finished to whitelist content of %r", repo_name, ip_file_to_read
)
logging.info(
"[%r] Starting to filter content of %r", repo_name, ip_file_to_read
)
with open(ip_file_to_read, "r", encoding="utf-8") as file_stream:
for line in file_stream:
if not line.strip():
continue
if DomainSyntaxChecker(line.strip()).is_valid():
output_domain_file.write(line)
elif IPSyntaxChecker(line.strip()).is_valid():
output_ip_file.write(line)
logging.info(
"[%r] Finished to filter content of %r", repo_name, ip_file_to_read
)
downloaded_ip_file.close()
downloaded_domain_file.close()
downloaded_clean_file.close()
downloaded_whitelisted_file.close()
if downloaded_ip_file.name != ip_file_to_deliver:
FileHelper(downloaded_ip_file.name).delete()
if downloaded_domain_file.name != domain_file_to_deliver:
FileHelper(downloaded_domain_file.name).delete()
if downloaded_whitelisted_file.name != domain_file_to_deliver:
FileHelper(downloaded_whitelisted_file.name).delete()
if downloaded_clean_file.name != domain_file_to_deliver:
FileHelper(downloaded_clean_file.name).delete()
output_domain_file.seek(0)
output_ip_file.seek(0)
output_domain_file.seek(0)
output_ip_file.seek(0)
return output_domain_file.name, output_ip_file.name
