def __init__( self, subject: Optional[str] = None, do_syntax_check_first: Optional[bool] = None, db_session: Optional[Session] = None, use_collection: Optional[bool] = None, ) -> None: self.dns_query_tool = DNSQueryTool() self.ipv4_reputation_query_tool = IPV4ReputationDataset() self.domain_syntax_checker = DomainSyntaxChecker() self.ip_syntax_checker = IPSyntaxChecker() self.url_syntax_checker = URLSyntaxChecker() self.params = ReputationCheckerParams() self.status = ReputationCheckerStatus() self.status.params = self.params self.status.dns_lookup_record = self.dns_query_tool.lookup_record super().__init__( subject, do_syntax_check_first=do_syntax_check_first, db_session=db_session, use_collection=use_collection, )
def test_is_not_valid(self) -> None: """ Tests the method which let us check if the given subject is valid for the case that the given subject is not valid. """ url_checker = URLSyntaxChecker() expected = False for subject in pyf_test_dataset.NOT_VALID_DOMAINS: url_checker.subject = f"{subject}/?is_admin=true" actual = url_checker.is_valid() self.assertEqual(expected, actual, subject)
def test_is_valid_subdomain(self) -> None: """ Tests the method which let us check if the given subject is valid for the case that a subdomain is given. """ url_checker = URLSyntaxChecker() expected = True for subject in pyf_test_dataset.VALID_SUBDOMAINS: url_checker.subject = f"https://{subject}/?is_admin=true" actual = url_checker.is_valid() self.assertEqual(expected, actual, subject)
def test_is_not_valid_not_rfc_compliant(self) -> None: """ Tests the method which let us check if the given subject is valid for the case that the given subject is not RFC compliant. """ expected = False given = "http://example.hello_world.org" actual = URLSyntaxChecker(given).is_valid() self.assertEqual(expected, actual)
def test_is_not_valid_not_extension(self) -> None: """ Tests the method which let us check if the given subject is valid for the case that the given subject has no valid extension. """ expected = False given = "http://example" actual = URLSyntaxChecker(given).is_valid() self.assertEqual(expected, actual)
def download_file(file: str, destination: str) -> bool: """ Downloads the given file (if it's an URL). :param file: The file to download. :param destination. The file to write. :return: A boolean which represent the action state. """ if URLSyntaxChecker(file).is_valid(): DownloadHelper(file).download_text(destination=destination) return True return False
def is_url(subject: str, **kwargs) -> bool: """ Checks if the given subject is syntactically a valid URL. .. warning:: This method may be removed in the future. It is still available for convenience. Please consider the following alternative example: :: from PyFunceble import DomainReputationChecker my_subject = "https://example.org" the_status = URLSyntaxChecker( my_subject ).get_status() # Get the status in dict format. print(the_status.to_dict()) # Get the status in json format. print(the_status.to_json()) # Check if it is a URL. print(f"{my_subject} is URL ? {the_status.is_valid()}") :parma subject: The subject to check. """ warnings.warn( "PyFunceble.is_url may be removed in the future." "Please consider using PyFunceble.URLSyntaxChecker explicitly.", DeprecationWarning, ) # pylint: disable=no-member return URLSyntaxChecker(subject, **kwargs).get_status().is_valid()
class ReputationCheckerBase(CheckerBase): """ Provides the base of all our reputation checker classes. :param str subject: Optional, The subject to work with. :param bool do_syntax_check_first: Optional, Activates/Disables the check of the status before the actual status gathering. """ dns_query_tool: Optional[DNSQueryTool] = None ipv4_reputation_query_tool: Optional[IPV4ReputationDataset] = None domain_syntax_checker: Optional[DomainSyntaxChecker] = None ip_syntax_checker: Optional[IPSyntaxChecker] = None url_syntax_checker: Optional[URLSyntaxChecker] = None status: Optional[ReputationCheckerStatus] = None params: Optional[ReputationCheckerParams] = None def __init__( self, subject: Optional[str] = None, do_syntax_check_first: Optional[bool] = None, db_session: Optional[Session] = None, use_collection: Optional[bool] = None, ) -> None: self.dns_query_tool = DNSQueryTool() self.ipv4_reputation_query_tool = IPV4ReputationDataset() self.domain_syntax_checker = DomainSyntaxChecker() self.ip_syntax_checker = IPSyntaxChecker() self.url_syntax_checker = URLSyntaxChecker() self.params = ReputationCheckerParams() self.status = ReputationCheckerStatus() self.status.params = self.params self.status.dns_lookup_record = self.dns_query_tool.lookup_record super().__init__( subject, do_syntax_check_first=do_syntax_check_first, db_session=db_session, use_collection=use_collection, ) @staticmethod def is_valid() -> bool: raise NotImplementedError() def subject_propagator(self) -> "CheckerBase": """ Propagate the currently set subject. .. warning:: You are not invited to run this method directly. """ self.dns_query_tool.set_subject(self.idna_subject) self.domain_syntax_checker.subject = self.idna_subject self.ip_syntax_checker.subject = self.idna_subject self.url_syntax_checker.subject = self.idna_subject self.status = ReputationCheckerStatus() self.status.params = self.params self.status.dns_lookup_record = self.dns_query_tool.lookup_record self.status.subject = self.subject self.status.idna_subject = self.idna_subject self.query_syntax_checker() return self def should_we_continue_test(self, status_post_syntax_checker: str) -> bool: """ Checks if we are allowed to continue a standard testing. """ return bool( not self.status.status or status_post_syntax_checker == PyFunceble.storage.STATUS.invalid) def query_syntax_checker(self) -> "ReputationCheckerBase": """ Queries the syntax checker. """ self.status.second_level_domain_syntax = ( self.domain_syntax_checker.is_valid_second_level()) self.status.subdomain_syntax = self.domain_syntax_checker.is_valid_subdomain( ) self.status.domain_syntax = bool( self.status.subdomain_syntax or self.status.second_level_domain_syntax) self.status.ipv4_syntax = self.ip_syntax_checker.is_valid_v4() self.status.ipv6_syntax = self.ip_syntax_checker.is_valid_v6() self.status.ipv4_range_syntax = self.ip_syntax_checker.is_valid_v4_range( ) self.status.ipv6_range_syntax = self.ip_syntax_checker.is_valid_v6_range( ) self.status.ip_syntax = bool(self.status.ipv4_syntax or self.status.ipv6_syntax) self.status.url_syntax = self.url_syntax_checker.is_valid() return self def query_a_record(self) -> Optional[List[str]]: """ Queries all the A record. """ raise NotImplementedError() def try_to_query_status_from_dns_lookup(self) -> "ReputationCheckerBase": """ Tries to query the status from the DNS lookup. """ PyFunceble.facility.Logger.info( "Started to try to query the status of %r from: DNS Lookup", self.status.idna_subject, ) if not self.status.ipv4_syntax: lookup_result = self.query_a_record() self.status.dns_lookup = lookup_result else: lookup_result = [self.status.subject] if lookup_result: for subject in lookup_result: if subject in self.ipv4_reputation_query_tool: self.status.status = PyFunceble.storage.STATUS.malicious self.status.status_source = "REPUTATION" PyFunceble.facility.Logger.info( "Could define the status of %r from: DNS Lookup", self.status.idna_subject, ) break PyFunceble.facility.Logger.info( "Finished to try to query the status of %r from: DNS Lookup", self.status.idna_subject, ) return self def try_to_query_status_from_syntax_lookup( self) -> "ReputationCheckerBase": """ Tries to query the status from the syntax. """ PyFunceble.facility.Logger.info( "Started to try to query the status of %r from: Syntax Lookup", self.status.idna_subject, ) if (not self.status.domain_syntax and not self.status.ip_syntax and not self.status.url_syntax): self.status.status = PyFunceble.storage.STATUS.invalid self.status.status_source = "SYNTAX" PyFunceble.facility.Logger.info( "Could define the status of %r from: Syntax Lookup", self.status.idna_subject, ) PyFunceble.facility.Logger.info( "Finished to try to query the status of %r from: Syntax Lookup", self.status.idna_subject, ) return self def try_to_query_status_from_collection(self) -> "ReputationCheckerBase": """ Tries to get and set the status from the Collection API. """ PyFunceble.facility.Logger.info( "Started to try to query the status of %r from: Collection Lookup", self.status.idna_subject, ) data = self.collection_query_tool[self.idna_subject] if data and "status" in data: if (self.collection_query_tool.preferred_status_origin == "frequent" and data["status"]["reputation"]["frequent"]): self.status.status = data["status"]["reputation"]["frequent"] self.status.status_source = "COLLECTION" elif (self.collection_query_tool.preferred_status_origin == "latest" and data["status"]["reputation"]["latest"]): self.status.status = data["status"]["reputation"]["latest"][ "status"] self.status.status_source = "COLLECTION" elif (self.collection_query_tool.preferred_status_origin == "recommended" and data["status"]["reputation"]["recommended"]): self.status.status = data["status"]["reputation"][ "recommended"] self.status.status_source = "COLLECTION" PyFunceble.facility.Logger.info( "Could define the status of %r from: Collection Lookup", self.status.idna_subject, ) PyFunceble.facility.Logger.info( "Finished to try to query the status of %r from: Collection Lookup", self.status.idna_subject, ) @CheckerBase.ensure_subject_is_given @CheckerBase.update_status_date_after_query def query_status(self) -> "ReputationCheckerBase": """ Queries the status and for for more action. """ status_post_syntax_checker = None if (not self.status.status and self.use_collection): # pragma: no cover ## Underlying tested self.try_to_query_status_from_collection() if not self.status.status and self.do_syntax_check_first: self.try_to_query_status_from_syntax_lookup() if self.status.status: status_post_syntax_checker = self.status.status if self.should_we_continue_test(status_post_syntax_checker): self.try_to_query_status_from_dns_lookup() if not self.status.status: self.status.status = PyFunceble.storage.STATUS.sane self.status.status_source = "REPUTATION" PyFunceble.facility.Logger.info( "Could not define the status of %r. Setting to %r", self.status.idna_subject, self.status.status, ) return self # pylint: disable=useless-super-delegation def get_status(self) -> Optional[ReputationCheckerStatus]: return super().get_status()
class Nameservers: """ Provides an interface to get the right nameserver to communicate with. """ nameservers: Optional[List[str]] = None nameserver_ports: Optional[dict] = None protocol: Optional[str] = None domain_syntax_checker: DomainSyntaxChecker = DomainSyntaxChecker() url_syntax_checker: URLSyntaxChecker = URLSyntaxChecker() url2netloc: Url2Netloc = Url2Netloc() def __init__(self, nameserver: Optional[List[str]] = None, protocol: str = "TCP") -> None: self.protocol = protocol if nameserver is not None: self.set_nameservers(nameserver) @staticmethod def split_nameserver_from_port( nameserver: str, *, default_port: int = 53, ) -> Tuple[str, int]: """ Splits the nameserver from its port.re :param nameserver: The nameserver to work with. :param default_port: The default port to apply, if none is found. """ if ":" in nameserver: splitted = nameserver.rsplit(":") if splitted[-1].isdigit(): return ":".join(splitted[:-1]), int(splitted[-1]) return ":".join(splitted), default_port return nameserver, default_port @classmethod def get_ip_from_nameserver(cls, nameserver: str) -> List[str]: """ Given a nameserver, this method resolve it in order to get the IP to contact. :param nameserver: The name to resolve. """ PyFunceble.facility.Logger.info( "Started to get ip from nameserver (%r)", nameserver) result = [] if cls.domain_syntax_checker.set_subject(nameserver).is_valid(): try: result.extend([ x.address for x in dns.resolver.Resolver().resolve(nameserver, "A") ]) except dns.exception.DNSException: pass try: result.extend([ x.address for x in dns.resolver.Resolver().resolve( nameserver, "AAAA") ]) except dns.exception.DNSException: pass else: result.append(nameserver) PyFunceble.facility.Logger.debug("IP from nameserver (%r):\n%r", nameserver, result) PyFunceble.facility.Logger.info( "Finished to get ip from nameserver (%r)", nameserver) return result def set_nameservers(self, value: List[str]) -> "Nameservers": """ Sets the nameserver to use. Side Effect: Also updates the :code:`nameserver_ports` variable. :raise TypeError: When the given :code:`value` is not a :py:class:`list`. :raise ValueError: When the given :code:`value` is emtpy. """ if not isinstance(value, list): raise TypeError(f"<value> should be {list}, {type(value)} given.") if not value: raise ValueError("<value> should not be empty.") self.nameserver_ports = {} self.nameservers = [] for nameserver in value: if self.protocol.lower() == "https": if not nameserver.startswith("https://"): netloc = self.url2netloc.set_data_to_convert( nameserver).get_converted() if "/" in nameserver: path = nameserver[nameserver.find("/"):] else: path = "" self.nameservers.append("https://" # pylint: disable=line-too-long f"{netloc}{path}") else: self.nameservers.append(nameserver) # 443 is because it's more likely to be for DOH. self.nameserver_ports.update({self.nameservers[-1]: 443}) continue server, port = self.split_nameserver_from_port(nameserver) for dns_ip in self.get_ip_from_nameserver(server): self.nameservers.append(dns_ip) self.nameserver_ports.update({dns_ip: port}) return self def get_nameservers(self) -> Optional[List[str]]: """ Provides the currently set nameservers. """ return self.nameservers def get_nameserver_ports(self) -> Optional[dict]: """ Provides the currently set nameserver_ports. """ return self.nameserver_ports def guess_and_set_nameservers(self) -> "Nameservers": """ Try to guess and set the nameserver to use. """ if PyFunceble.facility.ConfigLoader.is_already_loaded(): if PyFunceble.storage.CONFIGURATION.dns.server: if isinstance(PyFunceble.storage.CONFIGURATION.dns.server, list): self.set_nameservers( PyFunceble.storage.CONFIGURATION.dns.server) else: self.set_nameservers( [PyFunceble.storage.CONFIGURATION.dns.server]) else: # pragma: no cover ## Well, I don't like playing with the default resolver. self.set_nameservers( dns.resolver.get_default_resolver().nameservers) else: # pragma: no cover ## Well, I don't like playing with the default resolver. self.set_nameservers( dns.resolver.get_default_resolver().nameservers) return self def guess_all_settings( self, ) -> "Nameservers": # pragma: no cover ## Method themselves are more important """ Try to guess all settings. """ to_ignore = ["guess_all_settings"] for method in dir(self): if method in to_ignore or not method.startswith("guess_"): continue getattr(self, method)() return self
class AvailabilityCheckerBase(CheckerBase): """ Provides the base of all our availability checker classes. :param str subject: Optional, The subject to work with. :param bool use_extra_rules: Optional, Activates/Disables the usage of our own set of extra rules. :param bool use_whois_lookup: Optional, Activates/Disables the usage of the WHOIS lookup to gather the status of the given :code:`subject`. :param bool use_dns_lookup: Optional, Activates/Disables the usage of the DNS lookup to gather the status of the given :code:`subject`. :param bool use_netinfo_lookup: Optional, Activates/Disables the usage of the network information lookup module to gather the status of the given :code:`subject`. :param bool use_http_code_lookup: Optional, Activates/Disables the usage of the HTTP status code lookup to gather the status of the given :code:`subject`. :param bool use_reputation_lookup: Optional, Activates/Disables the usage of the reputation dataset lookup to gather the status of the given :code:`subject`. :param bool do_syntax_check_first: Optional, Activates/Disables the check of the status before the actual status gathering. :param bool use_whois_db: Optional, Activates/Disable the usage of a local database to store the WHOIS datasets. """ # pylint: disable=too-many-public-methods, too-many-instance-attributes STD_USE_EXTRA_RULES: bool = True STD_USE_WHOIS_LOOKUP: bool = True STD_USE_DNS_LOOKUP: bool = True STD_USE_NETINFO_LOOKUP: bool = True STD_USE_HTTP_CODE_LOOKUP: bool = True STD_USE_REPUTATION_LOOKUP: bool = False STD_USE_WHOIS_DB: bool = True dns_query_tool: Optional[DNSQueryTool] = None whois_query_tool: Optional[WhoisQueryTool] = None addressinfo_query_tool: Optional[AddressInfo] = None hostbyaddr_query_tool: Optional[HostByAddrInfo] = None http_status_code_query_tool: Optional[HTTPStatusCode] = None domain_syntax_checker: Optional[DomainSyntaxChecker] = None ip_syntax_checker: Optional[IPSyntaxChecker] = None url_syntax_checker: Optional[URLSyntaxChecker] = None extra_rules_handler: Optional[ExtraRulesHandler] = None _use_extra_rules: bool = False _use_whois_lookup: bool = False _use_dns_lookup: bool = False _use_netinfo_lookup: bool = False _use_http_code_lookup: bool = False _use_reputation_lookup: bool = False _use_whois_db: bool = False status: Optional[AvailabilityCheckerStatus] = None params: Optional[AvailabilityCheckerParams] = None def __init__( self, subject: Optional[str] = None, *, use_extra_rules: Optional[bool] = None, use_whois_lookup: Optional[bool] = None, use_dns_lookup: Optional[bool] = None, use_netinfo_lookup: Optional[bool] = None, use_http_code_lookup: Optional[bool] = None, use_reputation_lookup: Optional[bool] = None, do_syntax_check_first: Optional[bool] = None, db_session: Optional[Session] = None, use_whois_db: Optional[bool] = None, use_collection: Optional[bool] = None, ) -> None: self.dns_query_tool = DNSQueryTool() self.whois_query_tool = WhoisQueryTool() self.addressinfo_query_tool = AddressInfo() self.hostbyaddr_query_tool = HostByAddrInfo() self.http_status_code_query_tool = HTTPStatusCode() self.domain_syntax_checker = DomainSyntaxChecker() self.ip_syntax_checker = IPSyntaxChecker() self.url_syntax_checker = URLSyntaxChecker() self.extra_rules_handler = ExtraRulesHandler() self.db_session = db_session self.params = AvailabilityCheckerParams() self.status = AvailabilityCheckerStatus() self.status.params = self.params self.status.dns_lookup_record = self.dns_query_tool.lookup_record self.status.whois_lookup_record = self.whois_query_tool.lookup_record if use_extra_rules is not None: self.use_extra_rules = use_extra_rules else: self.guess_and_set_use_extra_rules() if use_whois_lookup is not None: self.use_whois_lookup = use_whois_lookup else: self.guess_and_set_use_whois_lookup() if use_dns_lookup is not None: self.use_dns_lookup = use_dns_lookup else: self.guess_and_set_dns_lookup() if use_netinfo_lookup is not None: self.use_netinfo_lookup = use_netinfo_lookup else: self.guess_and_set_use_netinfo_lookup() if use_http_code_lookup is not None: self.use_http_code_lookup = use_http_code_lookup else: self.guess_and_set_use_http_code_lookup() if use_reputation_lookup is not None: self.use_reputation_lookup = use_reputation_lookup else: self.guess_and_set_use_reputation_lookup() if use_whois_db is not None: self.use_whois_db = use_whois_db else: self.guess_and_set_use_whois_db() super().__init__( subject, do_syntax_check_first=do_syntax_check_first, db_session=db_session, use_collection=use_collection, ) @property def use_extra_rules(self) -> bool: """ Provides the current value of the :code:`_use_extra_rules` attribute. """ return self._use_extra_rules @use_extra_rules.setter def use_extra_rules(self, value: bool) -> None: """ Sets the value which authorizes the usage of the special rule. :param value: The value to set. :raise TypeError: When the given :code:`value` is not a :py:class:`bool`. """ if not isinstance(value, bool): raise TypeError(f"<value> should be {bool}, {type(value)} given.") self._use_extra_rules = self.params.use_extra_rules = value def set_use_extra_rules(self, value: bool) -> "AvailabilityCheckerBase": """ Sets the value which authorizes the usage of the special rule. :param value: The value to set. """ self.use_extra_rules = value return self @property def use_whois_lookup(self) -> bool: """ Provides the current value of the :code:`_use_whois_lookup` attribute. """ return self._use_whois_lookup @use_whois_lookup.setter def use_whois_lookup(self, value: bool) -> None: """ Sets the value which authorizes the usage of the WHOIS lookup. :param value: The value to set. :raise TypeError: When the given :code:`value` is not a :py:class:`bool`. """ if not isinstance(value, bool): raise TypeError(f"<value> should be {bool}, {type(value)} given.") self._use_whois_lookup = self.params.use_whois_lookup = value def set_use_whois_lookup(self, value: bool) -> "AvailabilityCheckerBase": """ Sets the value which authorizes the usage of the WHOIS lookup. :param value: The value to set. """ self.use_whois_lookup = value return self @property def use_dns_lookup(self) -> bool: """ Provides the current value of the :code:`_use_dns_lookup` attribute. """ return self._use_dns_lookup @use_dns_lookup.setter def use_dns_lookup(self, value: bool) -> None: """ Sets the value which authorizes the usage of the DNS Lookup. :param value: The value to set. :raise TypeError: When the given :code:`value` is not a :py:class:`bool`. """ if not isinstance(value, bool): raise TypeError(f"<value> should be {bool}, {type(value)} given.") self._use_dns_lookup = self.params.use_dns_lookup = value def set_use_dns_lookup(self, value: bool) -> "AvailabilityCheckerBase": """ Sets the value which authorizes the usage of the DNS Lookup. :param value: The value to set. """ self.use_dns_lookup = value return self @property def use_netinfo_lookup(self) -> bool: """ Provides the current value of the :code:`_use_netinfo_lookup` attribute. """ return self._use_netinfo_lookup @use_netinfo_lookup.setter def use_netinfo_lookup(self, value: bool) -> None: """ Sets the value which authorizes the usage of the network information lookup. :param value: The value to set. :raise TypeError: When the given :code:`value` is not a :py:class:`bool`. """ if not isinstance(value, bool): raise TypeError(f"<value> should be {bool}, {type(value)} given.") self._use_netinfo_lookup = self.params.use_netinfo_lookup = value def set_use_netinfo_lookup(self, value: bool) -> "AvailabilityCheckerBase": """ Sets the value which authorizes the usage of the network information lookup. :param value: The value to set. """ self.use_netinfo_lookup = value return self @property def use_http_code_lookup(self) -> None: """ Provides the current value of the :code:`_use_http_code_lookup` attribute. """ return self._use_http_code_lookup @use_http_code_lookup.setter def use_http_code_lookup(self, value: bool) -> None: """ Sets the value which authorizes the usage of the HTTP status code lookup. :param value: The value to set. :raise TypeError: When the given :code:`value` is not a :py:class:`bool`. """ if not isinstance(value, bool): raise TypeError(f"<value> should be {bool}, {type(value)} given.") self._use_http_code_lookup = self.params.use_http_code_lookup = value def set_use_http_code_lookup(self, value: bool) -> "AvailabilityCheckerBase": """ Sets the value which authorizes the usage of the HTTP status code lookup. :param value: The value to set. """ self.use_http_code_lookup = value return self @property def use_reputation_lookup(self) -> bool: """ Provides the current value of the :code:`_use_reputation_lookup` attribute. """ return self._use_reputation_lookup @use_reputation_lookup.setter def use_reputation_lookup(self, value: bool) -> None: """ Sets the value which authorizes the usage of the reputation lookup. :param value: The value to set. :raise TypeError: When the given :code:`value` is not a :py:class:`bool`. """ if not isinstance(value, bool): raise TypeError(f"<value> should be {bool}, {type(value)} given.") self._use_reputation_lookup = self.params.use_reputation_lookup = value def set_use_reputation_lookup(self, value: bool) -> "AvailabilityCheckerBase": """ Sets the value which authorizes the usage of the reputation lookup. :param value: The value to set. """ self.use_reputation_lookup = value return self @property def use_whois_db(self) -> bool: """ Provides the current value of the :code:`_use_whois_db` attribute. """ return self._use_whois_db @use_whois_db.setter def use_whois_db(self, value: bool) -> None: """ Sets the value which authorizes the usage of the WHOIS DB. :param value: The value to set. :param TypeError: When the given :code:`use_whois_db` is not a :py:class:`bool`. """ if not isinstance(value, bool): raise TypeError(f"<value> should be {bool}, {type(value)} given.") self._use_whois_db = self.params.use_whois_db = value def set_use_whois_db(self, value: bool) -> "AvailabilityCheckerBase": """ Sets the value which authorizes the usage of the WHOIS DB. :param value: The value to set. """ self.use_whois_db = value return self def subject_propagator(self) -> "CheckerBase": """ Propagate the currently set subject. .. warning:: You are not invited to run this method directly. """ self.dns_query_tool.set_subject(self.idna_subject) self.whois_query_tool.set_subject(self.idna_subject) self.addressinfo_query_tool.set_subject(self.idna_subject) self.hostbyaddr_query_tool.set_subject(self.idna_subject) self.http_status_code_query_tool.set_subject(self.idna_subject) self.domain_syntax_checker.subject = self.idna_subject self.ip_syntax_checker.subject = self.idna_subject self.url_syntax_checker.subject = self.idna_subject self.status = AvailabilityCheckerStatus() self.status.params = self.params self.status.dns_lookup_record = self.dns_query_tool.lookup_record self.status.whois_lookup_record = self.whois_query_tool.lookup_record self.status.subject = self.subject self.status.idna_subject = self.idna_subject self.status.status = None self.query_syntax_checker() return self def should_we_continue_test(self, status_post_syntax_checker: str) -> bool: """ Checks if we are allowed to continue a standard testing. """ return bool( not self.status.status or status_post_syntax_checker == PyFunceble.storage.STATUS.invalid) def guess_and_set_use_extra_rules(self) -> "AvailabilityCheckerBase": """ Try to guess and set the value of the :code:`use_extra_rules` attribute from the configuration file. """ if PyFunceble.facility.ConfigLoader.is_already_loaded(): self.use_extra_rules = PyFunceble.storage.CONFIGURATION.lookup.special else: self.use_extra_rules = self.STD_USE_EXTRA_RULES return self def guess_and_set_use_whois_lookup(self) -> "AvailabilityCheckerBase": """ Try to guess and set the value of the :code:`use_whois` attribute from the configuration file. """ if PyFunceble.facility.ConfigLoader.is_already_loaded(): self.use_whois_lookup = PyFunceble.storage.CONFIGURATION.lookup.whois else: self.use_whois_lookup = self.STD_USE_WHOIS_LOOKUP return self def guess_and_set_dns_lookup(self) -> "AvailabilityCheckerBase": """ Try to guess and set the value of the :code:`use_dns_lookup` attribute from the configuration file. """ if PyFunceble.facility.ConfigLoader.is_already_loaded(): self.use_dns_lookup = PyFunceble.storage.CONFIGURATION.lookup.dns else: self.use_dns_lookup = self.STD_USE_DNS_LOOKUP return self def guess_and_set_use_netinfo_lookup(self) -> "AvailabilityCheckerBase": """ Try to guess and set the value of the :code:`use_netinfo_lookup` attribute from the configuration file. """ if PyFunceble.facility.ConfigLoader.is_already_loaded(): self.use_netinfo_lookup = PyFunceble.storage.CONFIGURATION.lookup.netinfo else: self.use_netinfo_lookup = self.STD_USE_NETINFO_LOOKUP return self def guess_and_set_use_http_code_lookup(self) -> "AvailabilityCheckerBase": """ Try to guess and set the value of the :code:`use_http_code_lookup` attribute from the configuration file. """ if PyFunceble.facility.ConfigLoader.is_already_loaded(): self.use_http_code_lookup = ( PyFunceble.storage.CONFIGURATION.lookup.http_status_code) else: self.use_http_code_lookup = self.STD_USE_HTTP_CODE_LOOKUP return self def guess_and_set_use_reputation_lookup(self) -> "AvailabilityCheckerBase": """ Try to guess and set the value of the :code:`use_reputation_lookup` attribute from the configuration file. """ if PyFunceble.facility.ConfigLoader.is_already_loaded(): self.use_reputation_lookup = ( PyFunceble.storage.CONFIGURATION.lookup.reputation) else: self.use_reputation_lookup = self.STD_USE_REPUTATION_LOOKUP return self def guess_and_set_use_whois_db(self) -> "AvailabilityCheckerBase": """ Try to guess and set the value of the :code:`use_whois_db` attribute. """ if PyFunceble.facility.ConfigLoader.is_already_loaded(): self.use_whois_db = PyFunceble.storage.CONFIGURATION.cli_testing.whois_db else: self.use_whois_db = self.STD_USE_WHOIS_DB def guess_all_settings( self, ) -> "AvailabilityCheckerBase": # pragma: no cover ## Method are more important """ Try to guess all settings. """ to_ignore = ["guess_all_settings"] for method in dir(self): if method in to_ignore or not method.startswith("guess_"): continue getattr(self, method)() return self def query_syntax_checker(self) -> "AvailabilityCheckerBase": """ Queries the syntax checker. """ PyFunceble.facility.Logger.info("Started to check the syntax of %r", self.status.idna_subject) self.status.second_level_domain_syntax = ( self.domain_syntax_checker.is_valid_second_level()) self.status.subdomain_syntax = self.domain_syntax_checker.is_valid_subdomain( ) self.status.domain_syntax = bool(self.status.subdomain_syntax) or bool( self.status.second_level_domain_syntax) self.status.ipv4_syntax = self.ip_syntax_checker.is_valid_v4() self.status.ipv6_syntax = self.ip_syntax_checker.is_valid_v6() self.status.ipv4_range_syntax = self.ip_syntax_checker.is_valid_v4_range( ) self.status.ipv6_range_syntax = self.ip_syntax_checker.is_valid_v6_range( ) self.status.ip_syntax = bool(self.status.ipv4_syntax or self.status.ipv6_syntax) self.status.url_syntax = self.url_syntax_checker.is_valid() PyFunceble.facility.Logger.info("Finished to check the syntax of %r", self.status.idna_subject) return self @CheckerBase.ensure_subject_is_given def query_dns_record(self) -> Optional[Dict[str, Optional[List[str]]]]: """ Tries to query the DNS record(s) of the given subject. """ PyFunceble.facility.Logger.info( "Started to try to query the DNS record of %r.", self.status.idna_subject, ) result = {} if self.status.subdomain_syntax: lookup_order = ["NS", "A", "AAAA", "CNAME", "DNAME"] elif self.status.domain_syntax: lookup_order = ["NS", "CNAME", "A", "AAAA", "DNAME"] elif self.status.ip_syntax: lookup_order = ["PTR"] else: lookup_order = [] if lookup_order: for record_type in lookup_order: local_result = self.dns_query_tool.set_query_record_type( record_type).query() if local_result: result[record_type] = local_result break PyFunceble.facility.Logger.debug("DNS Record:\n%r", result) PyFunceble.facility.Logger.info( "Finished to try to query the DNS record of %r", self.status.idna_subject, ) return result def try_to_query_status_from_whois(self, ) -> "AvailabilityCheckerBase": """ Tries to get and the status from the WHOIS record. .. warning:: If the configuration is loaded, this method try to query from the best database first. If it's not found it will try to query it from the best WHOIS server then add it into the database (if the expiration date extraction is successful). .. note:: The addition into the WHOIS database is only done if this method is running in a process with a name that does not starts with :code:`PyFunceble` (case sensitive). """ PyFunceble.facility.Logger.info( "Started to try to query the status of %r from: WHOIS Lookup", self.status.idna_subject, ) if ( PyFunceble.facility.ConfigLoader.is_already_loaded() and self.use_whois_db ): # pragma: no cover ## Not interesting enough to spend time on it. whois_object = PyFunceble.checker.utils.whois.get_whois_dataset_object( db_session=self.db_session) known_record = whois_object[self.subject] if known_record and not isinstance(known_record, dict): # Comes from DB engine. known_record = known_record.to_dict() if not known_record: # We assume that expired dataset are never saved into the # dataset. self.status.expiration_date = ( self.whois_query_tool.get_expiration_date()) self.status.whois_record = self.whois_query_tool.lookup_record.record if (self.status.expiration_date and not multiprocessing.current_process().name.startswith( PyFunceble.storage.PROJECT_NAME.lower())): whois_object.update({ "subject": self.subject, "idna_subject": self.idna_subject, "expiration_date": self.status.expiration_date, "epoch": str( datetime.strptime(self.status.expiration_date, "%d-%b-%Y").timestamp()), }) else: self.status.expiration_date = known_record["expiration_date"] self.status.whois_record = None else: self.status.expiration_date = self.whois_query_tool.get_expiration_date( ) self.status.whois_record = self.whois_query_tool.lookup_record.record if self.status.expiration_date: self.status.status = PyFunceble.storage.STATUS.up self.status.status_source = "WHOIS" PyFunceble.facility.Logger.info( "Could define the status of %r from: WHOIS Lookup", self.status.idna_subject, ) PyFunceble.facility.Logger.info( "Finished to try to query the status of %r from: WHOIS Lookup", self.status.idna_subject, ) return self def try_to_query_status_from_dns(self) -> "AvailabilityCheckerBase": """ Tries to query the status from the DNS lookup. """ PyFunceble.facility.Logger.info( "Started to try to query the status of %r from: DNS Lookup", self.status.idna_subject, ) lookup_result = self.query_dns_record() if lookup_result: self.status.dns_lookup = lookup_result self.status.status = PyFunceble.storage.STATUS.up self.status.status_source = "DNSLOOKUP" PyFunceble.facility.Logger.info( "Could define the status of %r from: DNS Lookup", self.status.idna_subject, ) PyFunceble.facility.Logger.info( "Finished to try to query the status of %r from: DNS Lookup", self.status.idna_subject, ) return self def try_to_query_status_from_netinfo(self) -> "AvailabilityCheckerBase": """ Tries to query the status from the network information. """ PyFunceble.facility.Logger.info( "Started to try to query the status of %r from: NETINFO Lookup", self.status.idna_subject, ) if self.status.domain_syntax: lookup_result = self.addressinfo_query_tool.get_info() elif self.status.ip_syntax: lookup_result = self.hostbyaddr_query_tool.get_info() elif self.status.idna_subject.isdigit(): lookup_result = None else: lookup_result = self.addressinfo_query_tool.get_info() if lookup_result: self.status.netinfo = lookup_result self.status.status = PyFunceble.storage.STATUS.up self.status.status_source = "NETINFO" PyFunceble.facility.Logger.info( "Could define the status of %r from: NETINFO Lookup", self.status.idna_subject, ) PyFunceble.facility.Logger.info( "Finished to try to query the status of %r from: NETINFO Lookup", self.status.idna_subject, ) return self def try_to_query_status_from_http_status_code( self) -> "AvailabilityCheckerBase": """ Tries to query the status from the HTTP status code. """ PyFunceble.facility.Logger.info( "Started to try to query the status of %r from: HTTP Status code Lookup", self.status.idna_subject, ) if not self.status.url_syntax and not RegexHelper("[^a-z0-9._]").match( self.idna_subject, return_match=False): # The regex is there because while testing for domain, sometime we # may see something like mailto:[email protected] self.http_status_code_query_tool.set_subject( f"http://{self.idna_subject}:80") lookup_result = self.http_status_code_query_tool.get_status_code() if (lookup_result and lookup_result != self.http_status_code_query_tool.STD_UNKNOWN_STATUS_CODE): self.status.http_status_code = lookup_result if (PyFunceble.facility.ConfigLoader.is_already_loaded() ): # pragma: no cover ## Special behavior. dataset = PyFunceble.storage.HTTP_CODES else: dataset = PyFunceble.storage.STD_HTTP_CODES if (not self.status.status or self.status.status == PyFunceble.storage.STATUS.down ) and (self.status.http_status_code in dataset.list.up or self.status.http_status_code in dataset.list.potentially_up): self.status.status = PyFunceble.storage.STATUS.up self.status.status_source = "HTTP CODE" PyFunceble.facility.Logger.info( "Could define the status of %r from: HTTP Status code Lookup", self.status.idna_subject, ) else: self.status.http_status_code = None PyFunceble.facility.Logger.info( "Finished to try to query the status of %r from: HTTP Status code Lookup", self.status.idna_subject, ) return self def try_to_query_status_from_syntax_lookup( self) -> "AvailabilityCheckerBase": """ Tries to query the status from the syntax. """ PyFunceble.facility.Logger.info( "Started to try to query the status of %r from: Syntax Lookup", self.status.idna_subject, ) if (not self.status.domain_syntax and not self.status.ip_syntax and not self.status.url_syntax): self.status.status = PyFunceble.storage.STATUS.invalid self.status.status_source = "SYNTAX" PyFunceble.facility.Logger.info( "Could define the status of %r from: Syntax Lookup", self.status.idna_subject, ) PyFunceble.facility.Logger.info( "Finished to try to query the status of %r from: Syntax Lookup", self.status.idna_subject, ) return self def try_to_query_status_from_reputation(self) -> "AvailabilityCheckerBase": """ Tries to query the status from the reputation lookup. """ raise NotImplementedError() def try_to_query_status_from_collection(self) -> "AvailabilityCheckerBase": """ Tries to get and set the status from the Collection API. """ PyFunceble.facility.Logger.info( "Started to try to query the status of %r from: Collection Lookup", self.status.idna_subject, ) data = self.collection_query_tool.pull(self.idna_subject) if data and "status" in data: if (self.collection_query_tool.preferred_status_origin == "frequent" and data["status"]["availability"]["frequent"]): self.status.status = data["status"]["availability"]["frequent"] self.status.status_source = "COLLECTION" elif (self.collection_query_tool.preferred_status_origin == "latest" and data["status"]["availability"]["latest"]): self.status.status = data["status"]["availability"]["latest"][ "status"] self.status.status_source = "COLLECTION" elif (self.collection_query_tool.preferred_status_origin == "recommended" and data["status"]["availability"]["recommended"]): self.status.status = data["status"]["availability"][ "recommended"] self.status.status_source = "COLLECTION" PyFunceble.facility.Logger.info( "Could define the status of %r from: Collection Lookup", self.status.idna_subject, ) PyFunceble.facility.Logger.info( "Finished to try to query the status of %r from: Collection Lookup", self.status.idna_subject, ) @CheckerBase.ensure_subject_is_given @CheckerBase.update_status_date_after_query def query_status(self) -> "AvailabilityCheckerBase": """ Queries the status and for for more action. """ raise NotImplementedError() # pylint: disable=useless-super-delegation def get_status(self) -> Optional[AvailabilityCheckerStatus]: return super().get_status()
def __init__( self, subject: Optional[str] = None, *, use_extra_rules: Optional[bool] = None, use_whois_lookup: Optional[bool] = None, use_dns_lookup: Optional[bool] = None, use_netinfo_lookup: Optional[bool] = None, use_http_code_lookup: Optional[bool] = None, use_reputation_lookup: Optional[bool] = None, do_syntax_check_first: Optional[bool] = None, db_session: Optional[Session] = None, use_whois_db: Optional[bool] = None, use_collection: Optional[bool] = None, ) -> None: self.dns_query_tool = DNSQueryTool() self.whois_query_tool = WhoisQueryTool() self.addressinfo_query_tool = AddressInfo() self.hostbyaddr_query_tool = HostByAddrInfo() self.http_status_code_query_tool = HTTPStatusCode() self.domain_syntax_checker = DomainSyntaxChecker() self.ip_syntax_checker = IPSyntaxChecker() self.url_syntax_checker = URLSyntaxChecker() self.extra_rules_handler = ExtraRulesHandler() self.db_session = db_session self.params = AvailabilityCheckerParams() self.status = AvailabilityCheckerStatus() self.status.params = self.params self.status.dns_lookup_record = self.dns_query_tool.lookup_record self.status.whois_lookup_record = self.whois_query_tool.lookup_record if use_extra_rules is not None: self.use_extra_rules = use_extra_rules else: self.guess_and_set_use_extra_rules() if use_whois_lookup is not None: self.use_whois_lookup = use_whois_lookup else: self.guess_and_set_use_whois_lookup() if use_dns_lookup is not None: self.use_dns_lookup = use_dns_lookup else: self.guess_and_set_dns_lookup() if use_netinfo_lookup is not None: self.use_netinfo_lookup = use_netinfo_lookup else: self.guess_and_set_use_netinfo_lookup() if use_http_code_lookup is not None: self.use_http_code_lookup = use_http_code_lookup else: self.guess_and_set_use_http_code_lookup() if use_reputation_lookup is not None: self.use_reputation_lookup = use_reputation_lookup else: self.guess_and_set_use_reputation_lookup() if use_whois_db is not None: self.use_whois_db = use_whois_db else: self.guess_and_set_use_whois_db() super().__init__( subject, do_syntax_check_first=do_syntax_check_first, db_session=db_session, use_collection=use_collection, )