def __blacklist_request_if_needed(request: Request) -> Request: """If request's URL is blacklisted, update the request to point to a blacklisted URL.""" # FIXME there should be a better way to block those unwanted requests if request is None: raise McRequestException("Request is None.") url = request.url() if url is None: raise McRequestException("URL is None.") if len(url) == 0: raise McRequestException("URL is empty.") config = py_get_config() blacklist_url_pattern = None if 'blacklist_url_pattern' in config['mediawords']: blacklist_url_pattern = config['mediawords'][ 'blacklist_url_pattern'] if blacklist_url_pattern is not None and len( blacklist_url_pattern) > 0: if re.search(pattern=blacklist_url_pattern, string=url, flags=re.IGNORECASE | re.UNICODE): request.set_url("http://blacklistedsite.localhost/%s" % url) return request
def __blacklist_request_if_needed(self, request: Request) -> Request: """If request's URL is blacklisted, update the request to point to a blacklisted URL.""" # FIXME there should be a better way to block those unwanted requests if request is None: raise McRequestException("Request is None.") url = request.url() if url is None: raise McRequestException("URL is None.") if len(url) == 0: raise McRequestException("URL is empty.") blacklist_url_pattern = self._user_agent_config.blacklist_url_pattern() if blacklist_url_pattern: # MC_REWRITE_TO_PYTHON: a string might be coming from Perl if isinstance(blacklist_url_pattern, bytes): blacklist_url_pattern = decode_object_from_bytes_if_needed( blacklist_url_pattern) if isinstance(blacklist_url_pattern, str): blacklist_url_pattern = re.compile(blacklist_url_pattern, flags=re.IGNORECASE | re.UNICODE) if re.search(pattern=blacklist_url_pattern, string=url) is not None: request.set_url("http://0.0.0.1/%s" % url) return request
def __blacklist_request_if_needed(request: Request) -> Request: """If request's URL is blacklisted, update the request to point to a blacklisted URL.""" # FIXME there should be a better way to block those unwanted requests if request is None: raise McRequestException("Request is None.") url = request.url() if url is None: raise McRequestException("URL is None.") if len(url) == 0: raise McRequestException("URL is empty.") config = py_get_config() blacklist_url_pattern = None if 'blacklist_url_pattern' in config['mediawords']: blacklist_url_pattern = config['mediawords']['blacklist_url_pattern'] if blacklist_url_pattern is not None and len(blacklist_url_pattern) > 0: if re.search(pattern=blacklist_url_pattern, string=url, flags=re.IGNORECASE | re.UNICODE) is not None: request.set_url("http://0.0.0.1/%s" % url) return request