def __init__(self, ingest_client: IngestApi, ttl: Optional[int] = None):
self.ingest_client = ingest_client
self.ttl = ttl if ttl is not None else 300
self.cached_latest_links_schema = ttl_cache(ttl=self.ttl)(
self.latest_links_schema)
self.cached_latest_file_descriptor_schema = ttl_cache(ttl=self.ttl)(
self.latest_file_descriptor_schema)
def __init__(
self,
*,
title_cache_max_size: int = config.DEFAULT_CACHE_MAX_SIZE,
title_cache_ttl: float = config.DEFAULT_CACHE_TTL,
verify_ssl: bool = True,
):
log.debug(
"Cache parameters: config.DEFAULT_CACHE_MAX_SIZE=%s, title_cache_max_size=%s, title_cache_ttl=%s",
config.DEFAULT_CACHE_MAX_SIZE,
title_cache_max_size,
timedelta(seconds=title_cache_ttl),
)
self._content_amount_guesses = LFUCache(
maxsize=config.DEFAULT_CACHE_TTL
) # Don't use title_cache_max_size.
self._title_outer = ttl_cache(maxsize=title_cache_max_size,
ttl=title_cache_ttl)(
self._title_outer) # type: ignore
self.netloc = lru_cache(maxsize=title_cache_max_size)(
self.netloc) # type: ignore
if verify_ssl:
self._ssl_context = ssl.create_default_context()
assert self._ssl_context.verify_mode == ssl.CERT_REQUIRED
assert self._ssl_context.check_hostname
else:
self._ssl_context = ssl.SSLContext()
assert self._ssl_context.verify_mode == ssl.CERT_NONE
assert not self._ssl_context.check_hostname
log.warning(
"SSL verification is disabled for all requests made using this instance of %s.",
self.__class__.__qualname__,
)
def func_wrapper(method):
ttl_method = ttl_cache(*args, **kwargs)(method)
@functools.wraps(ttl_method)
def wrapped(*in_args, **in_kwargs):
try:
return ttl_method(*in_args, **in_kwargs)
except TypeError as err:
if 'unhashable type' in err.args[0]:
log.warning(
'%s: Returning results without cache. Cannot '
'use cache with unhashable type in arguments',
method.__name__)
return ttl_method.__wrapped__(*in_args, **in_kwargs)
raise
return wrapped
def __enable_method_cache(self, method_name):
"""Enable cache for a method.
Args:
method_name (str): name of method for which to enable cache
Returns:
bool: True is success, False if failed
"""
try:
method = getattr(self, method_name)
try:
return not inspect.isfunction(method.cache_info)
except AttributeError:
setattr(self, method_name, ttl_cache(ttl=self.cache_ttl)(method))
return True
except AttributeError:
# method doesn't exist on initializing class
return False
def __init__(self, feed_type: str):
html_url_suffix = feed_type if feed_type != "trending" else ''
html_urls = [
f'{config.HTML_URL_BASE}{html_url_suffix}?page={page_num}'
for page_num in range(1, config.NUM_PAGES_READ[feed_type] + 1)
]
self._html_requests = [
Request(html_url, headers={'User-Agent': config.USER_AGENT})
for html_url in html_urls
]
self._feed_type = feed_type
self._feed_type_desc = f'for "{self._feed_type}"'
self._feed_title = config.FEED_TITLE_TEMPLATE.format(
feed_type=self._feed_type.title())
self._hext_rule_extract = Rule(config.HTML_HEXT).extract
self._is_debug_logged = log.isEnabledFor(logging.DEBUG)
self._output = lru_cache(maxsize=1)(
self._output) # type: ignore # Instance level cache
self.feed = ttl_cache(maxsize=1, ttl=config.CACHE_TTL)(
self.feed) # type: ignore # Instance level cache