async def _cache_response(self, cache_key, name, response):
"""Cache the response, if any cache headers we understand are present."""
if not self.cache:
return
with time_block("Upstream cache set %s" % name, logger=self.log):
# cache the response
try:
pickle_response = pickle.dumps(response,
pickle.HIGHEST_PROTOCOL)
await self.cache.set(cache_key, pickle_response)
except Exception:
self.log.error("Upstream cache failed %s" % name,
exc_info=True)
def _cache_response(self, cache_key, name, response):
"""Cache the response, if any cache headers we understand are present."""
if not self.cache:
return
with time_block("Upstream cache set %s" % name):
# cache the response
try:
pickle_response = pickle.dumps(response, pickle.HIGHEST_PROTOCOL)
yield self.cache.set(
cache_key,
pickle_response,
)
except Exception:
app_log.error("Upstream cache failed %s" % name, exc_info=True)
def _cache_response(self, cache_key, name, response):
"""Cache the response, if any cache headers we understand are present."""
if not self.cache:
return
if not any(response.headers.get(key) for key in cache_headers):
# no cache headers, no point in caching the response
return
with time_block("Upstream cache set %s" % name):
# cache the response if there are any cache headers (use cache expiry?)
try:
pickle_response = pickle.dumps(response, pickle.HIGHEST_PROTOCOL)
yield self.cache.set(
cache_key,
pickle_response,
int(time.time() + self.expiry),
)
except Exception:
app_log.error("Upstream cache failed %s" % name, exc_info=True)
def _fetch_impl(self, request, callback):
tic = time.time()
if request.user_agent is None:
request.user_agent = 'Tornado-Async-Client'
# when logging, use the URL without params
name = request.url.split('?')[0]
cached_response = None
app_log.debug("Fetching %s", name)
cache_key = hashlib.sha256(request.url.encode('utf8')).hexdigest()
with time_block("Upstream cache get %s" % name):
cached_response = yield self._get_cached_response(cache_key, name)
if cached_response:
app_log.info("Upstream cache hit %s", name)
# add cache headers, if any
for resp_key, req_key in cache_headers.items():
value = cached_response.headers.get(resp_key)
if value:
request.headers[req_key] = value
else:
app_log.debug("Upstream cache miss %s", name)
response = yield gen.Task(
super(NBViewerAsyncHTTPClient, self).fetch_impl, request)
dt = time.time() - tic
if cached_response and (response.code == 304 or response.code >= 400):
log = app_log.info if response.code == 304 else app_log.warning
log("Upstream %s on %s in %.2f ms, using cached response",
response.code, name, 1e3 * dt)
response = self._update_cached_response(response, cached_response)
callback(response)
else:
if not response.error:
app_log.info("Fetched %s in %.2f ms", name, 1e3 * dt)
callback(response)
if not response.error:
yield self._cache_response(cache_key, name, response)
def _fetch_impl(self, request, callback):
tic = time.time()
if request.user_agent is None:
request.user_agent = 'Tornado-Async-Client'
# when logging, use the URL without params
name = request.url.split('?')[0]
cached_response = None
app_log.debug("Fetching %s", name)
cache_key = hashlib.sha256(request.url.encode('utf8')).hexdigest()
with time_block("Upstream cache get %s" % name):
cached_response = yield self._get_cached_response(cache_key, name)
if cached_response:
app_log.info("Upstream cache hit %s", name)
# add cache headers, if any
for resp_key, req_key in cache_headers.items():
value = cached_response.headers.get(resp_key)
if value:
request.headers[req_key] = value
else:
app_log.debug("Upstream cache miss %s", name)
response = yield gen.Task(super(NBViewerAsyncHTTPClient, self).fetch_impl, request)
dt = time.time() - tic
if cached_response and (response.code == 304 or response.code >= 400):
log = app_log.info if response.code == 304 else app_log.warning
log("Upstream %s on %s in %.2f ms, using cached response",
response.code, name, 1e3 * dt)
response = self._update_cached_response(response, cached_response)
callback(response)
else:
if not response.error:
app_log.info("Fetched %s in %.2f ms", name, 1e3 * dt)
callback(response)
if not response.error:
yield self._cache_response(cache_key, name, response)
