def _get_query_data(self, query_name, params=None) -> Tuple[Dict, Dict]:
query = get_redshift_query(query_name)
parameterized_query = query.build_full_params(params or dict())
try:
trigger_if_stale(parameterized_query)
except OSError as err:
get_logger().warning("Failed to trigger stale query refresh: " +
str(err))
if not parameterized_query.result_available:
raise QueryDataNotAvailableException()
response = parameterized_query.response_payload
return response["series"]["data"], response["series"]["metadata"]
def fetch_query_results(request, name):
if request.method == "OPTIONS":
response = HttpResponse(status=204)
else:
parameterized_query = _get_query_and_params(request, name)
if issubclass(parameterized_query.__class__, HttpResponse):
return parameterized_query
last_modified = parameterized_query.result_as_of
if last_modified:
last_modified = timegm(last_modified.utctimetuple())
response = None
is_cache_hit = parameterized_query.result_available
if is_cache_hit:
trigger_if_stale(parameterized_query)
# Try to return a minimal response
response = get_conditional_response(request,
last_modified=last_modified)
if not response:
if request.method == "HEAD":
response = HttpResponse(204)
else:
# Resort to a full response
response = _fetch_query_results(parameterized_query,
user=request.user)
# Add Last-Modified header
if response.status_code in (200, 204, 304):
response["Last-Modified"] = http_date(last_modified)
# Add Cache-Control headers
if parameterized_query.is_personalized or parameterized_query.has_premium_values:
patch_cache_control(response, no_cache=True, private=True)
else:
patch_cache_control(response, no_cache=True, public=True)
# Add CORS header if permitted - can be replaced by middleware in future
origin = urlparse(request.META.get("HTTP_ORIGIN", ""))
if origin.netloc in settings.ANALYTICS_CORS_ORIGIN_WHITELIST:
response["Access-Control-Allow-Origin"] = origin.geturl()
response["Access-Control-Allow-Methods"] = "GET, HEAD, OPTIONS"
response["Access-Control-Allow-Headers"] = ", ".join(
settings.ANALYTICS_CORS_HEADER_WHITELIST)
# Always patch vary header so browsers do not cache CORS
patch_vary_headers(response, ["Origin"])
return response
def refresh_meta_preview():
from hsreplaynet.utils.aws.redshift import get_redshift_query
query = get_redshift_query("archetype_popularity_distribution_stats")
ranks = [x for x in range(0, 21)]
regions = ["REGION_EU", "REGION_US", "REGION_KR", "REGION_CN"]
for rank in ranks:
for region in regions:
parameterized_query = query.build_full_params(
dict(TimeRange="LAST_1_DAY",
GameType="RANKED_STANDARD",
RankRange=RANK_MAP[rank],
Region=region))
trigger_if_stale(parameterized_query)
def get(self, request, **kwargs):
serializer = self.serializer_class(data=request.GET)
serializer.is_valid(raise_exception=True)
supplied_params = request.GET.copy()
supplied_params.update(serializer.validated_data)
del supplied_params["query"]
if "Region" in supplied_params:
supplied_params["Region"] = Region.from_int(
supplied_params["Region"]).name
try:
parameterized_query = serializer.query.build_full_params(
supplied_params)
except InvalidOrMissingQueryParameterError as e:
raise PermissionDenied(str(e)) from e
if parameterized_query.has_premium_values:
if not self._check_premium(request):
raise PermissionDenied("You do not have access to this query.")
last_modified = parameterized_query.result_as_of
if last_modified:
last_modified = timegm(last_modified.utctimetuple())
response = None
is_cache_hit = parameterized_query.result_available
if is_cache_hit:
trigger_if_stale(parameterized_query)
# Try to return a minimal response
response = get_conditional_response(request,
last_modified=last_modified)
if not response:
# Resort to a full response
response = _fetch_query_results(parameterized_query,
user=request.user)
response["last-modified"] = http_date(last_modified)
# Always send Cache-Control headers
if parameterized_query.is_personalized or parameterized_query.has_premium_values:
patch_cache_control(response, no_cache=True, private=True)
else:
patch_cache_control(response, no_cache=True, public=True)
return response
def _fetch_query_results(parameterized_query,
run_local=False,
user=None,
priority=None):
cache_is_populated = parameterized_query.cache_is_populated
is_cache_hit = parameterized_query.result_available
triggered_refresh = False
if is_cache_hit:
triggered_refresh = trigger_if_stale(parameterized_query, run_local,
priority)
response = HttpResponse(
content=parameterized_query.response_payload_data,
content_type=parameterized_query.response_payload_type)
elif cache_is_populated and parameterized_query.is_global:
if parameterized_query.is_backfillable and parameterized_query.is_personalized:
# Premium users should have cache entries even if the result set is empty
# So we should only reach this block if the user just subscribed
# And we haven't rerun the global query yet.
triggered_refresh = True
attempt_request_triggered_query_execution(parameterized_query,
run_local, priority)
result = {"msg": "Query is processing. Check back later."}
response = JsonResponse(result, status=202)
else:
# There is no content for this permutation of parameters
# For deck related queries this most likely means that someone hand crafted the URL
# Or if it's a card related query, then it's a corner case where there is no data
response = HttpResponse(status=204)
else:
# The cache is not populated yet for this query.
# Perhaps it's a new query or perhaps the cache was recently flushed.
# So attempt to trigger populating it
attempt_request_triggered_query_execution(parameterized_query,
run_local, priority)
result = {"msg": "Query is processing. Check back later."}
response = JsonResponse(result, status=202)
log.info("Query: %s Cache Populated: %s Cache Hit: %s Is Stale: %s" %
(cache_is_populated, parameterized_query.cache_key, is_cache_hit,
triggered_refresh))
query_fetch_metric_fields = {
"count": 1,
}
query_fetch_metric_fields.update(
parameterized_query.supplied_non_filters_dict)
influx.influx_metric("redshift_query_fetch",
query_fetch_metric_fields,
cache_populated=cache_is_populated,
cache_hit=is_cache_hit,
query_name=parameterized_query.query_name,
triggered_refresh=triggered_refresh,
**parameterized_query.supplied_filters_dict)
return response
def _get_redshift_query_data(self):
# Fetch the results for the target Redshift query, trigger a refresh as necessary,
# and cache the response for subsequent calls.
if not self.redshift_query_data:
parameterized_query = self.get_parameterized_query()
try:
trigger_if_stale(parameterized_query)
except OSError as err:
get_logger().warning(
"Failed to trigger stale query refresh: " + str(err))
if not parameterized_query.result_available:
raise QueryDataNotAvailableException()
response = parameterized_query.response_payload
self.redshift_query_data = self.filter_parameterized_query_response(
response["series"]["data"])
return self.redshift_query_data