def _subscribe(request):
email = request.POST["data[email]"]
try:
email_address = EmailAddress.objects.get(email=email)
user = email_address.user
user_exists = True
primary_email = find_best_email_for_user(user)
# If the user exists and the subscription webhook was about their primary email,
# update their marketing contact preference to reflect that.
if primary_email and primary_email.email == email:
if "email" not in user.settings:
user.settings["email"] = {"marketing": True}
else:
user.settings["email"]["marketing"] = True
user.save()
except EmailAddress.DoesNotExist:
user_exists = False
primary_email = False
influx_metric(
"mailchimp_webhook_request",
{"value": 1},
type="subscribe",
primary_email=primary_email,
user_exists=user_exists
)
def parse_upload_event(upload_event, meta):
orig_match_start = dateutil_parse(meta["match_start"])
match_start = get_valid_match_start(orig_match_start, upload_event.created)
if match_start != orig_match_start:
upload_event.tainted = True
upload_event.save()
upload_event.file.open(mode="rb")
log_bytes = upload_event.file.read()
if not log_bytes:
raise ValidationError("The uploaded log file is empty.")
influx_metric("raw_power_log_upload_num_bytes", {"size": len(log_bytes)})
powerlog = StringIO(log_bytes.decode("utf-8"))
upload_event.file.close()
try:
parser = InfluxInstrumentedParser(upload_event.shortid, meta)
parser._game_state_processor = "GameState"
parser._current_date = match_start
parser.read(powerlog)
except Exception as e:
log.exception("Got exception %r while parsing log", e)
raise ParsingError(str(e)) # from e
if not upload_event.test_data:
parser.write_payload()
return parser
def post(self, request):
marketing_prefs = request.POST.get("marketing", "") == "on"
request.user.settings["email"] = {
"marketing": marketing_prefs,
"updated": timezone.now().isoformat(),
}
request.user.save()
try:
client = get_mailchimp_client()
email = find_best_email_for_user(request.user)
client.lists.members.create_or_update(
settings.MAILCHIM_LIST_KEY_ID,
get_subscriber_hash(email.email), {
"email_address":
email.email,
"status_if_new":
get_mailchimp_subscription_status(request.user)
})
except Exception as e:
log.warning("Failed to contact MailChimp API: %s" % e)
influx_metric("mailchimp_request_failures", {"count": 1})
messages.info(request, _("Your email preferences have been saved."))
return redirect(self.success_url)
def reap_upload_events_asof(year, month, day, hour):
success_reaping_delay = settings.SUCCESSFUL_UPLOAD_EVENT_REAPING_DELAY_DAYS
nonsuccess_reaping_delay = settings.UNSUCCESSFUL_UPLOAD_EVENT_REAPING_DELAY_DAYS
cursor = connections["uploads"].cursor()
args = (
year,
month,
day,
hour,
success_reaping_delay,
nonsuccess_reaping_delay,
)
# Note: this stored proc will only delete the DB records
# The objects in S3 will age out naturally after 90 days
# according to our bucket's object lifecycle policy
cursor.callproc("reap_upload_events", args)
result_row = cursor.fetchone()
successful_reaped = result_row[0]
unsuccessful_reaped = result_row[1]
influx_metric("upload_events_reaped",
fields={
"successful_reaped": successful_reaped,
"unsuccessful_reaped": unsuccessful_reaped
})
cursor.close()
def reap_orphans_for_date(reaping_date):
inventory = get_reaping_inventory_for_date(reaping_date)
for hour, hour_inventory in inventory.items():
reaped_orphan_count = 0
for minute, minute_inventory in hour_inventory.items():
for shortid, keys in minute_inventory.items():
descriptor = keys.get("descriptor")
if is_safe_to_reap(shortid, keys):
log.debug("Reaping Descriptor: %r", descriptor)
aws.S3.delete_object(
Bucket=settings.S3_RAW_LOG_UPLOAD_BUCKET,
Key=keys["descriptor"])
reaped_orphan_count += 1
else:
log.debug("Skipping: %r (Unsafe to reap)", descriptor)
log.info("A total of %s descriptors reaped for hour: %s" %
(str(reaped_orphan_count), str(hour)))
# Report count of orphans to Influx
fields = {"count": reaped_orphan_count}
influx_metric("orphan_descriptors_reaped",
fields=fields,
timestamp=reaping_date,
hour=hour)
def get(self, request, id):
claim = get_uuid_object_or_404(AccountClaim, id=id)
log.info("Claim %r: Token=%r, User=%r", claim, claim.token,
claim.token.user)
if claim.token.user:
if claim.token.user.is_fake:
count = GameReplay.objects.filter(
user=claim.token.user).update(user=request.user)
log.info("Updated %r replays. Deleting %r.", count,
claim.token.user)
# For now we just delete the fake user, because we are not using it.
claim.token.user.delete()
else:
log.warning("%r is a real user. Deleting %r.",
claim.token.user, claim)
# Something's wrong. Get rid of the claim and reject the request.
claim.delete()
influx_metric("hsreplaynet_account_claim", {"count": 1},
error=1)
return HttpResponseForbidden(
"This token has already been claimed.")
claim.token.user = request.user
claim.token.save()
# Replays are claimed in AuthToken post_save signal (games.models)
claim.delete()
messages.info(request, "You have claimed your account. Nice!")
influx_metric("hsreplaynet_account_claim", {"count": 1})
return redirect(self.get_redirect_url(request))
def can_subscribe(self):
from djpaypal.models import BillingAgreement
user = self.request.user
if not user.is_authenticated:
return False
# refresh all pending and active BillingAgreements
for agreement in BillingAgreement.objects.filter(
user=user, state__in=["Pending", "Active"]
):
BillingAgreement.find_and_sync(agreement.id)
if user.is_premium:
return False
# reject if there's any remaining pending or active BillingAgreements
stale_agreements = BillingAgreement.objects.filter(
user=user, state__in=["Pending", "Active"]
)
if stale_agreements.count() > 0:
for agreement in stale_agreements:
# Log an error
influx_metric(
"hsreplaynet_paypal_stale_agreement",
{
"count": "1",
"id": agreement.id
},
state=agreement.state
)
return False
return True
def _trigger_if_stale(parameterized_query, run_local=False, priority=None):
did_preschedule = False
result = False
as_of = parameterized_query.result_as_of
if as_of is not None:
staleness = int((datetime.utcnow() - as_of).total_seconds())
else:
staleness = None
if parameterized_query.result_is_stale or run_local:
attempt_request_triggered_query_execution(parameterized_query,
run_local, priority)
result = True
elif staleness and staleness > settings.MINIMUM_QUERY_REFRESH_INTERVAL:
did_preschedule = True
parameterized_query.preschedule_refresh()
query_fetch_metric_fields = {
"count": 1,
}
if staleness:
query_fetch_metric_fields["staleness"] = staleness
query_fetch_metric_fields.update(
parameterized_query.supplied_non_filters_dict)
influx.influx_metric("redshift_response_payload_staleness",
query_fetch_metric_fields,
query_name=parameterized_query.query_name,
did_preschedule=did_preschedule,
**parameterized_query.supplied_filters_dict)
return result
def finish_async_redshift_query(event, context):
"""A handler triggered by the arrival of an UNLOAD manifest on S3
The S3 trigger must be configured manually with:
prefix = PROD
suffix = manifest
"""
logger = logging.getLogger(
"hsreplaynet.lambdas.finish_async_redshift_query")
catalogue = get_redshift_catalogue()
s3_event = event["Records"][0]["s3"]
bucket = s3_event["bucket"]["name"]
manifest_key = unquote(s3_event["object"]["key"])
if bucket == settings.S3_UNLOAD_BUCKET:
logger.info("Finishing query: %s" % manifest_key)
start_time = time.time()
parameterized_query = catalogue.refresh_cache_from_s3_manifest_key(
manifest_key=manifest_key)
query_execute_metric_fields = {
"duration_seconds": parameterized_query.most_recent_duration,
"unload_seconds": time.time() - start_time,
"query_handle": parameterized_query.most_recent_query_handle
}
query_execute_metric_fields.update(
parameterized_query.supplied_non_filters_dict)
influx_metric("finished_async_redshift_query",
query_execute_metric_fields,
query_name=parameterized_query.query_name,
**parameterized_query.supplied_filters_dict)
def get(self, request):
from djpaypal.models import PreparedBillingAgreement
token = request.GET.get("token", "")
if not token:
return self.fail(_("Unable to complete subscription."))
try:
prepared_agreement = PreparedBillingAgreement.objects.get(id=token)
except PreparedBillingAgreement.DoesNotExist:
return self.fail(_("Invalid subscription token."))
if prepared_agreement.user != self.request.user:
return self.fail(_("You are not logged in as the correct user."))
billing_agreement = prepared_agreement.execute()
# At this time, we expect the billing agreement to be active
# If that isn't the case, PayPal probably needs a moment to complete the payment
# For now, let's just send a metric
influx_metric("hsreplaynet_paypal_agreement_state", {
"count": "1",
"id": billing_agreement.id
},
state=billing_agreement.state)
# Null out the premium checkout timestamp so that we don't remind the user to
# complete the checkout process.
request.user.last_premium_checkout = None
request.user.save()
messages.success(self.request,
_("You are now subscribed with PayPal!"))
return redirect(self.get_success_url())
def do_execute_redshift_query(query_name, supplied_params, queue_name):
"""A common entry point for the actual execution of queries on Lambda"""
logger = logging.getLogger("hsreplaynet.lambdas.execute_redshift_query")
logger.info("Query Name: %s" % query_name)
logger.info("Query Params: %s" % supplied_params)
query = get_redshift_catalogue().get_query(query_name)
parameterized_query = query.build_full_params(supplied_params)
try:
wlm_queue = settings.REDSHIFT_QUERY_QUEUES[queue_name]["wlm_queue"]
with get_concurrent_redshift_query_queue_semaphore(queue_name):
_do_execute_query(parameterized_query, wlm_queue)
logger.info("Query Execution Complete")
return True
except NotAvailable:
logger.warn(
"The Redshift query queue was already at max concurrency. Skipping query."
)
metric_fields = {"count": 1}
metric_fields.update(parameterized_query.supplied_non_filters_dict)
influx_metric("redshift_query_lambda_execution_concurrency_exceeded",
metric_fields,
query_name=query_name,
**parameterized_query.supplied_filters_dict)
return False
def list(self, request, *args, **kwargs):
error = None
try:
queryset = self.filter_queryset(self.get_queryset())
# We pull back all users with matching account los, which may result in
# duplicates across region, so we need to do region filtering on the result set
# at the application level. It can't be pushed down to the database, since
# Django doesn't natively support filtering on tuples as part of an IN clause.
serializer_context = self.get_serializer_context()
def match_region(r):
return (
r.region,
r.account_lo) in serializer_context["redshift_query_data"]
region_filtered_queryset = filter(match_region, queryset)
serializer = self.get_serializer(region_filtered_queryset,
many=True)
return Response(serializer.data)
except QueryDataNotAvailableException as e:
error = type(e).__name__
return Response(status=status.HTTP_202_ACCEPTED)
except Exception as e:
error = type(e).__name__
raise e
finally:
influx.influx_metric("hsreplaynet_leaderboard_api", {"count": 1},
error=error)
def classify_into_archetype(self, player_class, save: bool = True) -> int:
game_format = self.format
signature_weights = ClusterSnapshot.objects.get_signature_weights(
game_format, player_class)
sig_archetype_id = classify_deck(self.dbf_map(), signature_weights)
# New Style Deck Prediction
nn_archetype_id = None
# nn_archetype_id = ClusterSetSnapshot.objects.predict_archetype_id(
# player_class,
# game_format,
# self,
# )
archetype_id = sig_archetype_id or nn_archetype_id
influx_metric("archetype_prediction_outcome", {
"count": 1,
"signature_weight_archetype_id": sig_archetype_id,
"neural_net_archetype_id": nn_archetype_id,
"archetype_id": archetype_id,
"deck_id": self.id,
},
success=archetype_id is not None,
signature_weight_success=sig_archetype_id is not None,
neural_net_success=nn_archetype_id is not None,
method_agreement=sig_archetype_id == nn_archetype_id,
player_class=player_class.name,
game_format=game_format.name)
if archetype_id and save:
self.update_archetype(archetype_id)
return archetype_id
def post(self, request):
serializer = self.serializer_class(data=request.data)
serializer.is_valid(raise_exception=True)
assert not request.user.is_fake
token = get_uuid_object_or_404(AuthToken, key=serializer.validated_data["token"])
if token.user and not token.user.is_fake:
influx_metric("hsreplaynet_account_claim", {"count": 1}, error=1)
raise ValidationError({
"error": "token_already_claimed", "detail": "This token has already been claimed."
})
with transaction.atomic():
# Update the token's replays to match the new user
replays_claimed = GameReplay.objects.filter(user=token.user).update(user=request.user)
if token.user:
# Delete the token-specific fake user
# If there was no user attached to the token, claiming it will attach the correct user.
token.user.delete()
# Update the token's actual user
token.user = request.user
# Save.
token.save()
influx_metric("hsreplaynet_account_claim", {"count": 1, "replays": replays_claimed})
return Response(status=HTTP_204_NO_CONTENT)
def _unsubscribe(request):
email = request.POST["data[email]"]
try:
email_address = EmailAddress.objects.get(email=email)
user = email_address.user
user_exists = True
primary_email = find_best_email_for_user(user)
# If the user exists and the subscription update was related to their primary
# email address, update their marketing contact preference. (We don't want to
# update the preference if their non-primary email address was unsubscribed,
# because that won't really reflect the actual state of their preference.)
if "email" in user.settings and primary_email and primary_email.email == email:
user.settings["email"]["marketing"] = False
user.save()
except EmailAddress.DoesNotExist:
user_exists = False
primary_email = False
influx_metric(
"mailchimp_webhook_request",
{"value": 1},
reason=request.POST.get("data[reason]"),
type="unsubscribe",
primary_email=primary_email,
user_exists=user_exists
)
def _log_mailchimp_error(e):
# Log a warning, an InfluxDB metric, and a Sentry alert.
log.warning("Failed to contact MailChimp API: %s" % e)
influx_metric("mailchimp_request_failures", {"count": 1})
error_handler(e)
def reap_orphans_for_date(reaping_date):
inventory = get_reaping_inventory_for_date(reaping_date)
for hour, hour_inventory in inventory.items():
reaped_orphan_count = 0
for minute, minute_inventory in hour_inventory.items():
for shortid, keys in minute_inventory.items():
if is_safe_to_reap(shortid, keys):
log.info("Reaping Descriptor: %r", keys["descriptor"])
aws.S3.delete_object(
Bucket=settings.S3_RAW_LOG_UPLOAD_BUCKET,
Key=keys["descriptor"]
)
reaped_orphan_count += 1
else:
log.info("Skipping Descriptor: %r (Unsafe To Reap)", keys["descriptor"])
log.info(
"A total of %s descriptors reaped for hour: %s" % (
str(reaped_orphan_count),
str(hour)
)
)
# Report count of orphans to Influx
fields = {
"count": reaped_orphan_count
}
influx_metric(
"orphan_descriptors_reaped",
fields=fields,
timestamp=reaping_date,
hour=hour
)
def _do_execute_query_work(parameterized_query, wlm_queue=None):
if not parameterized_query.result_is_stale:
log.info(
"Up-to-date cached data exists. Exiting without running query.")
else:
log.info("Cached data missing or stale. Executing query now.")
# DO EXPENSIVE WORK
start_ts = time.time()
exception_raised = False
exception_msg = None
try:
parameterized_query.refresh_result(wlm_queue)
except Exception as e:
exception_raised = True
exception_msg = str(e)
raise
finally:
end_ts = time.time()
duration_seconds = round(end_ts - start_ts, 2)
query_execute_metric_fields = {
"duration_seconds": duration_seconds,
"exception_message": exception_msg
}
query_execute_metric_fields.update(
parameterized_query.supplied_non_filters_dict)
influx_metric("redshift_query_execute",
query_execute_metric_fields,
exception_thrown=exception_raised,
query_name=parameterized_query.query_name,
**parameterized_query.supplied_filters_dict)
def _unsubscribe(request):
email = request.POST["data[email]"]
try:
email_address = EmailAddress.objects.get(email=email)
user = email_address.user
user_exists = True
# If the user exists, update their marketing contact preference.
if "email" in user.settings:
user.settings["email"]["marketing"] = False
user.save()
except EmailAddress.DoesNotExist:
user_exists = False
influx_metric(
"mailchimp_webhook_request",
{"value": 1},
reason=request.POST.get("data[reason]"),
type="unsubscribe",
user_exists=user_exists
)
def classify_into_archetype(self, player_class, save: bool = True) -> int:
game_format = self.format
signature_weights = ClusterSnapshot.objects.get_signature_weights(
game_format, player_class)
blocked_classifications = {}
# A failure callback for classify_deck so that we can get notified when a deck can't
# be classified to an archetype because it's missing a card or fails a rule check.
# This callback stores the results so that we can instrument them below.
def record_classification_failure(failure_description):
reason = failure_description.get("reason")
if reason:
if reason in blocked_classifications:
blocked_classifications[
reason] = blocked_classifications[reason] + 1
else:
blocked_classifications[reason] = 1
sig_archetype_id = classify_deck(
self.dbf_map(),
signature_weights,
failure_callback=record_classification_failure)
# Instrument any failures that occurred during the classification attempt.
for block_reason, count in blocked_classifications.items():
influx_metric("archetype_classification_blocked", {"count": count},
reason=block_reason)
# New Style Deck Prediction
nn_archetype_id = None
# nn_archetype_id = ClusterSetSnapshot.objects.predict_archetype_id(
# player_class,
# game_format,
# self,
# )
archetype_id = sig_archetype_id or nn_archetype_id
influx_metric("archetype_prediction_outcome", {
"count": 1,
"signature_weight_archetype_id": sig_archetype_id,
"neural_net_archetype_id": nn_archetype_id,
"archetype_id": archetype_id,
"deck_id": self.id,
},
success=archetype_id is not None,
signature_weight_success=sig_archetype_id is not None,
neural_net_success=nn_archetype_id is not None,
method_agreement=sig_archetype_id == nn_archetype_id,
player_class=player_class.name,
game_format=game_format.name)
if archetype_id and save:
self.update_archetype(archetype_id)
return archetype_id
def post(self, request):
# Record reason and message in influx
influx_metric("hsreplaynet_replays_delete", {"count": 1})
request.user.replays.update(is_deleted=True)
messages.info(self.request, _("Your replays have been deleted."))
return redirect(self.success_url)
def refresh_stale_redshift_queries(event, context):
"""A cron'd handler that attempts to refresh queries queued in Redis"""
logger = logging.getLogger(
"hsreplaynet.lambdas.refresh_stale_redshift_queries")
start_time = time.time()
logger.info("Start Time: %s" % str(start_time))
catalogue = get_redshift_catalogue()
scheduler = catalogue.scheduler
target_duration_seconds = 55
duration = 0
for queue_name in scheduler.query_queues:
influx_metric(
"redshift_refresh_queue_depth",
{"depth": scheduler.queue_size(queue_name)},
queue_name=queue_name,
)
# We run for 55 seconds, since the majority of queries take < 5 seconds to finish
# And the next scheduled invocation of this will be starting a minute after this one.
while duration < target_duration_seconds:
logger.info("Runtime duration: %s" % str(duration))
available_slots = scheduler.get_available_slots(
) - scheduler.get_queued_query_count()
logger.info("Available cluster slots: %s" % str(available_slots))
if available_slots <= 1:
sleep_duration = 5
logger.info("Sleeping for %i until more slots are available" %
sleep_duration)
# If only 1 slot remains leave it for ETL or an IMMEDIATE query.
time.sleep(sleep_duration)
current_time = time.time()
duration = current_time - start_time
continue
remaining_seconds = target_duration_seconds - duration
if remaining_seconds < 1:
break
logger.info("Will block for queued query for %s seconds" %
str(remaining_seconds))
try:
refreshed_query = scheduler.refresh_next_pending_query(
block_for=remaining_seconds,
# This uses available cluster resources to refresh queries early
force=REDSHIFT_PREEMPTIVELY_REFRESH_QUERIES)
if refreshed_query:
logger.info("Refreshed: %s" % refreshed_query.cache_key)
else:
logger.info("Nothing to refresh.")
except Exception:
logger.error("Error refreshing query")
sentry.captureException()
current_time = time.time()
duration = current_time - start_time
def _fetch_query_results(parameterized_query,
run_local=False,
user=None,
priority=None):
cache_is_populated = parameterized_query.cache_is_populated
is_cache_hit = parameterized_query.result_available
triggered_refresh = False
if is_cache_hit:
triggered_refresh = trigger_if_stale(parameterized_query, run_local,
priority)
response = HttpResponse(
content=parameterized_query.response_payload_data,
content_type=parameterized_query.response_payload_type)
elif cache_is_populated and parameterized_query.is_global:
if parameterized_query.is_backfillable and parameterized_query.is_personalized:
# Premium users should have cache entries even if the result set is empty
# So we should only reach this block if the user just subscribed
# And we haven't rerun the global query yet.
triggered_refresh = True
attempt_request_triggered_query_execution(parameterized_query,
run_local, priority)
result = {"msg": "Query is processing. Check back later."}
response = JsonResponse(result, status=202)
else:
# There is no content for this permutation of parameters
# For deck related queries this most likely means that someone hand crafted the URL
# Or if it's a card related query, then it's a corner case where there is no data
response = HttpResponse(status=204)
else:
# The cache is not populated yet for this query.
# Perhaps it's a new query or perhaps the cache was recently flushed.
# So attempt to trigger populating it
attempt_request_triggered_query_execution(parameterized_query,
run_local, priority)
result = {"msg": "Query is processing. Check back later."}
response = JsonResponse(result, status=202)
log.info("Query: %s Cache Populated: %s Cache Hit: %s Is Stale: %s" %
(cache_is_populated, parameterized_query.cache_key, is_cache_hit,
triggered_refresh))
query_fetch_metric_fields = {
"count": 1,
}
query_fetch_metric_fields.update(
parameterized_query.supplied_non_filters_dict)
influx.influx_metric("redshift_query_fetch",
query_fetch_metric_fields,
cache_populated=cache_is_populated,
cache_hit=is_cache_hit,
query_name=parameterized_query.query_name,
triggered_refresh=triggered_refresh,
**parameterized_query.supplied_filters_dict)
return response
def create(self, validated_data):
validated_data["user"] = self.context["request"].user
ret = WebhookEndpoint.objects.create(**validated_data)
influx_metric("hsreplaynet_webhook_create", {
"source": "api-oauth2",
"client_id": self.context["request"].auth.application.client_id,
"uuid": str(ret.uuid),
})
return ret
def new_user(self, request, sociallogin):
battletag = sociallogin.account.extra_data.get("battletag", "")
influx_metric(
"hsreplaynet_socialauth_signup",
{"count": 1},
provider=sociallogin.account.provider,
region=sociallogin.account.extra_data.get("region", "unknown"),
has_battletag=bool(battletag),
)
ret = super().new_user(request, sociallogin)
ret.battletag = battletag or ""
return ret
def process_upload_event(upload_event):
"""
Wrapper around do_process_upload_event() to set the event's
status and error/traceback as needed.
"""
upload_event.error = ""
upload_event.traceback = ""
if upload_event.status != UploadEventStatus.PROCESSING:
upload_event.status = UploadEventStatus.PROCESSING
upload_event.save()
try:
replay, do_flush_exporter = do_process_upload_event(upload_event)
except Exception as e:
from traceback import format_exc
upload_event.error = str(e)
upload_event.traceback = format_exc()
upload_event.status, reraise = handle_upload_event_exception(e, upload_event)
metric_fields = {"count": 1}
if upload_event.game:
metric_fields["shortid"] = str(upload_event.game.shortid)
influx_metric(
"upload_event_exception",
metric_fields,
error=upload_event.status.name.lower()
)
upload_event.save()
if reraise:
raise
else:
return
else:
upload_event.game = replay
upload_event.status = UploadEventStatus.SUCCESS
upload_event.save()
try:
with influx_timer("redshift_exporter_flush_duration"):
do_flush_exporter()
except Exception as e:
# Don't fail on this
error_handler(e)
influx_metric(
"flush_redshift_exporter_error",
{
"count": 1,
"error": str(e)
}
)
return replay
def replay_meets_recency_requirements(upload_event, global_game):
# We only load games in where the match_start date is within +/ 36 hours from
# The upload_date. This filters out really old replays people might upload
# Or replays from users with crazy system clocks.
# The purpose of this filtering is to do reduce variability and thrash in our vacuuming
# If we determine that vacuuming is not a bottleneck than we can consider
# relaxing this requirement.
meets_requirements, diff_hours = _dates_within_threshold(
global_game.match_start,
upload_event.log_upload_date,
settings.REDSHIFT_ETL_UPLOAD_DELAY_LIMIT_HOURS
)
if not meets_requirements:
influx_metric("replay_failed_recency_requirement", {"count": 1, "diff": diff_hours})
return meets_requirements
def _handle_request(self, request):
# Ensure that we're being called back with our secret key.
if request.GET.get("key") != settings.MAILCHIMP_WEBHOOK_KEY:
return HttpResponse(status=HTTP_403_FORBIDDEN)
event_type = request.POST.get("type")
if event_type == "unsubscribe" and request.POST.get("data[action]") == "unsub":
self._unsubscribe(request)
elif event_type:
influx_metric("mailchimp_webhook_request", {"value": 1}, type=event_type)
return HttpResponse(status=HTTP_200_OK)
def _publish_tag_changes(self,
user,
email_str,
tags_to_add,
tags_to_remove,
verbose=False):
list_key_id = settings.MAILCHIMP_LIST_KEY_ID
email_hash = get_subscriber_hash(email_str)
client = get_mailchimp_client()
# We may never have seen this user's email address before or sent it to MailChimp,
# so do a defensive subscriber creation.
try:
client.lists.members.create_or_update(
list_key_id, email_hash, {
"email_address": email_str,
"status_if_new": get_mailchimp_subscription_status(user)
})
influx_metric("mailchimp_requests", {"count": 1},
method="create_or_update")
self.mailchimp_api_requests += 1
# Tell MailChimp to add any tags that we added locally.
if len(tags_to_add) > 0:
tag_names = list(map(lambda tag: tag.name, tags_to_add))
if verbose:
print(
f"Sending request to add tags {tag_names} to {email_str}"
)
client.lists.members.tags.add(list_key_id, email_hash,
tag_names)
influx_metric("mailchimp_requests", {"count": 1},
method="add_tags")
self.mailchimp_api_requests += 1
# Tell MailChimp to remove any tags that we removed locally.
if len(tags_to_remove) > 0:
tag_names = list(map(lambda tag: tag.name, tags_to_remove))
if verbose:
print(
f"Sending request to remove tags {tag_names} from {email_str}"
)
client.lists.members.tags.delete(list_key_id, email_hash,
tag_names)
influx_metric("mailchimp_requests", {"count": 1},
method="delete_tags")
self.mailchimp_api_requests += 1
except Exception as e:
print("Failed to contact MailChimp API: %s" % e, flush=True)
influx_metric("mailchimp_request_failures", {"count": 1})
def list(self, request, *args, **kwargs):
error = None
try:
queryset = self.get_queryset()
serializer = self.get_serializer(queryset, many=True)
return Response([d for d in serializer.data if d])
except QueryDataNotAvailableException as e:
error = type(e).__name__
return Response(status=status.HTTP_202_ACCEPTED)
except Exception as e:
error = type(e).__name__
raise e
finally:
influx.influx_metric("hsreplaynet_partner_api", {"count": 1},
view="Cards",
application=request.auth.application,
error=error)
def customer_subscription_created_handler(event, event_data, event_type,
event_subtype):
if event.customer and event.customer.subscriber:
user = event.customer.subscriber
log.info("Received premium purchased signal for user: %s" %
user.username)
log.info("Scheduling personalized stats for immediate cache warming.")
context = PremiumUserCacheWarmingContext.from_user(user)
fields = {"count": 1, "user": user.username}
influx_metric(
"premium_purchase_cache_warm_event",
fields,
)
warm_redshift_cache_for_user_context(context)
def reap_upload_events_asof(year, month, day, hour):
success_reaping_delay = settings.SUCCESSFUL_UPLOAD_EVENT_REAPING_DELAY_DAYS
nonsuccess_reaping_delay = settings.UNSUCCESSFUL_UPLOAD_EVENT_REAPING_DELAY_DAYS
cursor = connection.cursor()
args = (year, month, day, hour, success_reaping_delay, nonsuccess_reaping_delay,)
# Note: this stored proc will only delete the DB records
# The objects in S3 will age out naturally after 90 days
# according to our bucket's object lifecycle policy
cursor.callproc("reap_upload_events", args)
result_row = cursor.fetchone()
successful_reaped = result_row[0]
unsuccessful_reaped = result_row[1]
influx_metric("upload_events_reaped", fields={
"successful_reaped": successful_reaped,
"unsuccessful_reaped": unsuccessful_reaped
})
cursor.close()
def parse_upload_event(upload_event, meta):
orig_match_start = dateutil_parse(meta["match_start"])
match_start = get_valid_match_start(orig_match_start, upload_event.created)
if match_start != orig_match_start:
upload_event.tainted = True
upload_event.save()
log_bytes = upload_event.log_bytes()
if not log_bytes:
raise ValidationError("The uploaded log file is empty.")
influx_metric("raw_power_log_upload_num_bytes", {"size": len(log_bytes)})
powerlog = StringIO(log_bytes.decode("utf-8"))
upload_event.file.close()
parser = LogParser()
parser._game_state_processor = "GameState"
parser._current_date = match_start
parser.read(powerlog)
return parser
def get(self, request, id):
claim = get_uuid_object_or_404(AccountClaim, id=id)
log.info("Claim %r: Token=%r, User=%r", claim, claim.token, claim.token.user)
if claim.token.user:
if claim.token.user.is_fake:
count = GameReplay.objects.filter(user=claim.token.user).update(user=request.user)
log.info("Updated %r replays. Deleting %r.", count, claim.token.user)
# For now we just delete the fake user, because we are not using it.
claim.token.user.delete()
else:
log.warning("%r is a real user. Deleting %r.", claim.token.user, claim)
# Something's wrong. Get rid of the claim and reject the request.
claim.delete()
influx_metric("hsreplaynet_account_claim", {"success": 0})
return HttpResponseForbidden("This token has already been claimed.")
claim.token.user = request.user
claim.token.save()
# Replays are claimed in AuthToken post_save signal (games.models)
claim.delete()
msg = "You have claimed your account. Yay!"
# XXX: using WARNING as a hack to ignore login/logout messages for now
messages.add_message(request, messages.WARNING, msg)
influx_metric("hsreplaynet_account_claim", {"success": 1})
return redirect(settings.LOGIN_REDIRECT_URL)
def find_or_create_replay(parser, entity_tree, meta, upload_event, global_game, players):
client_handle = meta.get("client_handle") or None
existing_replay = upload_event.game
shortid = existing_replay.shortid if existing_replay else upload_event.shortid
replay_xml_path = _generate_upload_path(global_game.match_start, shortid)
log.debug("Will save replay %r to %r", shortid, replay_xml_path)
# The user that owns the replay
user = upload_event.token.user if upload_event.token else None
friendly_player = players[meta["friendly_player"]]
opponent_revealed_deck = get_opponent_revealed_deck(
entity_tree,
friendly_player.player_id,
global_game.game_type
)
hsreplay_doc = create_hsreplay_document(parser, entity_tree, meta, global_game)
common = {
"global_game": global_game,
"client_handle": client_handle,
"spectator_mode": meta.get("spectator_mode", False),
"reconnecting": meta["reconnecting"],
"friendly_player_id": friendly_player.player_id,
}
defaults = {
"shortid": shortid,
"aurora_password": meta.get("aurora_password", ""),
"spectator_password": meta.get("spectator_password", ""),
"resumable": meta.get("resumable"),
"build": meta["build"],
"upload_token": upload_event.token,
"won": friendly_player.won,
"replay_xml": replay_xml_path,
"hsreplay_version": hsreplay_version,
"hslog_version": hslog_version,
"opponent_revealed_deck": opponent_revealed_deck,
}
# Create and save hsreplay.xml file
# Noop in the database, as it should already be set before the initial save()
xml_file = save_hsreplay_document(hsreplay_doc, shortid, existing_replay)
influx_metric("replay_xml_num_bytes", {"size": xml_file.size})
if existing_replay:
log.debug("Found existing replay %r", existing_replay.shortid)
# Clean up existing replay file
filename = existing_replay.replay_xml.name
if filename and filename != replay_xml_path and default_storage.exists(filename):
# ... but only if it's not the same path as the new one (it'll get overwridden)
log.debug("Deleting %r", filename)
default_storage.delete(filename)
# Now update all the fields
defaults.update(common)
for k, v in defaults.items():
setattr(existing_replay, k, v)
# Save the replay file
existing_replay.replay_xml.save("hsreplay.xml", xml_file, save=False)
# Finally, save to the db and exit early with created=False
existing_replay.save()
return existing_replay, False
# No existing replay, so we assign a default user/visibility to the replay
# (eg. we never update those fields on existing replays)
# We also prepare a webhook for triggering, if there's one.
if user:
defaults["user"] = user
defaults["visibility"] = user.default_replay_visibility
if client_handle:
# Get or create a replay object based on our defaults
replay, created = GameReplay.objects.get_or_create(defaults=defaults, **common)
log.debug("Replay %r has created=%r, client_handle=%r", replay.id, created, client_handle)
else:
# The client_handle is the minimum we require to update an existing replay.
# If we don't have it, we won't try deduplication, we instead get_or_create by shortid.
defaults.update(common)
replay, created = GameReplay.objects.get_or_create(defaults=defaults, shortid=shortid)
log.debug("Replay %r has created=%r (no client_handle)", replay.id, created)
# Save the replay file
replay.replay_xml.save("hsreplay.xml", xml_file, save=False)
if replay.shortid != upload_event.shortid:
# We must ensure an alias for this upload_event.shortid is recorded
# We use get or create in case this is not the first time processing this replay
ReplayAlias.objects.get_or_create(replay=replay, shortid=upload_event.shortid)
if user:
user.trigger_webhooks(replay)
return replay, created
def process_raw_upload(raw_upload, reprocess=False, log_group_name="", log_stream_name=""):
"""
Generic processing logic for raw log files.
"""
logger = logging.getLogger("hsreplaynet.lambdas.process_raw_upload")
obj, created = UploadEvent.objects.get_or_create(
shortid=raw_upload.shortid,
defaults={"status": UploadEventStatus.PENDING}
)
logger.debug("UploadEvent Created: %r", created)
if not created and not reprocess:
# This can occur two ways:
# 1) The client sends the PUT request twice
# 2) Re-enabling processing queues an upload to the stream and the S3 event fires
logger.info("Invocation is an instance of double_put. Exiting Early.")
influx_metric("raw_log_double_put", {
"count": 1,
"shortid": raw_upload.shortid,
"key": raw_upload.log_key
})
return
obj.log_group_name = log_group_name
obj.log_stream_name = log_stream_name
descriptor = raw_upload.descriptor
new_log_key = _generate_upload_key(raw_upload.timestamp, raw_upload.shortid)
new_bucket = settings.AWS_STORAGE_BUCKET_NAME
# Move power.log/descriptor.json to the other bucket if it's needed
raw_upload.prepare_upload_event_log_location(new_bucket, new_log_key)
upload_metadata = descriptor["upload_metadata"]
gateway_headers = descriptor["gateway_headers"]
if "User-Agent" in gateway_headers:
logger.debug("User Agent: %s", gateway_headers["User-Agent"])
else:
logger.debug("User Agent: UNKNOWN")
obj.file = new_log_key
obj.descriptor_data = json.dumps(descriptor)
obj.upload_ip = descriptor["source_ip"]
obj.canary = "canary" in upload_metadata and upload_metadata["canary"]
obj.user_agent = gateway_headers.get("User-Agent", "")[:100]
obj.status = UploadEventStatus.VALIDATING
try:
header = gateway_headers.get("Authorization", "")
token = AuthToken.get_token_from_header(header)
if not token:
msg = "Malformed or Invalid Authorization Header: %r" % (header)
logger.error(msg)
raise ValidationError(msg)
obj.token = token
api_key = gateway_headers.get("X-Api-Key", "")
if not api_key:
raise ValidationError("Missing X-Api-Key header. Please contact us for an API key.")
obj.api_key = APIKey.objects.get(api_key=api_key)
except (ValidationError, APIKey.DoesNotExist) as e:
logger.error("Exception: %r", e)
obj.status = UploadEventStatus.VALIDATION_ERROR
obj.error = e
obj.save()
logger.info("All state successfully saved to UploadEvent with id: %r", obj.id)
# If we get here, now everything is in the DB.
# Clear out the raw upload so it doesn't clog up the pipeline.
raw_upload.delete()
logger.info("Deleting objects from S3 succeeded.")
logger.info("Validation Error will be raised and we will not proceed to processing")
raise
else:
if "test_data" in upload_metadata or obj.token.test_data:
logger.debug("Upload Event Is TEST DATA")
if obj.token.test_data:
# When token.test_data = True, then all UploadEvents are test_data = True
obj.test_data = True
# Only old clients released during beta do not include a user agent
is_unsupported_client = not obj.user_agent
if is_unsupported_client:
logger.info("No UA provided. Marking as unsupported (client too old).")
influx_metric("upload_from_unsupported_client", {
"count": 1,
"shortid": raw_upload.shortid,
"api_key": obj.api_key.full_name
})
obj.status = UploadEventStatus.UNSUPPORTED_CLIENT
obj.save()
logger.debug("Saved: UploadEvent.id = %r", obj.id)
# If we get here, now everything is in the DB.
raw_upload.delete()
logger.debug("Deleting objects from S3 succeeded")
if is_unsupported_client:
# Wait until after we have deleted the raw_upload to exit
# But do not start processing if it's an unsupported client
logger.info("Exiting Without Processing - Unsupported Client")
return
serializer = UploadEventSerializer(obj, data=upload_metadata)
if serializer.is_valid():
logger.debug("UploadEvent passed serializer validation")
obj.status = UploadEventStatus.PROCESSING
serializer.save()
logger.debug("Starting GameReplay processing for UploadEvent")
obj.process()
else:
obj.error = serializer.errors
logger.info("UploadEvent failed validation with errors: %r", obj.error)
obj.status = UploadEventStatus.VALIDATION_ERROR
obj.save()
logger.debug("Done")