def _subscribe(request): email = request.POST["data[email]"] try: email_address = EmailAddress.objects.get(email=email) user = email_address.user user_exists = True primary_email = find_best_email_for_user(user) # If the user exists and the subscription webhook was about their primary email, # update their marketing contact preference to reflect that. if primary_email and primary_email.email == email: if "email" not in user.settings: user.settings["email"] = {"marketing": True} else: user.settings["email"]["marketing"] = True user.save() except EmailAddress.DoesNotExist: user_exists = False primary_email = False influx_metric( "mailchimp_webhook_request", {"value": 1}, type="subscribe", primary_email=primary_email, user_exists=user_exists )
def parse_upload_event(upload_event, meta): orig_match_start = dateutil_parse(meta["match_start"]) match_start = get_valid_match_start(orig_match_start, upload_event.created) if match_start != orig_match_start: upload_event.tainted = True upload_event.save() upload_event.file.open(mode="rb") log_bytes = upload_event.file.read() if not log_bytes: raise ValidationError("The uploaded log file is empty.") influx_metric("raw_power_log_upload_num_bytes", {"size": len(log_bytes)}) powerlog = StringIO(log_bytes.decode("utf-8")) upload_event.file.close() try: parser = InfluxInstrumentedParser(upload_event.shortid, meta) parser._game_state_processor = "GameState" parser._current_date = match_start parser.read(powerlog) except Exception as e: log.exception("Got exception %r while parsing log", e) raise ParsingError(str(e)) # from e if not upload_event.test_data: parser.write_payload() return parser
def post(self, request): marketing_prefs = request.POST.get("marketing", "") == "on" request.user.settings["email"] = { "marketing": marketing_prefs, "updated": timezone.now().isoformat(), } request.user.save() try: client = get_mailchimp_client() email = find_best_email_for_user(request.user) client.lists.members.create_or_update( settings.MAILCHIM_LIST_KEY_ID, get_subscriber_hash(email.email), { "email_address": email.email, "status_if_new": get_mailchimp_subscription_status(request.user) }) except Exception as e: log.warning("Failed to contact MailChimp API: %s" % e) influx_metric("mailchimp_request_failures", {"count": 1}) messages.info(request, _("Your email preferences have been saved.")) return redirect(self.success_url)
def reap_upload_events_asof(year, month, day, hour): success_reaping_delay = settings.SUCCESSFUL_UPLOAD_EVENT_REAPING_DELAY_DAYS nonsuccess_reaping_delay = settings.UNSUCCESSFUL_UPLOAD_EVENT_REAPING_DELAY_DAYS cursor = connections["uploads"].cursor() args = ( year, month, day, hour, success_reaping_delay, nonsuccess_reaping_delay, ) # Note: this stored proc will only delete the DB records # The objects in S3 will age out naturally after 90 days # according to our bucket's object lifecycle policy cursor.callproc("reap_upload_events", args) result_row = cursor.fetchone() successful_reaped = result_row[0] unsuccessful_reaped = result_row[1] influx_metric("upload_events_reaped", fields={ "successful_reaped": successful_reaped, "unsuccessful_reaped": unsuccessful_reaped }) cursor.close()
def reap_orphans_for_date(reaping_date): inventory = get_reaping_inventory_for_date(reaping_date) for hour, hour_inventory in inventory.items(): reaped_orphan_count = 0 for minute, minute_inventory in hour_inventory.items(): for shortid, keys in minute_inventory.items(): descriptor = keys.get("descriptor") if is_safe_to_reap(shortid, keys): log.debug("Reaping Descriptor: %r", descriptor) aws.S3.delete_object( Bucket=settings.S3_RAW_LOG_UPLOAD_BUCKET, Key=keys["descriptor"]) reaped_orphan_count += 1 else: log.debug("Skipping: %r (Unsafe to reap)", descriptor) log.info("A total of %s descriptors reaped for hour: %s" % (str(reaped_orphan_count), str(hour))) # Report count of orphans to Influx fields = {"count": reaped_orphan_count} influx_metric("orphan_descriptors_reaped", fields=fields, timestamp=reaping_date, hour=hour)
def get(self, request, id): claim = get_uuid_object_or_404(AccountClaim, id=id) log.info("Claim %r: Token=%r, User=%r", claim, claim.token, claim.token.user) if claim.token.user: if claim.token.user.is_fake: count = GameReplay.objects.filter( user=claim.token.user).update(user=request.user) log.info("Updated %r replays. Deleting %r.", count, claim.token.user) # For now we just delete the fake user, because we are not using it. claim.token.user.delete() else: log.warning("%r is a real user. Deleting %r.", claim.token.user, claim) # Something's wrong. Get rid of the claim and reject the request. claim.delete() influx_metric("hsreplaynet_account_claim", {"count": 1}, error=1) return HttpResponseForbidden( "This token has already been claimed.") claim.token.user = request.user claim.token.save() # Replays are claimed in AuthToken post_save signal (games.models) claim.delete() messages.info(request, "You have claimed your account. Nice!") influx_metric("hsreplaynet_account_claim", {"count": 1}) return redirect(self.get_redirect_url(request))
def can_subscribe(self): from djpaypal.models import BillingAgreement user = self.request.user if not user.is_authenticated: return False # refresh all pending and active BillingAgreements for agreement in BillingAgreement.objects.filter( user=user, state__in=["Pending", "Active"] ): BillingAgreement.find_and_sync(agreement.id) if user.is_premium: return False # reject if there's any remaining pending or active BillingAgreements stale_agreements = BillingAgreement.objects.filter( user=user, state__in=["Pending", "Active"] ) if stale_agreements.count() > 0: for agreement in stale_agreements: # Log an error influx_metric( "hsreplaynet_paypal_stale_agreement", { "count": "1", "id": agreement.id }, state=agreement.state ) return False return True
def _trigger_if_stale(parameterized_query, run_local=False, priority=None): did_preschedule = False result = False as_of = parameterized_query.result_as_of if as_of is not None: staleness = int((datetime.utcnow() - as_of).total_seconds()) else: staleness = None if parameterized_query.result_is_stale or run_local: attempt_request_triggered_query_execution(parameterized_query, run_local, priority) result = True elif staleness and staleness > settings.MINIMUM_QUERY_REFRESH_INTERVAL: did_preschedule = True parameterized_query.preschedule_refresh() query_fetch_metric_fields = { "count": 1, } if staleness: query_fetch_metric_fields["staleness"] = staleness query_fetch_metric_fields.update( parameterized_query.supplied_non_filters_dict) influx.influx_metric("redshift_response_payload_staleness", query_fetch_metric_fields, query_name=parameterized_query.query_name, did_preschedule=did_preschedule, **parameterized_query.supplied_filters_dict) return result
def finish_async_redshift_query(event, context): """A handler triggered by the arrival of an UNLOAD manifest on S3 The S3 trigger must be configured manually with: prefix = PROD suffix = manifest """ logger = logging.getLogger( "hsreplaynet.lambdas.finish_async_redshift_query") catalogue = get_redshift_catalogue() s3_event = event["Records"][0]["s3"] bucket = s3_event["bucket"]["name"] manifest_key = unquote(s3_event["object"]["key"]) if bucket == settings.S3_UNLOAD_BUCKET: logger.info("Finishing query: %s" % manifest_key) start_time = time.time() parameterized_query = catalogue.refresh_cache_from_s3_manifest_key( manifest_key=manifest_key) query_execute_metric_fields = { "duration_seconds": parameterized_query.most_recent_duration, "unload_seconds": time.time() - start_time, "query_handle": parameterized_query.most_recent_query_handle } query_execute_metric_fields.update( parameterized_query.supplied_non_filters_dict) influx_metric("finished_async_redshift_query", query_execute_metric_fields, query_name=parameterized_query.query_name, **parameterized_query.supplied_filters_dict)
def get(self, request): from djpaypal.models import PreparedBillingAgreement token = request.GET.get("token", "") if not token: return self.fail(_("Unable to complete subscription.")) try: prepared_agreement = PreparedBillingAgreement.objects.get(id=token) except PreparedBillingAgreement.DoesNotExist: return self.fail(_("Invalid subscription token.")) if prepared_agreement.user != self.request.user: return self.fail(_("You are not logged in as the correct user.")) billing_agreement = prepared_agreement.execute() # At this time, we expect the billing agreement to be active # If that isn't the case, PayPal probably needs a moment to complete the payment # For now, let's just send a metric influx_metric("hsreplaynet_paypal_agreement_state", { "count": "1", "id": billing_agreement.id }, state=billing_agreement.state) # Null out the premium checkout timestamp so that we don't remind the user to # complete the checkout process. request.user.last_premium_checkout = None request.user.save() messages.success(self.request, _("You are now subscribed with PayPal!")) return redirect(self.get_success_url())
def do_execute_redshift_query(query_name, supplied_params, queue_name): """A common entry point for the actual execution of queries on Lambda""" logger = logging.getLogger("hsreplaynet.lambdas.execute_redshift_query") logger.info("Query Name: %s" % query_name) logger.info("Query Params: %s" % supplied_params) query = get_redshift_catalogue().get_query(query_name) parameterized_query = query.build_full_params(supplied_params) try: wlm_queue = settings.REDSHIFT_QUERY_QUEUES[queue_name]["wlm_queue"] with get_concurrent_redshift_query_queue_semaphore(queue_name): _do_execute_query(parameterized_query, wlm_queue) logger.info("Query Execution Complete") return True except NotAvailable: logger.warn( "The Redshift query queue was already at max concurrency. Skipping query." ) metric_fields = {"count": 1} metric_fields.update(parameterized_query.supplied_non_filters_dict) influx_metric("redshift_query_lambda_execution_concurrency_exceeded", metric_fields, query_name=query_name, **parameterized_query.supplied_filters_dict) return False
def list(self, request, *args, **kwargs): error = None try: queryset = self.filter_queryset(self.get_queryset()) # We pull back all users with matching account los, which may result in # duplicates across region, so we need to do region filtering on the result set # at the application level. It can't be pushed down to the database, since # Django doesn't natively support filtering on tuples as part of an IN clause. serializer_context = self.get_serializer_context() def match_region(r): return ( r.region, r.account_lo) in serializer_context["redshift_query_data"] region_filtered_queryset = filter(match_region, queryset) serializer = self.get_serializer(region_filtered_queryset, many=True) return Response(serializer.data) except QueryDataNotAvailableException as e: error = type(e).__name__ return Response(status=status.HTTP_202_ACCEPTED) except Exception as e: error = type(e).__name__ raise e finally: influx.influx_metric("hsreplaynet_leaderboard_api", {"count": 1}, error=error)
def classify_into_archetype(self, player_class, save: bool = True) -> int: game_format = self.format signature_weights = ClusterSnapshot.objects.get_signature_weights( game_format, player_class) sig_archetype_id = classify_deck(self.dbf_map(), signature_weights) # New Style Deck Prediction nn_archetype_id = None # nn_archetype_id = ClusterSetSnapshot.objects.predict_archetype_id( # player_class, # game_format, # self, # ) archetype_id = sig_archetype_id or nn_archetype_id influx_metric("archetype_prediction_outcome", { "count": 1, "signature_weight_archetype_id": sig_archetype_id, "neural_net_archetype_id": nn_archetype_id, "archetype_id": archetype_id, "deck_id": self.id, }, success=archetype_id is not None, signature_weight_success=sig_archetype_id is not None, neural_net_success=nn_archetype_id is not None, method_agreement=sig_archetype_id == nn_archetype_id, player_class=player_class.name, game_format=game_format.name) if archetype_id and save: self.update_archetype(archetype_id) return archetype_id
def post(self, request): serializer = self.serializer_class(data=request.data) serializer.is_valid(raise_exception=True) assert not request.user.is_fake token = get_uuid_object_or_404(AuthToken, key=serializer.validated_data["token"]) if token.user and not token.user.is_fake: influx_metric("hsreplaynet_account_claim", {"count": 1}, error=1) raise ValidationError({ "error": "token_already_claimed", "detail": "This token has already been claimed." }) with transaction.atomic(): # Update the token's replays to match the new user replays_claimed = GameReplay.objects.filter(user=token.user).update(user=request.user) if token.user: # Delete the token-specific fake user # If there was no user attached to the token, claiming it will attach the correct user. token.user.delete() # Update the token's actual user token.user = request.user # Save. token.save() influx_metric("hsreplaynet_account_claim", {"count": 1, "replays": replays_claimed}) return Response(status=HTTP_204_NO_CONTENT)
def _unsubscribe(request): email = request.POST["data[email]"] try: email_address = EmailAddress.objects.get(email=email) user = email_address.user user_exists = True primary_email = find_best_email_for_user(user) # If the user exists and the subscription update was related to their primary # email address, update their marketing contact preference. (We don't want to # update the preference if their non-primary email address was unsubscribed, # because that won't really reflect the actual state of their preference.) if "email" in user.settings and primary_email and primary_email.email == email: user.settings["email"]["marketing"] = False user.save() except EmailAddress.DoesNotExist: user_exists = False primary_email = False influx_metric( "mailchimp_webhook_request", {"value": 1}, reason=request.POST.get("data[reason]"), type="unsubscribe", primary_email=primary_email, user_exists=user_exists )
def _log_mailchimp_error(e): # Log a warning, an InfluxDB metric, and a Sentry alert. log.warning("Failed to contact MailChimp API: %s" % e) influx_metric("mailchimp_request_failures", {"count": 1}) error_handler(e)
def reap_orphans_for_date(reaping_date): inventory = get_reaping_inventory_for_date(reaping_date) for hour, hour_inventory in inventory.items(): reaped_orphan_count = 0 for minute, minute_inventory in hour_inventory.items(): for shortid, keys in minute_inventory.items(): if is_safe_to_reap(shortid, keys): log.info("Reaping Descriptor: %r", keys["descriptor"]) aws.S3.delete_object( Bucket=settings.S3_RAW_LOG_UPLOAD_BUCKET, Key=keys["descriptor"] ) reaped_orphan_count += 1 else: log.info("Skipping Descriptor: %r (Unsafe To Reap)", keys["descriptor"]) log.info( "A total of %s descriptors reaped for hour: %s" % ( str(reaped_orphan_count), str(hour) ) ) # Report count of orphans to Influx fields = { "count": reaped_orphan_count } influx_metric( "orphan_descriptors_reaped", fields=fields, timestamp=reaping_date, hour=hour )
def _do_execute_query_work(parameterized_query, wlm_queue=None): if not parameterized_query.result_is_stale: log.info( "Up-to-date cached data exists. Exiting without running query.") else: log.info("Cached data missing or stale. Executing query now.") # DO EXPENSIVE WORK start_ts = time.time() exception_raised = False exception_msg = None try: parameterized_query.refresh_result(wlm_queue) except Exception as e: exception_raised = True exception_msg = str(e) raise finally: end_ts = time.time() duration_seconds = round(end_ts - start_ts, 2) query_execute_metric_fields = { "duration_seconds": duration_seconds, "exception_message": exception_msg } query_execute_metric_fields.update( parameterized_query.supplied_non_filters_dict) influx_metric("redshift_query_execute", query_execute_metric_fields, exception_thrown=exception_raised, query_name=parameterized_query.query_name, **parameterized_query.supplied_filters_dict)
def _unsubscribe(request): email = request.POST["data[email]"] try: email_address = EmailAddress.objects.get(email=email) user = email_address.user user_exists = True # If the user exists, update their marketing contact preference. if "email" in user.settings: user.settings["email"]["marketing"] = False user.save() except EmailAddress.DoesNotExist: user_exists = False influx_metric( "mailchimp_webhook_request", {"value": 1}, reason=request.POST.get("data[reason]"), type="unsubscribe", user_exists=user_exists )
def classify_into_archetype(self, player_class, save: bool = True) -> int: game_format = self.format signature_weights = ClusterSnapshot.objects.get_signature_weights( game_format, player_class) blocked_classifications = {} # A failure callback for classify_deck so that we can get notified when a deck can't # be classified to an archetype because it's missing a card or fails a rule check. # This callback stores the results so that we can instrument them below. def record_classification_failure(failure_description): reason = failure_description.get("reason") if reason: if reason in blocked_classifications: blocked_classifications[ reason] = blocked_classifications[reason] + 1 else: blocked_classifications[reason] = 1 sig_archetype_id = classify_deck( self.dbf_map(), signature_weights, failure_callback=record_classification_failure) # Instrument any failures that occurred during the classification attempt. for block_reason, count in blocked_classifications.items(): influx_metric("archetype_classification_blocked", {"count": count}, reason=block_reason) # New Style Deck Prediction nn_archetype_id = None # nn_archetype_id = ClusterSetSnapshot.objects.predict_archetype_id( # player_class, # game_format, # self, # ) archetype_id = sig_archetype_id or nn_archetype_id influx_metric("archetype_prediction_outcome", { "count": 1, "signature_weight_archetype_id": sig_archetype_id, "neural_net_archetype_id": nn_archetype_id, "archetype_id": archetype_id, "deck_id": self.id, }, success=archetype_id is not None, signature_weight_success=sig_archetype_id is not None, neural_net_success=nn_archetype_id is not None, method_agreement=sig_archetype_id == nn_archetype_id, player_class=player_class.name, game_format=game_format.name) if archetype_id and save: self.update_archetype(archetype_id) return archetype_id
def post(self, request): # Record reason and message in influx influx_metric("hsreplaynet_replays_delete", {"count": 1}) request.user.replays.update(is_deleted=True) messages.info(self.request, _("Your replays have been deleted.")) return redirect(self.success_url)
def refresh_stale_redshift_queries(event, context): """A cron'd handler that attempts to refresh queries queued in Redis""" logger = logging.getLogger( "hsreplaynet.lambdas.refresh_stale_redshift_queries") start_time = time.time() logger.info("Start Time: %s" % str(start_time)) catalogue = get_redshift_catalogue() scheduler = catalogue.scheduler target_duration_seconds = 55 duration = 0 for queue_name in scheduler.query_queues: influx_metric( "redshift_refresh_queue_depth", {"depth": scheduler.queue_size(queue_name)}, queue_name=queue_name, ) # We run for 55 seconds, since the majority of queries take < 5 seconds to finish # And the next scheduled invocation of this will be starting a minute after this one. while duration < target_duration_seconds: logger.info("Runtime duration: %s" % str(duration)) available_slots = scheduler.get_available_slots( ) - scheduler.get_queued_query_count() logger.info("Available cluster slots: %s" % str(available_slots)) if available_slots <= 1: sleep_duration = 5 logger.info("Sleeping for %i until more slots are available" % sleep_duration) # If only 1 slot remains leave it for ETL or an IMMEDIATE query. time.sleep(sleep_duration) current_time = time.time() duration = current_time - start_time continue remaining_seconds = target_duration_seconds - duration if remaining_seconds < 1: break logger.info("Will block for queued query for %s seconds" % str(remaining_seconds)) try: refreshed_query = scheduler.refresh_next_pending_query( block_for=remaining_seconds, # This uses available cluster resources to refresh queries early force=REDSHIFT_PREEMPTIVELY_REFRESH_QUERIES) if refreshed_query: logger.info("Refreshed: %s" % refreshed_query.cache_key) else: logger.info("Nothing to refresh.") except Exception: logger.error("Error refreshing query") sentry.captureException() current_time = time.time() duration = current_time - start_time
def _fetch_query_results(parameterized_query, run_local=False, user=None, priority=None): cache_is_populated = parameterized_query.cache_is_populated is_cache_hit = parameterized_query.result_available triggered_refresh = False if is_cache_hit: triggered_refresh = trigger_if_stale(parameterized_query, run_local, priority) response = HttpResponse( content=parameterized_query.response_payload_data, content_type=parameterized_query.response_payload_type) elif cache_is_populated and parameterized_query.is_global: if parameterized_query.is_backfillable and parameterized_query.is_personalized: # Premium users should have cache entries even if the result set is empty # So we should only reach this block if the user just subscribed # And we haven't rerun the global query yet. triggered_refresh = True attempt_request_triggered_query_execution(parameterized_query, run_local, priority) result = {"msg": "Query is processing. Check back later."} response = JsonResponse(result, status=202) else: # There is no content for this permutation of parameters # For deck related queries this most likely means that someone hand crafted the URL # Or if it's a card related query, then it's a corner case where there is no data response = HttpResponse(status=204) else: # The cache is not populated yet for this query. # Perhaps it's a new query or perhaps the cache was recently flushed. # So attempt to trigger populating it attempt_request_triggered_query_execution(parameterized_query, run_local, priority) result = {"msg": "Query is processing. Check back later."} response = JsonResponse(result, status=202) log.info("Query: %s Cache Populated: %s Cache Hit: %s Is Stale: %s" % (cache_is_populated, parameterized_query.cache_key, is_cache_hit, triggered_refresh)) query_fetch_metric_fields = { "count": 1, } query_fetch_metric_fields.update( parameterized_query.supplied_non_filters_dict) influx.influx_metric("redshift_query_fetch", query_fetch_metric_fields, cache_populated=cache_is_populated, cache_hit=is_cache_hit, query_name=parameterized_query.query_name, triggered_refresh=triggered_refresh, **parameterized_query.supplied_filters_dict) return response
def create(self, validated_data): validated_data["user"] = self.context["request"].user ret = WebhookEndpoint.objects.create(**validated_data) influx_metric("hsreplaynet_webhook_create", { "source": "api-oauth2", "client_id": self.context["request"].auth.application.client_id, "uuid": str(ret.uuid), }) return ret
def new_user(self, request, sociallogin): battletag = sociallogin.account.extra_data.get("battletag", "") influx_metric( "hsreplaynet_socialauth_signup", {"count": 1}, provider=sociallogin.account.provider, region=sociallogin.account.extra_data.get("region", "unknown"), has_battletag=bool(battletag), ) ret = super().new_user(request, sociallogin) ret.battletag = battletag or "" return ret
def process_upload_event(upload_event): """ Wrapper around do_process_upload_event() to set the event's status and error/traceback as needed. """ upload_event.error = "" upload_event.traceback = "" if upload_event.status != UploadEventStatus.PROCESSING: upload_event.status = UploadEventStatus.PROCESSING upload_event.save() try: replay, do_flush_exporter = do_process_upload_event(upload_event) except Exception as e: from traceback import format_exc upload_event.error = str(e) upload_event.traceback = format_exc() upload_event.status, reraise = handle_upload_event_exception(e, upload_event) metric_fields = {"count": 1} if upload_event.game: metric_fields["shortid"] = str(upload_event.game.shortid) influx_metric( "upload_event_exception", metric_fields, error=upload_event.status.name.lower() ) upload_event.save() if reraise: raise else: return else: upload_event.game = replay upload_event.status = UploadEventStatus.SUCCESS upload_event.save() try: with influx_timer("redshift_exporter_flush_duration"): do_flush_exporter() except Exception as e: # Don't fail on this error_handler(e) influx_metric( "flush_redshift_exporter_error", { "count": 1, "error": str(e) } ) return replay
def replay_meets_recency_requirements(upload_event, global_game): # We only load games in where the match_start date is within +/ 36 hours from # The upload_date. This filters out really old replays people might upload # Or replays from users with crazy system clocks. # The purpose of this filtering is to do reduce variability and thrash in our vacuuming # If we determine that vacuuming is not a bottleneck than we can consider # relaxing this requirement. meets_requirements, diff_hours = _dates_within_threshold( global_game.match_start, upload_event.log_upload_date, settings.REDSHIFT_ETL_UPLOAD_DELAY_LIMIT_HOURS ) if not meets_requirements: influx_metric("replay_failed_recency_requirement", {"count": 1, "diff": diff_hours}) return meets_requirements
def _handle_request(self, request): # Ensure that we're being called back with our secret key. if request.GET.get("key") != settings.MAILCHIMP_WEBHOOK_KEY: return HttpResponse(status=HTTP_403_FORBIDDEN) event_type = request.POST.get("type") if event_type == "unsubscribe" and request.POST.get("data[action]") == "unsub": self._unsubscribe(request) elif event_type: influx_metric("mailchimp_webhook_request", {"value": 1}, type=event_type) return HttpResponse(status=HTTP_200_OK)
def _publish_tag_changes(self, user, email_str, tags_to_add, tags_to_remove, verbose=False): list_key_id = settings.MAILCHIMP_LIST_KEY_ID email_hash = get_subscriber_hash(email_str) client = get_mailchimp_client() # We may never have seen this user's email address before or sent it to MailChimp, # so do a defensive subscriber creation. try: client.lists.members.create_or_update( list_key_id, email_hash, { "email_address": email_str, "status_if_new": get_mailchimp_subscription_status(user) }) influx_metric("mailchimp_requests", {"count": 1}, method="create_or_update") self.mailchimp_api_requests += 1 # Tell MailChimp to add any tags that we added locally. if len(tags_to_add) > 0: tag_names = list(map(lambda tag: tag.name, tags_to_add)) if verbose: print( f"Sending request to add tags {tag_names} to {email_str}" ) client.lists.members.tags.add(list_key_id, email_hash, tag_names) influx_metric("mailchimp_requests", {"count": 1}, method="add_tags") self.mailchimp_api_requests += 1 # Tell MailChimp to remove any tags that we removed locally. if len(tags_to_remove) > 0: tag_names = list(map(lambda tag: tag.name, tags_to_remove)) if verbose: print( f"Sending request to remove tags {tag_names} from {email_str}" ) client.lists.members.tags.delete(list_key_id, email_hash, tag_names) influx_metric("mailchimp_requests", {"count": 1}, method="delete_tags") self.mailchimp_api_requests += 1 except Exception as e: print("Failed to contact MailChimp API: %s" % e, flush=True) influx_metric("mailchimp_request_failures", {"count": 1})
def list(self, request, *args, **kwargs): error = None try: queryset = self.get_queryset() serializer = self.get_serializer(queryset, many=True) return Response([d for d in serializer.data if d]) except QueryDataNotAvailableException as e: error = type(e).__name__ return Response(status=status.HTTP_202_ACCEPTED) except Exception as e: error = type(e).__name__ raise e finally: influx.influx_metric("hsreplaynet_partner_api", {"count": 1}, view="Cards", application=request.auth.application, error=error)
def customer_subscription_created_handler(event, event_data, event_type, event_subtype): if event.customer and event.customer.subscriber: user = event.customer.subscriber log.info("Received premium purchased signal for user: %s" % user.username) log.info("Scheduling personalized stats for immediate cache warming.") context = PremiumUserCacheWarmingContext.from_user(user) fields = {"count": 1, "user": user.username} influx_metric( "premium_purchase_cache_warm_event", fields, ) warm_redshift_cache_for_user_context(context)
def reap_upload_events_asof(year, month, day, hour): success_reaping_delay = settings.SUCCESSFUL_UPLOAD_EVENT_REAPING_DELAY_DAYS nonsuccess_reaping_delay = settings.UNSUCCESSFUL_UPLOAD_EVENT_REAPING_DELAY_DAYS cursor = connection.cursor() args = (year, month, day, hour, success_reaping_delay, nonsuccess_reaping_delay,) # Note: this stored proc will only delete the DB records # The objects in S3 will age out naturally after 90 days # according to our bucket's object lifecycle policy cursor.callproc("reap_upload_events", args) result_row = cursor.fetchone() successful_reaped = result_row[0] unsuccessful_reaped = result_row[1] influx_metric("upload_events_reaped", fields={ "successful_reaped": successful_reaped, "unsuccessful_reaped": unsuccessful_reaped }) cursor.close()
def parse_upload_event(upload_event, meta): orig_match_start = dateutil_parse(meta["match_start"]) match_start = get_valid_match_start(orig_match_start, upload_event.created) if match_start != orig_match_start: upload_event.tainted = True upload_event.save() log_bytes = upload_event.log_bytes() if not log_bytes: raise ValidationError("The uploaded log file is empty.") influx_metric("raw_power_log_upload_num_bytes", {"size": len(log_bytes)}) powerlog = StringIO(log_bytes.decode("utf-8")) upload_event.file.close() parser = LogParser() parser._game_state_processor = "GameState" parser._current_date = match_start parser.read(powerlog) return parser
def get(self, request, id): claim = get_uuid_object_or_404(AccountClaim, id=id) log.info("Claim %r: Token=%r, User=%r", claim, claim.token, claim.token.user) if claim.token.user: if claim.token.user.is_fake: count = GameReplay.objects.filter(user=claim.token.user).update(user=request.user) log.info("Updated %r replays. Deleting %r.", count, claim.token.user) # For now we just delete the fake user, because we are not using it. claim.token.user.delete() else: log.warning("%r is a real user. Deleting %r.", claim.token.user, claim) # Something's wrong. Get rid of the claim and reject the request. claim.delete() influx_metric("hsreplaynet_account_claim", {"success": 0}) return HttpResponseForbidden("This token has already been claimed.") claim.token.user = request.user claim.token.save() # Replays are claimed in AuthToken post_save signal (games.models) claim.delete() msg = "You have claimed your account. Yay!" # XXX: using WARNING as a hack to ignore login/logout messages for now messages.add_message(request, messages.WARNING, msg) influx_metric("hsreplaynet_account_claim", {"success": 1}) return redirect(settings.LOGIN_REDIRECT_URL)
def find_or_create_replay(parser, entity_tree, meta, upload_event, global_game, players): client_handle = meta.get("client_handle") or None existing_replay = upload_event.game shortid = existing_replay.shortid if existing_replay else upload_event.shortid replay_xml_path = _generate_upload_path(global_game.match_start, shortid) log.debug("Will save replay %r to %r", shortid, replay_xml_path) # The user that owns the replay user = upload_event.token.user if upload_event.token else None friendly_player = players[meta["friendly_player"]] opponent_revealed_deck = get_opponent_revealed_deck( entity_tree, friendly_player.player_id, global_game.game_type ) hsreplay_doc = create_hsreplay_document(parser, entity_tree, meta, global_game) common = { "global_game": global_game, "client_handle": client_handle, "spectator_mode": meta.get("spectator_mode", False), "reconnecting": meta["reconnecting"], "friendly_player_id": friendly_player.player_id, } defaults = { "shortid": shortid, "aurora_password": meta.get("aurora_password", ""), "spectator_password": meta.get("spectator_password", ""), "resumable": meta.get("resumable"), "build": meta["build"], "upload_token": upload_event.token, "won": friendly_player.won, "replay_xml": replay_xml_path, "hsreplay_version": hsreplay_version, "hslog_version": hslog_version, "opponent_revealed_deck": opponent_revealed_deck, } # Create and save hsreplay.xml file # Noop in the database, as it should already be set before the initial save() xml_file = save_hsreplay_document(hsreplay_doc, shortid, existing_replay) influx_metric("replay_xml_num_bytes", {"size": xml_file.size}) if existing_replay: log.debug("Found existing replay %r", existing_replay.shortid) # Clean up existing replay file filename = existing_replay.replay_xml.name if filename and filename != replay_xml_path and default_storage.exists(filename): # ... but only if it's not the same path as the new one (it'll get overwridden) log.debug("Deleting %r", filename) default_storage.delete(filename) # Now update all the fields defaults.update(common) for k, v in defaults.items(): setattr(existing_replay, k, v) # Save the replay file existing_replay.replay_xml.save("hsreplay.xml", xml_file, save=False) # Finally, save to the db and exit early with created=False existing_replay.save() return existing_replay, False # No existing replay, so we assign a default user/visibility to the replay # (eg. we never update those fields on existing replays) # We also prepare a webhook for triggering, if there's one. if user: defaults["user"] = user defaults["visibility"] = user.default_replay_visibility if client_handle: # Get or create a replay object based on our defaults replay, created = GameReplay.objects.get_or_create(defaults=defaults, **common) log.debug("Replay %r has created=%r, client_handle=%r", replay.id, created, client_handle) else: # The client_handle is the minimum we require to update an existing replay. # If we don't have it, we won't try deduplication, we instead get_or_create by shortid. defaults.update(common) replay, created = GameReplay.objects.get_or_create(defaults=defaults, shortid=shortid) log.debug("Replay %r has created=%r (no client_handle)", replay.id, created) # Save the replay file replay.replay_xml.save("hsreplay.xml", xml_file, save=False) if replay.shortid != upload_event.shortid: # We must ensure an alias for this upload_event.shortid is recorded # We use get or create in case this is not the first time processing this replay ReplayAlias.objects.get_or_create(replay=replay, shortid=upload_event.shortid) if user: user.trigger_webhooks(replay) return replay, created
def process_raw_upload(raw_upload, reprocess=False, log_group_name="", log_stream_name=""): """ Generic processing logic for raw log files. """ logger = logging.getLogger("hsreplaynet.lambdas.process_raw_upload") obj, created = UploadEvent.objects.get_or_create( shortid=raw_upload.shortid, defaults={"status": UploadEventStatus.PENDING} ) logger.debug("UploadEvent Created: %r", created) if not created and not reprocess: # This can occur two ways: # 1) The client sends the PUT request twice # 2) Re-enabling processing queues an upload to the stream and the S3 event fires logger.info("Invocation is an instance of double_put. Exiting Early.") influx_metric("raw_log_double_put", { "count": 1, "shortid": raw_upload.shortid, "key": raw_upload.log_key }) return obj.log_group_name = log_group_name obj.log_stream_name = log_stream_name descriptor = raw_upload.descriptor new_log_key = _generate_upload_key(raw_upload.timestamp, raw_upload.shortid) new_bucket = settings.AWS_STORAGE_BUCKET_NAME # Move power.log/descriptor.json to the other bucket if it's needed raw_upload.prepare_upload_event_log_location(new_bucket, new_log_key) upload_metadata = descriptor["upload_metadata"] gateway_headers = descriptor["gateway_headers"] if "User-Agent" in gateway_headers: logger.debug("User Agent: %s", gateway_headers["User-Agent"]) else: logger.debug("User Agent: UNKNOWN") obj.file = new_log_key obj.descriptor_data = json.dumps(descriptor) obj.upload_ip = descriptor["source_ip"] obj.canary = "canary" in upload_metadata and upload_metadata["canary"] obj.user_agent = gateway_headers.get("User-Agent", "")[:100] obj.status = UploadEventStatus.VALIDATING try: header = gateway_headers.get("Authorization", "") token = AuthToken.get_token_from_header(header) if not token: msg = "Malformed or Invalid Authorization Header: %r" % (header) logger.error(msg) raise ValidationError(msg) obj.token = token api_key = gateway_headers.get("X-Api-Key", "") if not api_key: raise ValidationError("Missing X-Api-Key header. Please contact us for an API key.") obj.api_key = APIKey.objects.get(api_key=api_key) except (ValidationError, APIKey.DoesNotExist) as e: logger.error("Exception: %r", e) obj.status = UploadEventStatus.VALIDATION_ERROR obj.error = e obj.save() logger.info("All state successfully saved to UploadEvent with id: %r", obj.id) # If we get here, now everything is in the DB. # Clear out the raw upload so it doesn't clog up the pipeline. raw_upload.delete() logger.info("Deleting objects from S3 succeeded.") logger.info("Validation Error will be raised and we will not proceed to processing") raise else: if "test_data" in upload_metadata or obj.token.test_data: logger.debug("Upload Event Is TEST DATA") if obj.token.test_data: # When token.test_data = True, then all UploadEvents are test_data = True obj.test_data = True # Only old clients released during beta do not include a user agent is_unsupported_client = not obj.user_agent if is_unsupported_client: logger.info("No UA provided. Marking as unsupported (client too old).") influx_metric("upload_from_unsupported_client", { "count": 1, "shortid": raw_upload.shortid, "api_key": obj.api_key.full_name }) obj.status = UploadEventStatus.UNSUPPORTED_CLIENT obj.save() logger.debug("Saved: UploadEvent.id = %r", obj.id) # If we get here, now everything is in the DB. raw_upload.delete() logger.debug("Deleting objects from S3 succeeded") if is_unsupported_client: # Wait until after we have deleted the raw_upload to exit # But do not start processing if it's an unsupported client logger.info("Exiting Without Processing - Unsupported Client") return serializer = UploadEventSerializer(obj, data=upload_metadata) if serializer.is_valid(): logger.debug("UploadEvent passed serializer validation") obj.status = UploadEventStatus.PROCESSING serializer.save() logger.debug("Starting GameReplay processing for UploadEvent") obj.process() else: obj.error = serializer.errors logger.info("UploadEvent failed validation with errors: %r", obj.error) obj.status = UploadEventStatus.VALIDATION_ERROR obj.save() logger.debug("Done")