def predict_archetype_id(self, deck): event = self._to_prediction_event(deck) if settings.USE_ARCHETYPE_PREDICTION_LAMBDA or settings.ENV_AWS: with influx_timer("callout_to_predict_deck_archetype"): response = LAMBDA.invoke( FunctionName="predict_deck_archetype", InvocationType="RequestResponse", # Synchronous invocation Payload=json.dumps(event), ) if response[ "StatusCode"] == 200 and "FunctionError" not in response: result = json.loads( response["Payload"].read().decode("utf8")) else: raise RuntimeError( response["Payload"].read().decode("utf8")) else: from keras_handler import handler result = handler(event, None) predicted_class = result["predicted_class"] id_encoding = self.one_hot_external_ids(inverse=True) predicted_archetype_id = id_encoding[predicted_class] if predicted_archetype_id == -1: return None else: return predicted_archetype_id
def validate_parser(parser, meta): # Validate upload if len(parser.games) != 1: raise ValidationError("Expected exactly 1 game, got %i" % (len(parser.games))) packet_tree = parser.games[0] with influx_timer("replay_exporter_duration"): exporter = RedshiftPublishingExporter( packet_tree, stream_prefix=fetch_active_stream_prefix() ).export() game = exporter.game if len(game.players) != 2: raise ValidationError("Expected 2 players, found %i" % (len(game.players))) for player in game.players: # Set the player's name player.name = parser.games[0].manager.get_player_by_id(player.id).name if player.name is None: # If it's None, this is an unsupported replay. log.error("Cannot find player %i name. Replay not supported.", player.player_id) raise GameTooShort("The game was too short to parse correctly") heroes = list(player.heroes) if not heroes: raise UnsupportedReplay("No hero found for player %r" % (player.name)) player._hero = heroes[0] try: db_hero = Card.objects.get(id=player._hero.card_id) except Card.DoesNotExist: raise UnsupportedReplay("Hero %r not found." % (player._hero)) if db_hero.type != CardType.HERO: raise ValidationError("%r is not a valid hero." % (player._hero)) friendly_player_id = packet_tree.export(cls=FriendlyPlayerExporter) if friendly_player_id: meta["friendly_player"] = friendly_player_id elif "friendly_player" not in meta: raise ValidationError("Friendly player ID not present at upload and could not guess it.") # We ignore "reconnecting" from the API, we only trust the log. # if "reconnecting" not in meta: # meta["reconnecting"] = False # There are two ways of identifying a reconnected game: # In reconnected games, the initial CREATE_GAME packet contains a STEP and STATE value. # In older versions of HS (pre-13xxx), STATE is RUNNING even in the CREATE_GAME packet. # Thankfully, looking at STEP is consistent across all versions, so we use that. # It will be Step.INVALID if it's NOT a reconnected game. meta["reconnecting"] = not not game.initial_step # Add the start/end time to meta dict meta["start_time"] = packet_tree.start_time meta["end_time"] = packet_tree.end_time return game, exporter
def get_game_info(global_game, replay): player1 = replay.player(1) player2 = replay.player(2) with influx_timer("generate_redshift_player_decklists_duration"): player1_decklist = player1.deck_list.as_dbf_json() player2_decklist = player2.deck_list.as_dbf_json() if settings.REDSHIFT_USE_MATCH_START_AS_GAME_DATE and global_game.match_start: game_date = global_game.match_start.date() else: game_date = timezone.now().date() game_info = { "game_id": int(global_game.id), "shortid": replay.shortid, "game_type": int(global_game.game_type), "scenario_id": global_game.scenario_id, "ladder_season": global_game.ladder_season, "brawl_season": global_game.brawl_season, "game_date": game_date, "players": { "1": { "deck_id": int(player1.deck_list.id), "archetype_id": get_archetype_id(player1), "deck_list": player1_decklist, "rank": 0 if player1.legend_rank else player1.rank if player1.rank else -1, "legend_rank": player1.legend_rank, "full_deck_known": player1.deck_list.size == 30 }, "2": { "deck_id": int(player2.deck_list.id), "archetype_id": get_archetype_id(player2), "deck_list": player2_decklist, "rank": 0 if player2.legend_rank else player2.rank if player2.rank else -1, "legend_rank": player2.legend_rank, "full_deck_known": player2.deck_list.size == 30, }, } } return game_info
def process_upload_event(upload_event): """ Wrapper around do_process_upload_event() to set the event's status and error/traceback as needed. """ upload_event.error = "" upload_event.traceback = "" if upload_event.status != UploadEventStatus.PROCESSING: upload_event.status = UploadEventStatus.PROCESSING upload_event.save() try: replay, do_flush_exporter = do_process_upload_event(upload_event) except Exception as e: from traceback import format_exc upload_event.error = str(e) upload_event.traceback = format_exc() upload_event.status, reraise = handle_upload_event_exception(e, upload_event) metric_fields = {"count": 1} if upload_event.game: metric_fields["shortid"] = str(upload_event.game.shortid) influx_metric( "upload_event_exception", metric_fields, error=upload_event.status.name.lower() ) upload_event.save() if reraise: raise else: return else: upload_event.game = replay upload_event.status = UploadEventStatus.SUCCESS upload_event.save() try: with influx_timer("redshift_exporter_flush_duration"): do_flush_exporter() except Exception as e: # Don't fail on this error_handler(e) influx_metric( "flush_redshift_exporter_error", { "count": 1, "error": str(e) } ) return replay
def do_flush_exporter(): # Only if we were able to claim the advisory lock do we proceed here. if can_attempt_redshift_load: log.debug( "Redshift lock acquired. Will attempt to flush to redshift") if should_load_into_redshift(upload_event, global_game): with influx_timer("generate_redshift_game_info_duration"): game_info = get_game_info(global_game, replay) exporter.set_game_info(game_info) try: with influx_timer("flush_exporter_to_firehose_duration"): flush_failures_report = flush_exporter_to_firehose( exporter, records_to_flush=get_records_to_flush()) for target_table, errors in flush_failures_report.items( ): for error in errors: influx_metric( "firehose_flush_failure", { "stream_name": error["stream_name"], "error_code": error["error_code"], "error_message": error["error_message"], "count": 1 }, target_table=target_table) except Exception: raise else: global_game.loaded_into_redshift = timezone.now() global_game.save() # Okay to release the advisory lock once loaded_into_redshift is set # It will also be released automatically when the lambda exits. global_game.release_redshift_lock() else: log.debug( "Did not acquire redshift lock. Will not flush to redshift")
def do_flush_exporter(): # Only if we were able to claim the advisory lock do we proceed here. if can_attempt_redshift_load: log.debug("Redshift lock acquired. Will attempt to flush to redshift") if should_load_into_redshift(upload_event, global_game): with influx_timer("generate_redshift_game_info_duration"): game_info = get_game_info(global_game, replay) exporter.set_game_info(game_info) try: with influx_timer("flush_exporter_to_firehose_duration"): flush_exporter_to_firehose(exporter) except: raise else: global_game.loaded_into_redshift = timezone.now() global_game.save() # Okay to release the advisory lock once loaded_into_redshift is set # It will also be released automatically when the lambda exits. global_game.release_redshift_lock() else: log.debug("Did not acquire redshift lock. Will not flush to redshift")