def load_replay_into_redshift(event, context): """A handler that loads a replay into Redshift""" logger = logging.getLogger("hsreplaynet.lambdas.load_replay_into_redshift") replay_bucket = event["replay_bucket"] replay_key = event["replay_key"] metadata_str = event["metadata"] obj = S3.get_object(Bucket=replay_bucket, Key=replay_key) body_data = obj["Body"].read() log_str = decompress(body_data, 15 + 32) out = BytesIO() out.write(log_str) out.seek(0) try: replay = HSReplayDocument.from_xml_file(out) metadata = json.loads(metadata_str) global_game_id = metadata["game_id"] from hsreplaynet.games.models import GlobalGame global_game = GlobalGame.objects.get(id=global_game_id) packet_tree = replay.to_packet_tree()[0] exporter = RedshiftPublishingExporter(packet_tree).export() exporter.set_game_info(metadata) flush_exporter_to_firehose(exporter) except Exception: logger.info(metadata_str) raise else: global_game.loaded_into_redshift = datetime.now() global_game.save()
def do_flush_exporter(): # Only if we were able to claim the advisory lock do we proceed here. if can_attempt_redshift_load: log.debug("Redshift lock acquired. Will attempt to flush to redshift") if should_load_into_redshift(upload_event, global_game): with influx_timer("generate_redshift_game_info_duration"): game_info = get_game_info(global_game, replay) exporter.set_game_info(game_info) try: with influx_timer("flush_exporter_to_firehose_duration"): flush_exporter_to_firehose(exporter) except: raise else: global_game.loaded_into_redshift = timezone.now() global_game.save() # Okay to release the advisory lock once loaded_into_redshift is set # It will also be released automatically when the lambda exits. global_game.release_redshift_lock() else: log.debug("Did not acquire redshift lock. Will not flush to redshift")
def do_flush_exporter(): # Only if we were able to claim the advisory lock do we proceed here. if can_attempt_redshift_load: log.debug( "Redshift lock acquired. Will attempt to flush to redshift") if should_load_into_redshift(upload_event, global_game): with influx_timer("generate_redshift_game_info_duration"): game_info = get_game_info(global_game, replay) exporter.set_game_info(game_info) try: with influx_timer("flush_exporter_to_firehose_duration"): flush_failures_report = flush_exporter_to_firehose( exporter, records_to_flush=get_records_to_flush()) for target_table, errors in flush_failures_report.items( ): for error in errors: influx_metric( "firehose_flush_failure", { "stream_name": error["stream_name"], "error_code": error["error_code"], "error_message": error["error_message"], "count": 1 }, target_table=target_table) except Exception: raise else: global_game.loaded_into_redshift = timezone.now() global_game.save() # Okay to release the advisory lock once loaded_into_redshift is set # It will also be released automatically when the lambda exits. global_game.release_redshift_lock() else: log.debug( "Did not acquire redshift lock. Will not flush to redshift")