def load_replay_into_redshift(event, context): """A handler that loads a replay into Redshift""" logger = logging.getLogger("hsreplaynet.lambdas.load_replay_into_redshift") replay_bucket = event["replay_bucket"] replay_key = event["replay_key"] metadata_str = event["metadata"] obj = S3.get_object(Bucket=replay_bucket, Key=replay_key) body_data = obj["Body"].read() log_str = decompress(body_data, 15 + 32) out = BytesIO() out.write(log_str) out.seek(0) try: replay = HSReplayDocument.from_xml_file(out) metadata = json.loads(metadata_str) global_game_id = metadata["game_id"] from hsreplaynet.games.models import GlobalGame global_game = GlobalGame.objects.get(id=global_game_id) packet_tree = replay.to_packet_tree()[0] exporter = RedshiftPublishingExporter(packet_tree).export() exporter.set_game_info(metadata) flush_exporter_to_firehose(exporter) except Exception: logger.info(metadata_str) raise else: global_game.loaded_into_redshift = datetime.now() global_game.save()
def test_digest_symmetric(self): with open( self._replay_path("annotated.symmetric_a.25770.hsreplay.xml"), "r") as f1: replay = HSReplayDocument.from_xml_file(f1) exporter1 = GameDigestExporter(replay.to_packet_tree()[0]) exporter1.export() with open( self._replay_path("annotated.symmetric_b.25770.hsreplay.xml"), "r") as f2: replay = HSReplayDocument.from_xml_file(f2) exporter2 = GameDigestExporter(replay.to_packet_tree()[0]) exporter2.export() assert exporter1.digest == exporter2.digest
def do_process_raw_upload(raw_upload, is_reprocessing): process_raw_upload(raw_upload, is_reprocessing) # Begin asserting correctness created_upload_event = UploadEvent.objects.get(shortid=raw_upload.shortid) assert str(created_upload_event.token_uuid) == str( raw_upload.auth_token.key) source_ip = raw_upload.descriptor["event"]["requestContext"]["identity"][ "sourceIp"] assert created_upload_event.upload_ip == source_ip replay = created_upload_event.game assert replay.opponent_revealed_deck is not None assert replay.opponent_revealed_deck.size > 0 validate_fuzzy_date_match(raw_upload.timestamp, replay.global_game.match_start) validate_player_data(raw_upload, replay, 1) validate_player_data(raw_upload, replay, 2) for player_id in (1, 2): for card in replay.global_game.players.get( player_id=player_id).deck_list: assert card.collectible replay_data = HSReplayDocument.from_xml_file(replay.replay_xml) exporter = GameDigestExporter(replay_data.to_packet_tree()[0]) exporter.export() assert exporter.digest == replay.global_game.digest
def test_digest_multi_hero_brawl(self): with open( self._replay_path( "annotated.multi_hero_brawl.25252.hsreplay.xml"), "r") as f: replay = HSReplayDocument.from_xml_file(f) exporter = GameDigestExporter(replay.to_packet_tree()[0]) exporter.export() assert exporter.player_1 == { "entity_id": 2, "hero_entity": 30, "hi": 144115193835963207, "lo": 37760170, "damage_sequence": [ 2, 3, 5, 6, 7, 9, 10, 11, 13, 16, 17, 19, 21, 16, 22, 28, 24, 21, 18, 15, 12, 9, 6, 14, 11, 8, 16, 13, 14, 11, 19, 16, 12, 20, 21, 18, 20, 17, 20 ], "draw_sequence": [ 25, 8, 33, 26, 30, 32, 23, 9, 6, 29, 14, 16, 17, 33, 15, 19, 4, 20, 21, 13, 24, 27, 31, 28, 7, 11, 22, 18, 12, 10 ], "deck": {5, 8} } assert exporter.player_2 == { "entity_id": 3, "hero_entity": 196, "hi": 0, "lo": 0, "damage_sequence": [ 2, 6, 8, 10, 14, 3, 7, 11, 19, 27, 14, 17, 21, 25, 27, 35, 27, 33, 37, 45, 51, 55, 58, 61, 59, 35, 36, 41, 46, 49, 50, 54, 57, 60 ], "draw_sequence": [ 58, 41, 50, 45, 47, 63, 54, 56, 48, 46, 41, 43, 39, 132, 112, 125, 119, 115, 114, 106, 127, 131, 109, 128, 113, 107, 108, 55, 60, 51, 42, 61, 44, 35, 105, 116, 110, 103, 126, 124, 104, 52, 40, 37, 36, 38, 49, 34, 62, 35, 57, 53, 51, 61, 44, 55, 59, 60 ], "deck": {129, 130, 111, 117, 118, 120, 121, 122, 123} } assert exporter.digest == "97deb57c7f51d220b823a6f3bc80471c191ea07c"
def test_digest(self): with open(self._replay_path("annotated.druid_vs_warlock.xml"), "r") as f: replay = HSReplayDocument.from_xml_file(f) exporter = GameDigestExporter(replay.to_packet_tree()[0]) exporter.export() assert exporter.player_1 == { "entity_id": 2, "hero_entity": 78, "hi": 144115198130930503, "lo": 15856412, "damage_sequence": [2, 1, 2, 4, 7, 10, 15, 19, 22, 28], "draw_sequence": [ 27, 34, 15, 33, 23, 28, 4, 21, 13, 31, 43, 15, 38, 7, 30, 39, 35, 18, 40, 29, 32, 22, 8, 44, 91, 26, 12, 24, 25, 9, 14, 123, 34, 45, 138 ], "deck": {27} } assert exporter.player_2 == { "entity_id": 3, "hero_entity": 196, "hi": 144115198130930503, "lo": 16145813, "damage_sequence": [ 1, 3, 4, 5, 6, 8, 9, 11, 13, 15, 11, 13, 15, 12, 14, 15, 18, 20, 21, 22, 23, 28, 1, 2, 3, 6, 9, 11, 13, 16, 18, 20, 22, 2, 4, 12, 13, 10, 14, 16, 17, 18, 22, 24, 12, 13, 16 ], "draw_sequence": [ 50, 75, 68, 49, 77, 65, 73, 74, 64, 66, 71, 70, 53, 54, 51, 58, 75, 55, 48, 59, 63, 60, 56, 52, 76, 61, 57, 62, 72, 50, 67, 69 ], "deck": set() } assert exporter.digest == "ebb641612dd204e185bccd7764a8c5e4245d3c58"
def loadRandomizedLog(self): # --- Load the folder to search files in --- folderPath = "./Games/Standardized/" xmlFiles = [ os.path.join(root, name) for root, dirs, files in os.walk(folderPath) for name in files if name.endswith((".xml")) ] fileName = random.choice(xmlFiles) # load the file to game game = hsDoc.from_xml_file(fileName) return game
def read(self, line): bucket, key, metadata = self.read_line_protocol(line) fh = self.get_file_handle(bucket, key) if not fh: return line, None try: replay = HSReplayDocument.from_xml_file(fh) except Exception: if self.DEBUG: raise else: return line, None return line, {"replay": replay, "metadata": metadata}
def test_digest_deck_swap(self): with open(self._replay_path("annotated.togwaggle.25770.hsreplay.xml"), "r") as f: replay = HSReplayDocument.from_xml_file(f) exporter = GameDigestExporter(replay.to_packet_tree()[0]) exporter.export() assert exporter.player_1 == { "entity_id": 2, "hero_entity": 27, "hi": 144115202425897799, "lo": 41940206, "damage_sequence": [1, 2, 5, 9, 14], "draw_sequence": [ 11, 21, 19, 30, 8, 20, 11, 26, 33, 12, 7, 27, 21, 10, 29, 16, 15, 14, 4, 19, 9, 24, 25, 31, 6 ], "deck": {5, 13, 17, 18, 22, 23, 28, 32} } assert exporter.player_2 == { "entity_id": 3, "hero_entity": 76, "hi": 144115202425897799, "lo": 83394735, "damage_sequence": [ 2, 5, 7, 9, 10, 12, 13, 16, 19, 21, 23, 26, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 22, 23 ], "draw_sequence": [ 45, 47, 54, 48, 70, 71, 46, 42, 45, 47, 59, 38, 34, 41, 40, 44, 49, 52, 58, 43, 66, 61, 51, 55, 35, 50, 65, 48, 67, 62 ], "deck": {39, 53, 54, 60} } assert exporter.digest == "87b1517af0eb67a9aa98f6f90f7d6839ae344149"
def main(): parser = ArgumentParser() parser.add_argument("files", nargs="*") args = parser.parse_args(sys.argv[1:]) default_date = datetime.now() for filename in args.files: with open(filename) as f: if filename.endswith(".xml"): xml_in = f.read() else: doc_in = HSReplayDocument.from_log_file(f, date=default_date, build=BUILD) xml_in = doc_in.to_xml(pretty=True) xml_file_in = BytesIO(xml_in.encode("utf-8")) doc_out = HSReplayDocument.from_xml_file(xml_file_in) assert doc_out.build, "Can't find build in output file" xml_out = doc_out.to_xml(pretty=True) if xml_in != xml_out: with open("in.xml", "w") as f, open("out.xml", "w") as f2: f.write(xml_in) f2.write(xml_out) raise Exception("%r: Log -> XML -> Document -> XML: FAIL" % (filename)) else: print("%r: Log -> XML -> Document -> XML: SUCCESS" % (filename)) packet_tree_in = doc_out.to_packet_tree() doc_out2 = HSReplayDocument.from_packet_tree(packet_tree_in, build=doc_out.build) xml_out2 = doc_out2.to_xml(pretty=True) if xml_in != xml_out2: with open("in.xml", "w") as f, open("out2.xml", "w") as f2: f.write(xml_in) f2.write(xml_out2) raise Exception( "%r: Document -> PacketTree -> Document: FAIL" % (filename)) else: print("%r: Document -> PacketTree -> Document: SUCCESS" % (filename))
def test_digest_hero_change(self): with open( self._replay_path("annotated.hero_change.25770.hsreplay.xml"), "r") as f: replay = HSReplayDocument.from_xml_file(f) exporter = GameDigestExporter(replay.to_packet_tree()[0]) exporter.export() assert exporter.player_1 == { "entity_id": 2, "hero_entity": 27, "hi": 144115193835963207, "lo": 153376707, "damage_sequence": [3, 7, 11, 15, 19, 21], "draw_sequence": [ 24, 39, 33, 12, 9, 19, 22, 32, 8, 21, 35, 23, 38, 6, 41, 34, 25, 31, 18, 5, 15, 26, 40, 33, 20, 7, 14, 27 ], "deck": {4, 13, 30} } assert exporter.player_2 == { "entity_id": 3, "hero_entity": 74, "hi": 0, "lo": 0, "damage_sequence": [1, 4, 5, 6, 9, 12, 15, 17, 22, 25, 26, 28, 32], "draw_sequence": [ 47, 66, 46, 60, 48, 46, 49, 52, 61, 71, 53, 42, 63, 50, 58, 43, 56, 54, 51, 68 ], "deck": {44, 45, 55, 57, 59, 62, 64, 65, 67, 69, 70} } assert exporter.digest == "94b7626ed21f4572e16ba8201f6f6cf5b918e164"
def test_create_dynamodb_game_replay(auth_token): upload_event = UploadEvent( id="1", shortid="ccSgiGQaenVzXzwGYbaUTPGrv", token_uuid=auth_token.key, ) meta = { "game_type": enums.BnetGameType.BGT_RANKED_STANDARD, "ladder_season": 42, "format": enums.FormatType.FT_STANDARD, "friendly_player": 1, "reconnecting": False, "scenario_id": 2, "start_time": datetime(year=2018, month=8, day=8, hour=19, minute=20, second=2, microsecond=606936), "end_time": datetime(year=2018, month=8, day=8, hour=19, minute=31, second=10), "player1": { "rank": 20, "stars": 30, "deck": [ "BOT_447", "BOT_447", "EX1_319", "EX1_319", "LOOT_014", "LOOT_014", "BOT_263", "BOT_263", "BOT_568", "CS2_065", "CS2_065", "EX1_596", "EX1_596", "BOT_443", "BOT_443", "LOOT_013", "LOOT_013", "BOT_224", "BOT_224", "BOT_226", "BOT_226", "ICC_466", "ICC_466", "ICC_075", "ICC_075", "EX1_310", "EX1_310", "BOT_521", "BOT_521", "ICC_831", ], "deck_id": 1337, "cardback": 136, }, "player2": { "rank": 19, "cardback": 138, }, } path = os.path.join(LOG_DATA_DIR, "hsreplaynet-tests", "replays", "whizbang_friendly.annotated.xml") with open(path, "r") as f: replay = HSReplayDocument.from_xml_file(f) packet_tree = replay.to_packet_tree()[0] entity_tree = EntityTreeExporter(packet_tree).export().game replay_xml = "foo.xml" replay = create_dynamodb_game_replay(upload_event, meta, entity_tree, replay_xml) assert replay assert replay.user_id == auth_token.user.id assert replay.match_start == 1533756002606 assert replay.match_end == 1533756670000 assert replay.short_id == upload_event.shortid assert replay.digest is None assert replay.game_type == enums.BnetGameType.BGT_RANKED_STANDARD assert replay.format_type == enums.FormatType.FT_STANDARD assert replay.game_type_match_start == "2:1533756002606" assert replay.ladder_season == 42 assert replay.brawl_season is None assert replay.scenario_id == 2 assert replay.num_turns == 31 assert replay.friendly_player_account_hilo == "144115193835963207_127487329" assert replay.friendly_player_battletag == "Masture#1176" assert replay.friendly_player_is_first assert replay.friendly_player_rank == 20 assert replay.friendly_player_rank_stars == 30 assert replay.friendly_player_legend_rank is None assert replay.friendly_player_wins is None assert replay.friendly_player_losses is None assert replay.friendly_player_class == enums.CardClass.WARLOCK assert replay.friendly_player_deck == \ "AAECAf0GApfTAo+CAw4w9wTCCPYIm8sC980C8dAC8tAC9PcC0/gCqvkCt/0Cw/0C+v4CAA==" assert replay.friendly_player_blizzard_deck_id == 1337 assert replay.friendly_player_cardback_id == 136 assert replay.friendly_player_final_state == enums.PlayState.WON assert replay.opponent_account_hilo == "144115193835963207_50318740" assert replay.opponent_battletag == "GinyuGamer#1677" assert not replay.opponent_is_ai assert replay.opponent_rank == 19 assert replay.opponent_legend_rank is None assert replay.opponent_class == enums.CardClass.PALADIN assert replay.opponent_hero == 671 assert replay.opponent_revealed_deck == \ "AAECAZ8FDYoGlgm5wQLjywKc4gKL5QKb8AKl9QKE/ALW/gKggAPMgQPeggMEiMcC/PwC4f4CkYADAA==" assert replay.opponent_predicted_deck is None assert replay.opponent_final_state == enums.PlayState.LOST assert replay.replay_xml == replay_xml assert not replay.disconnected assert not replay.reconnecting assert replay.hslog_version assert replay.visibility == auth_token.user.default_replay_visibility assert replay.views == 0
def test_replays_api(auth_token, client, mocker): mocker.patch("hsreplaynet.api.serializers.replays.classify_deck", return_value=1) upload_event = UploadEvent( id="1", shortid="ccSgiGQaenVzXzwGYbaUTPGrv", token_uuid=auth_token.key, ) path = os.path.join(LOG_DATA_DIR, "hsreplaynet-tests", "replays", "whizbang_friendly.annotated.xml") with open(path, "r") as f: replay = HSReplayDocument.from_xml_file(f) packet_tree = replay.to_packet_tree()[0] meta = { "game_type": enums.BnetGameType.BGT_RANKED_STANDARD, "ladder_season": 42, "format": enums.FormatType.FT_STANDARD, "friendly_player": 1, "reconnecting": False, "scenario_id": 2, "start_time": packet_tree.start_time, "end_time": packet_tree.end_time, "player1": { "rank": 20, "stars": 30, "deck": [ "BOT_447", "BOT_447", "EX1_319", "EX1_319", "LOOT_014", "LOOT_014", "BOT_263", "BOT_263", "BOT_568", "CS2_065", "CS2_065", "EX1_596", "EX1_596", "BOT_443", "BOT_443", "LOOT_013", "LOOT_013", "BOT_224", "BOT_224", "BOT_226", "BOT_226", "ICC_466", "ICC_466", "ICC_075", "ICC_075", "EX1_310", "EX1_310", "BOT_521", "BOT_521", "ICC_831", ], "deck_id": 1337, "cardback": 136, }, "player2": { "rank": 19, "cardback": 138, }, } entity_tree = EntityTreeExporter(packet_tree).export().game replay_xml = "foo.xml" replay = create_dynamodb_game_replay(upload_event, meta, entity_tree, replay_xml) replay.save() user_id = auth_token.user.id response = client.get("/api/v1/replays/?user_id=%d" % (user_id)) assert response.status_code == status.HTTP_200_OK payload = _parse_streaming_json(response.streaming_content)[0] assert payload["user_id"] == user_id assert payload["match_start"] == "2018-08-08T21:20:02.610000Z" assert payload["match_end"] == "2018-08-08T21:31:10.236000Z" assert payload["shortid"] == "ccSgiGQaenVzXzwGYbaUTPGrv" assert payload["game_type"] == enums.BnetGameType.BGT_RANKED_STANDARD assert payload["format_type"] == enums.FormatType.FT_STANDARD assert payload["friendly_player_account_hi"] == "144115193835963207" assert payload["friendly_player_account_lo"] == "127487329" assert payload["friendly_player_battletag"] == "Masture#1176" assert payload["friendly_player_rank"] == 20 assert payload["friendly_player_archetype_id"] == 1 assert payload["opponent_account_hi"] == "144115193835963207" assert payload["opponent_account_lo"] == "50318740" assert payload["opponent_battletag"] == "GinyuGamer#1677" assert payload["opponent_rank"] == 19 assert payload["opponent_archetype_id"] is None assert payload["replay_xml"] == "foo.xml" assert payload["disconnected"] is False assert payload["reconnecting"] is False assert payload["visibility"] == Visibility.Public assert payload["views"] == 0