def save_hsreplay_xml(self, parser, meta):
from hsreplay.document import HSReplayDocument
global_game = self.global_game
hsreplay_doc = HSReplayDocument.from_parser(parser, build=self.build)
game_xml = hsreplay_doc.games[0]
game_xml.game_type = global_game.game_type
game_xml.id = global_game.game_handle
if self.reconnecting:
game_xml.reconnecting = True
game_tree = parser.games[0]
for player in game_tree.game.players:
player_meta = meta.get("player%i" % (player.player_id), {})
player_xml = game_xml.players[player.player_id - 1]
player_xml.rank = player_meta.get("rank")
player_xml.legendRank = player_meta.get("legend_rank")
player_xml.cardback = player_meta.get("cardback")
player_xml.deck = player_meta.get("deck")
xml_str = hsreplay_doc.to_xml()
self.hsreplay_version = hsreplay_doc.version
# Clean up existing replays first
if self.replay_xml.name and default_storage.exists(self.replay_xml.name):
self.replay_xml.delete(save=False)
xml_file = ContentFile(xml_str)
self.replay_xml.save("hsreplay.xml", xml_file, save=False)
return xml_file
def save_hsreplay_xml(self, parser, meta):
from hsreplay.document import HSReplayDocument
global_game = self.global_game
hsreplay_doc = HSReplayDocument.from_parser(parser,
build=global_game.build)
game_xml = hsreplay_doc.games[0]
game_xml.game_type = global_game.game_type
game_xml.id = global_game.game_server_game_id
if self.reconnecting:
game_xml.reconnecting = True
game_tree = parser.games[0]
for player in game_tree.game.players:
player_meta = meta.get("player%i" % (player.player_id), {})
player_xml = game_xml.players[player.player_id - 1]
player_xml.rank = player_meta.get("rank")
player_xml.legendRank = player_meta.get("legend_rank")
player_xml.cardback = player_meta.get("cardback")
player_xml.deck = player_meta.get("deck")
xml_str = hsreplay_doc.to_xml()
self.hsreplay_version = hsreplay_doc.version
# Clean up existing replays first
if self.replay_xml.name and default_storage.exists(
self.replay_xml.name):
self.replay_xml.delete(save=False)
xml_file = ContentFile(xml_str)
self.replay_xml.save("hsreplay.xml", xml_file, save=False)
return xml_file
def load_replay_into_redshift(event, context):
"""A handler that loads a replay into Redshift"""
logger = logging.getLogger("hsreplaynet.lambdas.load_replay_into_redshift")
replay_bucket = event["replay_bucket"]
replay_key = event["replay_key"]
metadata_str = event["metadata"]
obj = S3.get_object(Bucket=replay_bucket, Key=replay_key)
body_data = obj["Body"].read()
log_str = decompress(body_data, 15 + 32)
out = BytesIO()
out.write(log_str)
out.seek(0)
try:
replay = HSReplayDocument.from_xml_file(out)
metadata = json.loads(metadata_str)
global_game_id = metadata["game_id"]
from hsreplaynet.games.models import GlobalGame
global_game = GlobalGame.objects.get(id=global_game_id)
packet_tree = replay.to_packet_tree()[0]
exporter = RedshiftPublishingExporter(packet_tree).export()
exporter.set_game_info(metadata)
flush_exporter_to_firehose(exporter)
except Exception:
logger.info(metadata_str)
raise
else:
global_game.loaded_into_redshift = datetime.now()
global_game.save()
def test_digest_symmetric(self):
with open(
self._replay_path("annotated.symmetric_a.25770.hsreplay.xml"),
"r") as f1:
replay = HSReplayDocument.from_xml_file(f1)
exporter1 = GameDigestExporter(replay.to_packet_tree()[0])
exporter1.export()
with open(
self._replay_path("annotated.symmetric_b.25770.hsreplay.xml"),
"r") as f2:
replay = HSReplayDocument.from_xml_file(f2)
exporter2 = GameDigestExporter(replay.to_packet_tree()[0])
exporter2.export()
assert exporter1.digest == exporter2.digest
def do_process_raw_upload(raw_upload, is_reprocessing):
process_raw_upload(raw_upload, is_reprocessing)
# Begin asserting correctness
created_upload_event = UploadEvent.objects.get(shortid=raw_upload.shortid)
assert str(created_upload_event.token_uuid) == str(
raw_upload.auth_token.key)
source_ip = raw_upload.descriptor["event"]["requestContext"]["identity"][
"sourceIp"]
assert created_upload_event.upload_ip == source_ip
replay = created_upload_event.game
assert replay.opponent_revealed_deck is not None
assert replay.opponent_revealed_deck.size > 0
validate_fuzzy_date_match(raw_upload.timestamp,
replay.global_game.match_start)
validate_player_data(raw_upload, replay, 1)
validate_player_data(raw_upload, replay, 2)
for player_id in (1, 2):
for card in replay.global_game.players.get(
player_id=player_id).deck_list:
assert card.collectible
replay_data = HSReplayDocument.from_xml_file(replay.replay_xml)
exporter = GameDigestExporter(replay_data.to_packet_tree()[0])
exporter.export()
assert exporter.digest == replay.global_game.digest
def main():
parser = ArgumentParser()
parser.add_argument("files", nargs="*")
args = parser.parse_args(sys.argv[1:])
default_date = datetime.now()
for filename in args.files:
with open(filename) as f:
if filename.endswith(".xml"):
xml_in = f.read()
else:
doc_in = HSReplayDocument.from_log_file(f,
date=default_date,
build=BUILD)
xml_in = doc_in.to_xml(pretty=True)
xml_file_in = BytesIO(xml_in.encode("utf-8"))
doc_out = HSReplayDocument.from_xml_file(xml_file_in)
assert doc_out.build, "Can't find build in output file"
xml_out = doc_out.to_xml(pretty=True)
if xml_in != xml_out:
with open("in.xml", "w") as f, open("out.xml", "w") as f2:
f.write(xml_in)
f2.write(xml_out)
raise Exception("%r: Log -> XML -> Document -> XML: FAIL" %
(filename))
else:
print("%r: Log -> XML -> Document -> XML: SUCCESS" %
(filename))
packet_tree_in = doc_out.to_packet_tree()
doc_out2 = HSReplayDocument.from_packet_tree(packet_tree_in,
build=doc_out.build)
xml_out2 = doc_out2.to_xml(pretty=True)
if xml_in != xml_out2:
with open("in.xml", "w") as f, open("out2.xml", "w") as f2:
f.write(xml_in)
f2.write(xml_out2)
raise Exception(
"%r: Document -> PacketTree -> Document: FAIL" %
(filename))
else:
print("%r: Document -> PacketTree -> Document: SUCCESS" %
(filename))
def test_digest_multi_hero_brawl(self):
with open(
self._replay_path(
"annotated.multi_hero_brawl.25252.hsreplay.xml"),
"r") as f:
replay = HSReplayDocument.from_xml_file(f)
exporter = GameDigestExporter(replay.to_packet_tree()[0])
exporter.export()
assert exporter.player_1 == {
"entity_id":
2,
"hero_entity":
30,
"hi":
144115193835963207,
"lo":
37760170,
"damage_sequence": [
2, 3, 5, 6, 7, 9, 10, 11, 13, 16, 17, 19, 21, 16, 22, 28,
24, 21, 18, 15, 12, 9, 6, 14, 11, 8, 16, 13, 14, 11, 19,
16, 12, 20, 21, 18, 20, 17, 20
],
"draw_sequence": [
25, 8, 33, 26, 30, 32, 23, 9, 6, 29, 14, 16, 17, 33, 15,
19, 4, 20, 21, 13, 24, 27, 31, 28, 7, 11, 22, 18, 12, 10
],
"deck": {5, 8}
}
assert exporter.player_2 == {
"entity_id":
3,
"hero_entity":
196,
"hi":
0,
"lo":
0,
"damage_sequence": [
2, 6, 8, 10, 14, 3, 7, 11, 19, 27, 14, 17, 21, 25, 27, 35,
27, 33, 37, 45, 51, 55, 58, 61, 59, 35, 36, 41, 46, 49, 50,
54, 57, 60
],
"draw_sequence": [
58, 41, 50, 45, 47, 63, 54, 56, 48, 46, 41, 43, 39, 132,
112, 125, 119, 115, 114, 106, 127, 131, 109, 128, 113, 107,
108, 55, 60, 51, 42, 61, 44, 35, 105, 116, 110, 103, 126,
124, 104, 52, 40, 37, 36, 38, 49, 34, 62, 35, 57, 53, 51,
61, 44, 55, 59, 60
],
"deck": {129, 130, 111, 117, 118, 120, 121, 122, 123}
}
assert exporter.digest == "97deb57c7f51d220b823a6f3bc80471c191ea07c"
def loadMacGame(self):
f = open("/Applications/Hearthstone/Logs/Power.log", "r")
myList = []
for line in f:
myList.append(line)
f.close()
pars = Pars()
pars.read(myList)
game = hsDoc.from_parser(pars, build=None)
return game
def loadPcGame(self):
# TODO: Rewrite game loading to be less time consuming
# --- Connect to Hearthstone Client Directly ---
f = open("C:\Program Files (x86)\Hearthstone\Logs\Power.log", "r")
myList = []
for line in f:
myList.append(line)
f.close()
pars = Pars()
pars.read(myList)
game = hsDoc.from_parser(pars, build=None)
return game
def main():
parser = ArgumentParser()
parser.add_argument("files", nargs="*")
parser.add_argument("--processor", dest="processor", default="GameState")
parser.add_argument("--default-date",
dest="date",
type=date_arg,
help="Format: YYYY-MM-DD")
# https://stackoverflow.com/questions/9226516/
args = parser.parse_args(sys.argv[1:])
for filename in args.files:
with open(filename) as f:
doc = HSReplayDocument.from_log_file(f, args.processor, args.date)
print((doc.to_xml()))
def test_digest(self):
with open(self._replay_path("annotated.druid_vs_warlock.xml"),
"r") as f:
replay = HSReplayDocument.from_xml_file(f)
exporter = GameDigestExporter(replay.to_packet_tree()[0])
exporter.export()
assert exporter.player_1 == {
"entity_id":
2,
"hero_entity":
78,
"hi":
144115198130930503,
"lo":
15856412,
"damage_sequence": [2, 1, 2, 4, 7, 10, 15, 19, 22, 28],
"draw_sequence": [
27, 34, 15, 33, 23, 28, 4, 21, 13, 31, 43, 15, 38, 7, 30,
39, 35, 18, 40, 29, 32, 22, 8, 44, 91, 26, 12, 24, 25, 9,
14, 123, 34, 45, 138
],
"deck": {27}
}
assert exporter.player_2 == {
"entity_id":
3,
"hero_entity":
196,
"hi":
144115198130930503,
"lo":
16145813,
"damage_sequence": [
1, 3, 4, 5, 6, 8, 9, 11, 13, 15, 11, 13, 15, 12, 14, 15,
18, 20, 21, 22, 23, 28, 1, 2, 3, 6, 9, 11, 13, 16, 18, 20,
22, 2, 4, 12, 13, 10, 14, 16, 17, 18, 22, 24, 12, 13, 16
],
"draw_sequence": [
50, 75, 68, 49, 77, 65, 73, 74, 64, 66, 71, 70, 53, 54, 51,
58, 75, 55, 48, 59, 63, 60, 56, 52, 76, 61, 57, 62, 72, 50,
67, 69
],
"deck":
set()
}
assert exporter.digest == "ebb641612dd204e185bccd7764a8c5e4245d3c58"
def loadRandomizedLog(self):
# --- Load the folder to search files in ---
folderPath = "./Games/Standardized/"
xmlFiles = [
os.path.join(root, name)
for root, dirs, files in os.walk(folderPath) for name in files
if name.endswith((".xml"))
]
fileName = random.choice(xmlFiles)
# load the file to game
game = hsDoc.from_xml_file(fileName)
return game
def read(self, line):
bucket, key, metadata = self.read_line_protocol(line)
fh = self.get_file_handle(bucket, key)
if not fh:
return line, None
try:
replay = HSReplayDocument.from_xml_file(fh)
except Exception:
if self.DEBUG:
raise
else:
return line, None
return line, {"replay": replay, "metadata": metadata}
def create_hsreplay_document(parser, entity_tree, meta, global_game):
hsreplay_doc = HSReplayDocument.from_parser(parser, build=meta["build"])
game_xml = hsreplay_doc.games[0]
game_xml.game_type = global_game.game_type
game_xml.id = global_game.game_handle
if meta["reconnecting"]:
game_xml.reconnecting = True
for player in entity_tree.players:
player_meta = meta.get("player%i" % (player.player_id), {})
player_xml = game_xml.players[player.player_id - 1]
player_xml.rank = player_meta.get("rank")
player_xml.legendRank = player_meta.get("legend_rank")
player_xml.cardback = player_meta.get("cardback")
player_xml.deck = player_meta.get("deck")
return hsreplay_doc
def create_hsreplay_document(parser, entity_tree, meta, global_game):
hsreplay_doc = HSReplayDocument.from_parser(parser, build=meta["build"])
game_xml = hsreplay_doc.games[0]
game_xml.game_type = global_game.game_type
game_xml.id = global_game.game_handle
if meta["reconnecting"]:
game_xml.reconnecting = True
for player in entity_tree.players:
player_meta = meta.get("player%i" % (player.player_id), {})
player_xml = game_xml.players[player.player_id - 1]
player_xml.rank = player_meta.get("rank")
player_xml.legendRank = player_meta.get("legend_rank")
player_xml.cardback = player_meta.get("cardback")
player_xml.deck = player_meta.get("deck")
return hsreplay_doc
def test_digest_deck_swap(self):
with open(self._replay_path("annotated.togwaggle.25770.hsreplay.xml"),
"r") as f:
replay = HSReplayDocument.from_xml_file(f)
exporter = GameDigestExporter(replay.to_packet_tree()[0])
exporter.export()
assert exporter.player_1 == {
"entity_id":
2,
"hero_entity":
27,
"hi":
144115202425897799,
"lo":
41940206,
"damage_sequence": [1, 2, 5, 9, 14],
"draw_sequence": [
11, 21, 19, 30, 8, 20, 11, 26, 33, 12, 7, 27, 21, 10, 29,
16, 15, 14, 4, 19, 9, 24, 25, 31, 6
],
"deck": {5, 13, 17, 18, 22, 23, 28, 32}
}
assert exporter.player_2 == {
"entity_id":
3,
"hero_entity":
76,
"hi":
144115202425897799,
"lo":
83394735,
"damage_sequence": [
2, 5, 7, 9, 10, 12, 13, 16, 19, 21, 23, 26, 29, 28, 27, 26,
25, 24, 23, 22, 21, 20, 19, 18, 17, 22, 23
],
"draw_sequence": [
45, 47, 54, 48, 70, 71, 46, 42, 45, 47, 59, 38, 34, 41, 40,
44, 49, 52, 58, 43, 66, 61, 51, 55, 35, 50, 65, 48, 67, 62
],
"deck": {39, 53, 54, 60}
}
assert exporter.digest == "87b1517af0eb67a9aa98f6f90f7d6839ae344149"
def test_digest_hero_change(self):
with open(
self._replay_path("annotated.hero_change.25770.hsreplay.xml"),
"r") as f:
replay = HSReplayDocument.from_xml_file(f)
exporter = GameDigestExporter(replay.to_packet_tree()[0])
exporter.export()
assert exporter.player_1 == {
"entity_id":
2,
"hero_entity":
27,
"hi":
144115193835963207,
"lo":
153376707,
"damage_sequence": [3, 7, 11, 15, 19, 21],
"draw_sequence": [
24, 39, 33, 12, 9, 19, 22, 32, 8, 21, 35, 23, 38, 6, 41,
34, 25, 31, 18, 5, 15, 26, 40, 33, 20, 7, 14, 27
],
"deck": {4, 13, 30}
}
assert exporter.player_2 == {
"entity_id":
3,
"hero_entity":
74,
"hi":
0,
"lo":
0,
"damage_sequence":
[1, 4, 5, 6, 9, 12, 15, 17, 22, 25, 26, 28, 32],
"draw_sequence": [
47, 66, 46, 60, 48, 46, 49, 52, 61, 71, 53, 42, 63, 50, 58,
43, 56, 54, 51, 68
],
"deck": {44, 45, 55, 57, 59, 62, 64, 65, 67, 69, 70}
}
assert exporter.digest == "94b7626ed21f4572e16ba8201f6f6cf5b918e164"
def test_replays_api(auth_token, client, mocker):
mocker.patch("hsreplaynet.api.serializers.replays.classify_deck",
return_value=1)
upload_event = UploadEvent(
id="1",
shortid="ccSgiGQaenVzXzwGYbaUTPGrv",
token_uuid=auth_token.key,
)
path = os.path.join(LOG_DATA_DIR, "hsreplaynet-tests", "replays",
"whizbang_friendly.annotated.xml")
with open(path, "r") as f:
replay = HSReplayDocument.from_xml_file(f)
packet_tree = replay.to_packet_tree()[0]
meta = {
"game_type": enums.BnetGameType.BGT_RANKED_STANDARD,
"ladder_season": 42,
"format": enums.FormatType.FT_STANDARD,
"friendly_player": 1,
"reconnecting": False,
"scenario_id": 2,
"start_time": packet_tree.start_time,
"end_time": packet_tree.end_time,
"player1": {
"rank":
20,
"stars":
30,
"deck": [
"BOT_447",
"BOT_447",
"EX1_319",
"EX1_319",
"LOOT_014",
"LOOT_014",
"BOT_263",
"BOT_263",
"BOT_568",
"CS2_065",
"CS2_065",
"EX1_596",
"EX1_596",
"BOT_443",
"BOT_443",
"LOOT_013",
"LOOT_013",
"BOT_224",
"BOT_224",
"BOT_226",
"BOT_226",
"ICC_466",
"ICC_466",
"ICC_075",
"ICC_075",
"EX1_310",
"EX1_310",
"BOT_521",
"BOT_521",
"ICC_831",
],
"deck_id":
1337,
"cardback":
136,
},
"player2": {
"rank": 19,
"cardback": 138,
},
}
entity_tree = EntityTreeExporter(packet_tree).export().game
replay_xml = "foo.xml"
replay = create_dynamodb_game_replay(upload_event, meta, entity_tree,
replay_xml)
replay.save()
user_id = auth_token.user.id
response = client.get("/api/v1/replays/?user_id=%d" % (user_id))
assert response.status_code == status.HTTP_200_OK
payload = _parse_streaming_json(response.streaming_content)[0]
assert payload["user_id"] == user_id
assert payload["match_start"] == "2018-08-08T21:20:02.610000Z"
assert payload["match_end"] == "2018-08-08T21:31:10.236000Z"
assert payload["shortid"] == "ccSgiGQaenVzXzwGYbaUTPGrv"
assert payload["game_type"] == enums.BnetGameType.BGT_RANKED_STANDARD
assert payload["format_type"] == enums.FormatType.FT_STANDARD
assert payload["friendly_player_account_hi"] == "144115193835963207"
assert payload["friendly_player_account_lo"] == "127487329"
assert payload["friendly_player_battletag"] == "Masture#1176"
assert payload["friendly_player_rank"] == 20
assert payload["friendly_player_archetype_id"] == 1
assert payload["opponent_account_hi"] == "144115193835963207"
assert payload["opponent_account_lo"] == "50318740"
assert payload["opponent_battletag"] == "GinyuGamer#1677"
assert payload["opponent_rank"] == 19
assert payload["opponent_archetype_id"] is None
assert payload["replay_xml"] == "foo.xml"
assert payload["disconnected"] is False
assert payload["reconnecting"] is False
assert payload["visibility"] == Visibility.Public
assert payload["views"] == 0
def test_create_dynamodb_game_replay(auth_token):
upload_event = UploadEvent(
id="1",
shortid="ccSgiGQaenVzXzwGYbaUTPGrv",
token_uuid=auth_token.key,
)
meta = {
"game_type":
enums.BnetGameType.BGT_RANKED_STANDARD,
"ladder_season":
42,
"format":
enums.FormatType.FT_STANDARD,
"friendly_player":
1,
"reconnecting":
False,
"scenario_id":
2,
"start_time":
datetime(year=2018,
month=8,
day=8,
hour=19,
minute=20,
second=2,
microsecond=606936),
"end_time":
datetime(year=2018, month=8, day=8, hour=19, minute=31, second=10),
"player1": {
"rank":
20,
"stars":
30,
"deck": [
"BOT_447",
"BOT_447",
"EX1_319",
"EX1_319",
"LOOT_014",
"LOOT_014",
"BOT_263",
"BOT_263",
"BOT_568",
"CS2_065",
"CS2_065",
"EX1_596",
"EX1_596",
"BOT_443",
"BOT_443",
"LOOT_013",
"LOOT_013",
"BOT_224",
"BOT_224",
"BOT_226",
"BOT_226",
"ICC_466",
"ICC_466",
"ICC_075",
"ICC_075",
"EX1_310",
"EX1_310",
"BOT_521",
"BOT_521",
"ICC_831",
],
"deck_id":
1337,
"cardback":
136,
},
"player2": {
"rank": 19,
"cardback": 138,
},
}
path = os.path.join(LOG_DATA_DIR, "hsreplaynet-tests", "replays",
"whizbang_friendly.annotated.xml")
with open(path, "r") as f:
replay = HSReplayDocument.from_xml_file(f)
packet_tree = replay.to_packet_tree()[0]
entity_tree = EntityTreeExporter(packet_tree).export().game
replay_xml = "foo.xml"
replay = create_dynamodb_game_replay(upload_event, meta, entity_tree,
replay_xml)
assert replay
assert replay.user_id == auth_token.user.id
assert replay.match_start == 1533756002606
assert replay.match_end == 1533756670000
assert replay.short_id == upload_event.shortid
assert replay.digest is None
assert replay.game_type == enums.BnetGameType.BGT_RANKED_STANDARD
assert replay.format_type == enums.FormatType.FT_STANDARD
assert replay.game_type_match_start == "2:1533756002606"
assert replay.ladder_season == 42
assert replay.brawl_season is None
assert replay.scenario_id == 2
assert replay.num_turns == 31
assert replay.friendly_player_account_hilo == "144115193835963207_127487329"
assert replay.friendly_player_battletag == "Masture#1176"
assert replay.friendly_player_is_first
assert replay.friendly_player_rank == 20
assert replay.friendly_player_rank_stars == 30
assert replay.friendly_player_legend_rank is None
assert replay.friendly_player_wins is None
assert replay.friendly_player_losses is None
assert replay.friendly_player_class == enums.CardClass.WARLOCK
assert replay.friendly_player_deck == \
"AAECAf0GApfTAo+CAw4w9wTCCPYIm8sC980C8dAC8tAC9PcC0/gCqvkCt/0Cw/0C+v4CAA=="
assert replay.friendly_player_blizzard_deck_id == 1337
assert replay.friendly_player_cardback_id == 136
assert replay.friendly_player_final_state == enums.PlayState.WON
assert replay.opponent_account_hilo == "144115193835963207_50318740"
assert replay.opponent_battletag == "GinyuGamer#1677"
assert not replay.opponent_is_ai
assert replay.opponent_rank == 19
assert replay.opponent_legend_rank is None
assert replay.opponent_class == enums.CardClass.PALADIN
assert replay.opponent_hero == 671
assert replay.opponent_revealed_deck == \
"AAECAZ8FDYoGlgm5wQLjywKc4gKL5QKb8AKl9QKE/ALW/gKggAPMgQPeggMEiMcC/PwC4f4CkYADAA=="
assert replay.opponent_predicted_deck is None
assert replay.opponent_final_state == enums.PlayState.LOST
assert replay.replay_xml == replay_xml
assert not replay.disconnected
assert not replay.reconnecting
assert replay.hslog_version
assert replay.visibility == auth_token.user.default_replay_visibility
assert replay.views == 0
