def process_sing_file(self, replay_file_name):
archive = mpyq.MPQArchive(replay_file_name)
contents = archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(contents)
base_build = header['m_version']['m_baseBuild']
try:
protocol = versions.build(base_build)
except Exception as e:
self.fail('Unsupported base build: {0} ({1!s})'.format(base_build, e))
required_internal_files = ['replay.details',
'replay.details.backup',
'replay.initData',
'replay.game.events',
'replay.message.events',
'replay.tracker.events'
]
for internal_file_name in required_internal_files:
contents = archive.read_file(internal_file_name)
self.assertIsNotNone(contents)
# just decode init data
if internal_file_name is 'replay.initData':
result = protocol.decode_replay_initdata(contents)
self.assertIsNotNone(result)
print('processed {}, base_build: {}'.format(replay_file_name, base_build))
def get_events(replay_file):
"""
:param replay_file: path to a sc2replay file
:return: events: a list of upgrade, unit born, and unit death events
"""
archive = mpyq.MPQArchive(replay_file)
header = versions.latest().decode_replay_header(
archive.header['user_data_header']['content'])
base_build = header['m_version']['m_baseBuild']
decoder = versions.build(base_build)
game_events_gen = decoder.decode_replay_game_events(
archive.read_file('replay.game.events'))
tracker_events_gen = decoder.decode_replay_tracker_events(
archive.read_file('replay.tracker.events'))
relevant_tracker_event_types = [
"NNet.Replay.Tracker.SUnitDiedEvent",
"NNet.Replay.Tracker.SUnitBornEvent",
"NNet.Replay.Tracker.SUnitTypeChangeEvent",
"NNet.Replay.Tracker.SUnitInitEvent",
"NNet.Replay.Tracker.SPlayerSetupEvent"
]
tracker_events = [
event for event in tracker_events_gen
if event['_event'] in relevant_tracker_event_types
]
game_events = [
event for event in game_events_gen
if event['_event'] == "NNet.Game.SGameUserLeaveEvent"
]
return tracker_events + game_events
def __init__(self, path, debug):
self.DEBUG = debug
self.invalid = False
try:
self.archive = mpyq.MPQArchive(path)
except:
self.invalid = True
return
self.archive_contents = self.archive.header['user_data_header'][
'content']
_header = versions.latest().decode_replay_header(self.archive_contents)
self.build = _header['m_version']['m_baseBuild']
try:
self.protocol = versions.build(self.build)
except:
next_lower = closest_version(self.build, versions)[0]
self.protocol = versions.build(next_lower)
try:
self.meta = json.loads(
self.archive.read_file('replay.gamemetadata.json').decode(
'utf-8'))
except:
self.invalid = True
return
self._events = None
self._tracker_events = None
self._init_data = None
self._details = None
self._attribute_events = None
self.tracked_units = TrackedUnits()
self.game_ended = False
self.winner = None
self._phase = "Build"
self.players = []
self._game = {"wave": 0, "builders": {}}
self.towers = []
self.towerList = {}
self.sends = []
self.buildersByWave = []
self.buildersOnWave = {}
self.workers = []
self.speedUps = []
self.kills = []
self.gasNumber = {}
self.upgradeNumber = {}
self.filename = path
self._db = None
def extract_replay_info(loc):
"""
Replays are stored as MPQArchives and need to be extracted using the scprotocol
provided by blizzard. This function takes that protocol and uses it to extract
the data inside the replay. Items returned are player names, map name, player races,
official blizzard map, the patch version, and the location of the starcraft2 replay.
:param loc: File path location where a single StarCraft2 replay is
:rtype: Dictionary
:return: Returns data from the replay
"""
if os.path.isfile(loc):
archive = mpyq.MPQArchive(loc)
contents = archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(contents)
base_build = header['m_version']['m_baseBuild']
try:
protocol = versions.build(base_build)
except ImportError:
print("Replay too old, protocol does not exist: " + loc)
return {}
# Get patch version from replay header
patch = str(header['m_version']['m_major']) + "." + \
str(header['m_version']['m_minor']) + "." + \
str(header['m_version']['m_revision'])
contents = archive.read_file('replay.details')
details = protocol.decode_replay_details(contents)
try:
map_name = details['m_title']
player1_name = details['m_playerList'][0]['m_name'].decode('utf-8')
player1_race = details['m_playerList'][0]['m_race'].decode('utf-8')
player2_name = details['m_playerList'][1]['m_name'].decode('utf-8')
player2_race = details['m_playerList'][1]['m_race'].decode('utf-8')
blizz_map = details['m_isBlizzardMap']
# Removes clan tag of the player
if r"><sp/>" in player1_name:
player1_name = player1_name.split("><sp/>")[1]
if r"><sp/>" in player2_name:
player2_name = player2_name.split("><sp/>")[1]
return {
"map": map_name,
"blizz_map": blizz_map,
"player1_name": str(player1_name),
"player1_race": str(player1_race),
"player2_name": str(player2_name),
"player2_race": str(player2_race),
"patch": patch,
"file": loc
}
except:
return {}
def process_sing_file(self, replay_file_name):
archive = mpyq.MPQArchive(replay_file_name)
contents = archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(contents)
base_build = header['m_version']['m_baseBuild']
try:
protocol = versions.build(base_build)
except Exception, e:
self.fail('Unsupported base build: {0} ({1})'.format(
base_build, str(e)))
def __init__(self, archive):
self.contents = archive.header['user_data_header']['content']
self.header = versions.latest().decode_replay_header(self.contents)
self.protocol = versions.build(self.header['m_version']['m_baseBuild'])
self.protocolDetails = self.protocol.decode_replay_details(
archive.read_file('replay.details'))
self.protocolInitData = self.protocol.decode_replay_initdata(
archive.read_file('replay.initData'))
self.playerInfo = self.getPlayerInfo()
self.playerIndex = self.getPlayerIndex()
self.oppIndex = self.getOpponentIndex()
def __init__(self, replay_path):
self.__archive = mpyq.MPQArchive(replay_path)
# Read the protocol header, this can be read with any protocol
contents = self.__archive.header['user_data_header']['content']
self.__header = latest().decode_replay_header(contents)
# The header's baseBuild determines which protocol to use
self.__base_build = self.__header['m_version']['m_baseBuild']
try:
self.__protocol = build(self.__base_build)
except:
print('Unsupported base build: ' + str(self.__base_build))
sys.exit(1)
def process_uploaded_replay(replayFiles):
print("Task Running...")
for file in replayFiles:
archive = mpyq.MPQArchive(file)
contents = archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(contents)
baseBuild = header['m_version']['m_baseBuild']
try:
protocol = versions.build(baseBuild)
analyze_sentiments(archive, protocol)
except ImportError as err:
print(err.args)
def load_protocol(replay_name):
"""Get a replay decoder protocol.
"""
archive = MPQArchive(replay_name)
contents = archive.header['user_data_header']['content']
latest_protocol = latest()
header = latest_protocol.decode_replay_header(contents)
build_version = header['m_version']['m_baseBuild']
try:
protocol = build(build_version)
except ImportError:
raise S2ProtocolNotFoundError(build_version)
else:
return archive, protocol
def __init__(self, path):
self.archive = mpyq.MPQArchive(path)
self.fallback_versions = None
_header_contents = self.archive.header['user_data_header']['content']
protocol = versions.latest()
_header = protocol.decode_replay_header(_header_contents)
self.base_build = _header['m_version']['m_baseBuild']
try:
self.protocol = versions.build(self.base_build)
except ImportError:
# fall back version
next_lower = closest_version(self.base_build, versions)[0]
log.info(
f"Missing Protocol {self.base_build}. Using Next Lower {next_lower}"
)
self.protocol = versions.build(next_lower)
self.fallback_versions = closest_version(self.base_build, versions)
self._header = _header
self._events = None
self._init_data = None
self._details = None
self._players = []
self._teams = None
self._attribute_events = None
self._tracker_events = None
self.match_events = []
self.snapshots: typing.List[Snapshot] = []
self.tracked_units = TrackedUnits()
self._time = None
self.units = []
self._message_lookup = {}
self._message_keys = []
self.segments = {
'early': {},
'three_teams': {},
'two_teams': {},
'final': {}
}
meta = json.loads(
self.archive.read_file('replay.gamemetadata.json').decode('utf-8'))
self.meta = meta
if meta['Title'] not in ['Zone Control CE', 'Zone Control CE Dev']:
raise NotZCReplay("Not a valid replay for game")
# Check to see if the replay is complete or not
if max([p['m_result']
for p in self.details.get('m_playerList', [])]) == 0:
raise IncompleteReplay("Replay is incomplete")
def cli():
parser = argparse.ArgumentParser(
prog='s2repdump',
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument('replay_file', help='.SC2Replay file to load')
parser.add_argument('-v',
'--verbose',
help='verbose logging; stacks up to 3',
action='count',
default=0)
parser.add_argument('-q', '--quiet', action='store_true')
parser.add_argument('--version',
action='version',
version='%(prog)s ' +
('%s (s2protocol %s)' %
(S2REPDUMP_VERSION, versions.latest().__name__[8:])))
# parser.add_argument('--json', help='json', action='store_true')
parser.add_argument('--players',
help='print info about players',
action='store_true')
parser.add_argument('--chat', help='chat messages', action='store_true')
parser.add_argument('--bank-list',
help='list SC2Bank\'s',
action='store_true')
parser.add_argument('--bank-rebuild',
help='rebuild SC2Bank files',
action='store_true')
parser.add_argument('--out',
help='output directory',
type=str,
default='./out')
parser.add_argument(
'--strict-mode',
help='do not try to decode replays if there\'s not matching protocol',
action='store_true')
args = parser.parse_args()
args.verbose = min(args.verbose, 3)
setup_logger()
logging.getLogger().setLevel(
[logging.WARN, logging.INFO, logging.DEBUG,
logging.NOTSET][args.verbose])
if args.quiet:
logging.getLogger().setLevel(logging.CRITICAL)
main(args)
def build_replay(self, path):
self.archive = mpyq.MPQArchive(path)
replay = self.archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(replay)
self.contents = self.archive.read_file('replay.tracker.events')
self.details = self.archive.read_file('replay.details')
self.game_events = self.archive.read_file('replay.game.events')
self.init_data = self.archive.read_file('replay.initData')
self.metadata = json.loads(
self.archive.read_file('replay.gamemetadata.json'))
base_build = header['m_version']['m_baseBuild']
try:
return (replay, versions.build(base_build))
except Exception as e:
raise Exception('Unsupported base build: {0} ({1!s})'.format(
base_build, e))
def race_winrate(directory):
# Using mypq, load the replay file
matcher = re.compile(r'\.SC2Replay$', re.IGNORECASE)
replays = [file for file in os.listdir(directory) if matcher.search(file)]
print("Found %d replays to scan" % len(replays))
# KEY: Name. VALUE: PlayerObject
race_dictionary = {
"Protoss": "P",
"Zerg": "Z",
"Terran": "T",
"异虫": "Z",
"星灵": "P",
"人类": "T"
}
matchup_dictionary = {"PvZ": 0, "PvT": 0, "ZvT": 0}
for replay in replays:
try:
# necessary stuff from s2protocol
archive = mpyq.MPQArchive(os.path.join(directory, replay))
contents = archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(contents)
base_build = header['m_version']['m_baseBuild']
protocol = versions.build(base_build)
# get the general info about the replay
contents = archive.read_file('replay.details')
result = protocol.decode_replay_details(contents)
player_list = result['m_playerList']
# player result is 1 if won, 2 if not.
player_result = [
player_list[0]['m_result'] == 1,
player_list[1]['m_result'] == 1
]
# ex: [P, Z]
player_races = [
player_list[0]['m_race'].decode('UTF-8'),
player_list[1]['m_race'].decode('UTF-8')
]
player_races = [race_dictionary[race] for race in player_races]
except:
print("error")
def update_battle_net_cache(replays, bnet_base):
"""Download the battle.net cache files needed by replays."""
test_looks_like_battle_net(bnet_base)
downloaded = 0
failed = set()
for replay_path in replays:
try:
archive = mpyq.MPQArchive(replay_path)
except ValueError:
print("Failed to parse replay:", replay_path)
continue
extracted = archive.extract()
contents = archive.header["user_data_header"]["content"]
header = s2versions.latest().decode_replay_header(contents)
base_build = header["m_version"]["m_baseBuild"]
prot = s2versions.build(base_build)
details_bytes = (extracted.get(b"replay.details")
or extracted.get(b"replay.details.backup"))
details = prot.decode_replay_details(details_bytes)
for map_handle in details["m_cacheHandles"]:
# server = map_handle[4:8].decode("utf-8").strip("\x00 ")
map_hash = binascii.b2a_hex(map_handle[8:]).decode("utf8")
file_type = map_handle[0:4].decode("utf8")
cache_path = os.path.join(bnet_base, "Cache", map_hash[0:2],
map_hash[2:4],
"%s.%s" % (map_hash, file_type))
url = DEPOT_URL_TEMPLATE.format(hash=map_hash, type=file_type)
if not os.path.exists(cache_path) and url not in failed:
mkdirs(os.path.dirname(cache_path))
print(url)
try:
urllib.request.urlretrieve(url, cache_path)
except urllib.error.HTTPError as e:
print("Download failed:", e)
failed.add(url)
else:
downloaded += 1
return downloaded
def read_protocol(archive: MPQArchive):
content = archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(content)
base_build = header['m_version']['m_baseBuild']
try:
return versions.build(base_build)
# https://github.com/Blizzard/s2protocol/issues/99
# If the protocol is not defined, the previous protocol often works anyway
except ImportError as e:
# print(f'Unable to import protocol{base_build}.py. Module does not exist')
base_path = os.path.dirname(versions.__file__)
protocols = set([p for p in os.listdir(base_path)])
while base_build > 0:
base_build -= 1
candidate = f'protocol{base_build}.py'
if candidate in protocols:
# print(f'Falling back to {candidate}')
return versions.build(base_build)
raise e
def get_already_played(self):
rs = replay_stats()
processes = []
for files in listdir(self.replay_folder):
if (path.splitext(files)[1] == '.SC2Replay'):
archive = mpyq.MPQArchive(self.replay_folder + '\\' + files)
contents = str(archive.header['user_data_header']['content'])
#figure out build version of replay
header = versions.latest().decode_replay_header(contents)
baseBuild = header['m_version']['m_baseBuild']
protocol = versions.build(baseBuild)
#decode game events
contents = archive.read_file('replay.details')
details = protocol.decode_replay_details(contents)
datetime_of_replay = datetime.utcfromtimestamp(
((details['m_timeUTC']) / (10000000) - 11644473600 +
((details['m_timeLocalOffset']) / 10000000)))
p1 = Process()
if (datetime_of_replay.date() == datetime.today().date()):
p1 = Process(target=self.minutes_in_replay,
args=((self.replay_folder + '\\' + files),
rs.replay_stats_dict, True))
#start of week credit https://stackoverflow.com/questions/39441639/getting-the-date-of-the-first-day-of-the-week?rq=1
elif (datetime_of_replay.date() >=
(datetime.today() - timedelta(
days=datetime.today().isoweekday() % 7)).date()):
p1 = Process(target=self.minutes_in_replay,
args=((self.replay_folder + '\\' + files),
rs.replay_stats_dict))
processes.append(p1)
p1.start()
for process in processes:
process.join()
return rs.replay_stats_dict
def main():
parser = argparse.ArgumentParser()
parser.add_argument('replay_file',
help='.SC2Replay file to load',
nargs='?')
args = parser.parse_args()
# Check/test the replay file
if args.replay_file is None:
print(sys.stderr, ".S2Replay file not specified")
sys.exit(1)
archive = mpyq.MPQArchive(args.replay_file)
# HEADER
# contents = archive.header['user_data_header']['content']
# header = versions.latest().decode_replay_header(contents)
contents = read_contents(archive, 'replay.game.events')
details = versions.latest().decode_replay_game_events(contents)
for x in details['m_playerList']:
print('hello')
def main():
"""
Get command line arguments and invoke the command line functionality.
"""
filters = []
parser = argparse.ArgumentParser()
parser.add_argument('replay_file', help='.SC2Replay file to load',
nargs='?')
parser.add_argument("--gameevents", help="print game events",
action="store_true")
parser.add_argument("--messageevents", help="print message events",
action="store_true")
parser.add_argument("--trackerevents", help="print tracker events",
action="store_true")
parser.add_argument("--attributeevents", help="print attributes events",
action="store_true")
parser.add_argument("--attributeparse", help="parse attributes events",
action="store_true")
parser.add_argument("--header", help="print protocol header",
action="store_true")
parser.add_argument("--metadata", help="print game metadata",
action="store_true")
parser.add_argument("--details", help="print protocol details",
action="store_true")
parser.add_argument("--details_backup", help="print protocol anoynmized details",
action="store_true")
parser.add_argument("--initdata", help="print protocol initdata",
action="store_true")
parser.add_argument("--all", help="print all data",
action="store_true")
parser.add_argument("--quiet", help="disable printing",
action="store_true")
parser.add_argument("--stats", help="print stats",
action="store_true")
parser.add_argument("--diff", help="diff two protocols",
default=None,
action="store")
parser.add_argument("--versions", help="show all protocol versions",
action="store_true")
parser.add_argument("--types", help="show type information in event output",
action="store_true")
parser.add_argument("--json", help="print output as json",
action="store_true")
parser.add_argument("--ndjson", help="print output as ndjson (newline delimited)",
action="store_true")
parser.add_argument("--profile", help="Whether to profile or not",
action="store_true")
args = parser.parse_args()
if args.profile:
pr = cProfile.Profile()
pr.enable()
# TODO: clean up the command line arguments to allow cleaner sub-command
# style commands
# List all protocol versions
if args.versions:
files = list_all()
pattern = re.compile('^protocol([0-9]+).py$')
captured = []
for f in files:
captured.append(pattern.match(f).group(1))
if len(captured) == 8:
print(captured[0:8])
captured = []
print(captured)
return
# Diff two protocols
if args.diff and args.diff is not None:
version_list = args.diff.split(',')
if len(version_list) < 2:
print("--diff requires two versions separated by comma e.g. --diff=1,2",
file=sys.stderr)
sys.exit(1)
diff(version_list[0], version_list[1])
return
# Check/test the replay file
if args.replay_file is None:
print(".S2Replay file not specified", file=sys.stderr)
sys.exit(1)
archive = MPQArchive(args.replay_file)
filters = []
if args.json:
filters.insert(0, JSONOutputFilter(sys.stdout))
elif args.ndjson:
filters.insert(0, NDJSONOutputFilter(sys.stdout))
elif not args.quiet:
filters.insert(0, PrettyPrintFilter(sys.stdout))
if args.types:
filters.insert(0, TypeDumpFilter())
if args.stats:
filters.insert(0, StatCollectionFilter())
def process_event(event):
for f in filters:
event = f.process(event)
# Read the protocol header, this can be read with any protocol
contents = archive.header['user_data_header']['content']
header = latest().decode_replay_header(contents)
if args.header:
process_event(header)
# The header's baseBuild determines which protocol to use
baseBuild = header['m_version']['m_baseBuild']
try:
protocol = build(baseBuild)
except Exception as e:
print('Unsupported base build: {0} ({1!s})'.format(baseBuild, e),
file=sys.stderr)
sys.exit(1)
# Process game metadata
if args.all or args.metadata:
contents = read_contents(archive, 'replay.gamemetadata.json')
process_event(json.loads(contents))
# Print protocol details
if args.all or args.details:
contents = read_contents(archive, 'replay.details')
details = protocol.decode_replay_details(contents)
details = process_details_data(details)
process_event(details)
# Print protocol details
if args.all or args.details_backup:
contents = read_contents(archive, 'replay.details.backup')
details_backup = protocol.decode_replay_details(contents)
details_backup = process_details_data(details_backup)
process_event(details_backup)
# Print protocol init data
if args.all or args.initdata:
contents = read_contents(archive, 'replay.initData')
initdata = protocol.decode_replay_initdata(contents)
initdata = process_init_data(initdata)
process_event(initdata)
# Print game events and/or game events stats
if args.all or args.gameevents:
contents = read_contents(archive, 'replay.game.events')
map(process_event, protocol.decode_replay_game_events(contents))
# Print message events
if args.all or args.messageevents:
contents = read_contents(archive, 'replay.message.events')
map(process_event, protocol.decode_replay_message_events(contents))
# Print tracker events
if args.all or args.trackerevents:
if hasattr(protocol, 'decode_replay_tracker_events'):
contents = read_contents(archive, 'replay.tracker.events')
map(process_event, protocol.decode_replay_tracker_events(contents))
# Print attributes events
if args.all or args.attributeevents or args.attributeparse:
contents = read_contents(archive, 'replay.attributes.events')
attributes = protocol.decode_replay_attributes_events(contents)
# Process raw attribute events structure
if args.attributeevents:
process_event(attributes)
# Convert attributes to higher level requested data, will
# call prcess_event for each new event that it creates
if args.attributeparse:
process_scope_attributes(attributes['scopes'], process_event)
for f in filters:
f.finish()
if args.profile:
pr.disable()
print("Profiler Results")
print("----------------")
s = get_stream()
sortby = 'cumulative'
ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
ps.print_stats()
print(s.getvalue())
def organize_replays(directory, output_directory, teams, aliases):
"""copies replays to another directory with standardized format
Args:
directory (string): replay directory
output_directory (string): new replay directory
teams (dict): dict with key = player, value = team
"""
# Using mypq, load the replay file
matcher = re.compile(r'\.SC2Replay$', re.IGNORECASE)
replays = [file for file in os.listdir(directory) if matcher.search(file)]
print("Found %d replays to scan" % len(replays))
#The dates corresponding to each week
week_time = define_cea_date_ranges()
race_dictionary = {
"Protoss": "P",
"Zerg": "Z",
"Terran": "T",
"异虫": "Z",
"星灵": "P",
"人类": "T"
}
map_dictionary = {
"Automaton LE": "Automaton LE",
"机械城 天梯版": "Automaton LE",
"Kings Cove LE": "Kings Cove LE",
"国王藏宝地天梯版": "Kings Cove LE",
"Year Zero LE": "Year Zero LE",
"New Repugnancy LE": "New Repugnancy LE",
"Cyber Forest LE": "Cyber Forest LE",
"赛博森林天梯版": "Cyber Forest LE",
"Port Aleksander LE": "Port Aleksander LE",
"Kairos Junction LE": "Kairos Junction LE",
"Acropolis LE": "Acropolis LE",
"Thunderbird LE": "Thunderbird LE",
"Turbo Cruise 84 LE": "Turbo Cruise 84 LE",
"Triton LE": "Triton LE",
"Disco Bloodbath LE": "Disco Bloodbath LE",
"Winters Gate LE": "Winters Gate LE",
"Ephemeron LE": "Ephemeron LE",
"World of Sleepers LE": "World of Sleepers LE",
}
# Windows has to close the file before moving them, so
# files must be stored in a dictionary.
renamed_files = {}
# 2 dimensional dictionary that stores matchups per week.
# KEY 1: Week, VALUE 1: Dictionary<string,list>
# ex: matchup_dictionary['Week1']['Microsoft Macrohard']
matchup_dictionary = {}
for replay in replays:
try:
# necessary stuff from s2protocol
archive = mpyq.MPQArchive(os.path.join(directory, replay))
contents = archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(contents)
base_build = header['m_version']['m_baseBuild']
# Build 76114 was never added to s2protocol.
if base_build == 76811:
protocol = versions.build(76114)
else:
protocol = versions.build(base_build)
# get the general info about the replay
contents = archive.read_file('replay.details')
result = protocol.decode_replay_details(contents)
player_list = result['m_playerList']
# string array with 2 player names, i.e [Feniks, DarthNoob]
player_names = [
erase_punctuation(player_list[0]['m_name']),
erase_punctuation(player_list[1]['m_name'])
]
# resolve aliases for players who play under several accounts
for i in range(len(player_names)):
if player_names[i].lower() in aliases:
player_names[i] = aliases[player_names[i].lower()]
# ex: [P, Z]
player_races = [
player_list[0]['m_race'].decode('UTF-8'),
player_list[1]['m_race'].decode('UTF-8')
]
player_races = [race_dictionary[race] for race in player_races]
# ex: [Alexa 12 Pool, Google Noobernetes]
player_teams = [
find_team(teams, player_names[0]),
find_team(teams, player_names[1])
]
# Keep naming consistent by always putting players alphabetized by team
if player_teams[1] < player_teams[0]:
player_races = [player_races[1], player_races[0]]
player_names = [player_names[1], player_names[0]]
player_teams = [player_teams[1], player_teams[0]]
# ex: Week4
replay_time = result['m_timeUTC']
week_played = get_date_played(week_time, get_time(replay_time))
# Updates the Matchup Dictionary
if week_played not in matchup_dictionary:
matchup_dictionary[week_played] = {}
matchup_dictionary[week_played].setdefault(
player_teams[0], []).append(player_names[1])
matchup_dictionary[week_played].setdefault(
player_teams[1], []).append(player_names[0])
# ex: Kings Cove LE
map_name = result['m_title'].decode('UTF-8').translate(
str.maketrans('', '', string.punctuation))
# In case map name is not in English.
if not map_name.replace(
" ", "").isalnum() and map_name not in map_dictionary:
print("Map name %s not recognized" % map_name)
print("\t%s, %s" % (week_played, map_name))
print("\t%s: %s (%s)" %
(player_teams[0], player_names[0], player_races[0]))
print("\t%s: %s (%s)" %
(player_teams[1], player_names[1], player_races[1]))
continue
elif map_name in map_dictionary:
map_name = map_dictionary[map_name]
src = os.path.join(directory, replay)
# don't continue for unknown players so they can be fixed
if UNKNOWN_TEAM in player_teams:
print(
"Couldn't find the team for one of the players. Here's what we know:"
)
print("\t%s, %s" % (week_played, map_name))
print("\t%s: %s (%s)" %
(player_teams[0], player_names[0], player_races[0]))
print("\t%s: %s (%s)" %
(player_teams[1], player_names[1], player_races[1]))
continue
# copy into team/player/matchup folders
copy_into_path(
src, "-".join(
[player_teams[1], player_names[1], map_name, week_played]),
[
player_teams[0],
"%s (%s)" %
(player_names[0], player_races[0]), "vs " + player_races[1]
])
copy_into_path(
src, "-".join(
[player_teams[0], player_names[0], map_name, week_played]),
[
player_teams[1],
"%s (%s)" %
(player_names[1], player_races[1]), "vs " + player_races[0]
])
# rename the original to avoid name conflicts and make it clear what's been processed
to_rename = "-".join([
week_played, player_teams[0], player_teams[1], player_names[0],
player_names[1], player_races[0], player_races[1], map_name
]).replace(" ", "_") + ".SC2Replay"
dst = os.path.join(output_directory, to_rename)
if src.lower() != dst.lower():
counts['replays processed'] += 1
os.makedirs(output_directory, exist_ok=True)
renamed_files[src] = dst
else:
counts['replays were already processed'] += 1
except:
print("Error processing replay: %s" % replay)
traceback.print_exc()
for key, value in renamed_files.items():
shutil.move(key, value)
for count_name, count in sorted(counts.items()):
print(count, count_name)
# Identify players who are not recognized
identify_unknown_players(matchup_dictionary, teams)
def test_latest(self):
p = _versions.latest()
self.assertIsNotNone(p)
def __init__(self, filename, strict_mode=False):
def read_archive_contents(name):
content = self.archive.read_file(name)
if not content:
logging.warning('MPQ missing file: "%s"' % name)
return content
def must_read_archive_contents(name):
content = read_archive_contents(name)
if not content:
logging.critical('MPQ missing required file: "%s"' % name)
sys.exit(1)
return content
self.archive = mpyq.MPQArchive(filename)
content = self.archive.header['user_data_header']['content']
self.header = versions.latest().decode_replay_header(content)
self.proto_build = self.header['m_version']['m_baseBuild']
logging.info('Protocol build %d' % (self.proto_build))
# >= 24764 (after HotS came out)
# in WoL observers weren't seperated from players and working slot concept didn't exist
self.features.user_id_driven = self.proto_build >= 24764
self.features.working_slots = self.proto_build >= 24764
# https://liquipedia.net/starcraft2/Patch_2.0.4
# tracker section should be present in replays from build 24944
self.features.tracker_present = self.proto_build >= 25604
# https://liquipedia.net/starcraft2/Patch_2.0.8
# SPlayerSetupEvent should be included in the tracker from build 25604
self.features.tracker_player_pid = self.proto_build >= 25604
try:
self.protocol = versions.build(self.proto_build)
except ImportError as e:
logging.warning('Unsupported protocol: (%s)' % (str(e)))
if strict_mode:
logging.critical('Aborting, because of strict mode.')
sys.exit(1)
proto_mods = [
int(re.sub(r'^protocol([0-9]+)\.py$', '\\1', x))
for x in versions.list_all()
]
tmp = [abs(i - self.proto_build) for i in proto_mods]
idx = tmp.index(min(tmp))
# always favorize newer protos in case of up to date replays
if self.proto_build > 70000 and len(proto_mods) == (idx + 1):
idx += 1
fallbackBuild = proto_mods[idx]
self.protocol = versions.build(fallbackBuild)
logging.warning('Attempting to use %s instead' %
self.protocol.__name__)
# read files
self.details = self.protocol.decode_replay_details(
must_read_archive_contents('replay.details'))
self.init_data = self.protocol.decode_replay_initdata(
must_read_archive_contents('replay.initData'))
self.gameevents = peekable(
self.protocol.decode_replay_game_events(
must_read_archive_contents('replay.game.events')))
content = read_archive_contents('replay.message.events')
self.messageevents = peekable(
self.protocol.decode_replay_message_events(content
) if content else None)
content = read_archive_contents('replay.tracker.events')
self.features.tracker_present = bool(content)
self.trackerevents = peekable(
self.protocol.decode_replay_tracker_events(content
) if content else None)
# setup
self.participants = setup_participants(self)
self.banks = setup_banks(self)
def cli():
ver = ('%s (s2protocol %s)' %
(S2REPDUMP_VERSION, versions.latest().__name__[8:]))
parser = argparse.ArgumentParser(
prog='s2repdump',
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument('replay_file', help='.SC2Replay file to load')
comg = parser.add_argument_group('common')
comg.add_argument('-v',
'--verbose',
help='verbose logging; stacks up to 3',
action='count',
default=0)
comg.add_argument('-q', '--quiet', action='store_true')
comg.add_argument('-V',
'--version',
action='version',
version='%(prog)s ' + ver)
comg.add_argument('-j',
'--json',
help='output data as JSON',
action='store_true')
comg.add_argument('-J',
'--json-compact',
help='output data as compact JSON',
action='store_true')
comg.add_argument('-O',
'--out',
help='output directory',
type=str,
default='./out')
comg.add_argument(
'-f',
'--force',
action='store_true',
help=
'force certain operations that otherwise would\'ve been aborted - such overwriting existing files'
)
comg.add_argument(
'--strict',
help='do not try to decode replays if there\'s not matching protocol',
action='store_true')
comg = parser.add_argument_group('actions')
comg.add_argument('-d',
'--decode',
choices=['info', 'players', 'chat', 'banks'],
type=str,
action='append',
default=[],
help='decode and output specified data section')
comg.add_argument('-R',
'--bank-rebuild',
help='rebuild SC2Bank files',
action='store_true')
args = parser.parse_args()
if args.json_compact:
args.json = True
args.verbose = min(args.verbose, 3)
setup_logger()
logging.getLogger().setLevel(
[logging.WARN, logging.INFO, logging.DEBUG,
logging.NOTSET][args.verbose])
if args.quiet:
logging.getLogger().setLevel(logging.CRITICAL)
try:
main(args)
except:
logging.exception('Unexpected error occured')
sys.exit(1)
def __init__(self, replay_path=''):
self.players = []
self.series_flag = -1
self.archive = None
self.baseBuild = None
self.header = None
self.protocol = None
self.details = None
self.local_path = replay_path
self.replay_name = ''
self.UTC_timestamp = 0
if replay_path is not '':
#generate MPQ archive
self.archive = mpyq.MPQArchive(replay_path)
#get the replays protocol version
contents = self.archive.header['user_data_header']['content']
self.header = versions.latest().decode_replay_header(contents)
# The header's baseBuild determines which protocol to use
#part of this code was modified from
#s2_cli.py @ https://github.com/Blizzard/s2protocol/tree/master/s2protocol
self.baseBuild = self.header['m_version']['m_baseBuild']
try:
self.protocol = versions.build(self.baseBuild)
except Exception as e:
print >> sys.stderr, 'Unsupported base build: {0} ({1})'.format(
self.baseBuild, str(e))
sys.exit(1)
#replay details
contents = self.archive.read_file('replay.details')
self.details = self.protocol.decode_replay_details(contents)
self.UTC_timestamp = self.details['m_timeUTC']
#pre process for matchup and names
num_players = len(self.details['m_playerList'])
for i in range(num_players):
player = None
name = self.details['m_playerList'][i]['m_name'].decode()
race = self.details['m_playerList'][i]['m_race']
clan = ''
team = self.details['m_playerList'][i]['m_teamId']
if name.find('<') > -1:
info = self.__beautify_name(name)
clan = info[0]
name = info[1]
player = create_player(name, race, clan, team)
self.players.append(player)
if clan is '':
self.replay_name = self.replay_name + name + ' vs '
else:
self.replay_name = self.replay_name + clan + ' ' + name + ' vs '
#must take last ' vs ' off and put proper exstension on
self.replay_name = self.replay_name[:-4] + '.SC2Replay'
def minutes_in_replay(self, filename, rs, *args):
archive = mpyq.MPQArchive(filename)
contents = str(archive.header['user_data_header']['content'])
metadata = ast.literal_eval(
archive.read_file('replay.gamemetadata.json'))
#figure out build version of replay
header = versions.latest().decode_replay_header(contents)
baseBuild = header['m_version']['m_baseBuild']
protocol = versions.build(baseBuild)
contents = archive.read_file('replay.initData')
initData = protocol.decode_replay_initdata(contents)
#data only matters if it's a 1v1
if (initData['m_syncLobbyState']['m_gameDescription']['m_gameOptions']
['m_competitive'] and initData['m_syncLobbyState']
['m_gameDescription']['m_maxPlayers'] == 2):
contents = archive.read_file('replay.details')
details = protocol.decode_replay_details(contents)
player_race = ""
enemy_race = ""
result = ""
for player in details['m_playerList']:
if (player['m_toon']['m_id'] == self.player_id):
player_race = player['m_race']
for meta_player in metadata['Players']:
if meta_player['PlayerID'] == player[
'm_workingSetSlotId'] + 1:
result = meta_player["Result"]
#Have to make a copy because nested dicts don't update for Manager dicts
contents = archive.read_file('replay.game.events')
game_events = protocol.decode_replay_game_events(contents)
for event in game_events:
if event['_event'] == 'NNet.Game.SGameUserLeaveEvent':
#could possible use metadata 'duration'?
#possibly more accurate: first player leave vs duration of replay
game_duration = (event['_gameloop'] / 22.44444 /
60)
break
#needs to be done out of / after for loop due to weird behavior with syncing dict
minutes_played_today_copy = rs["minutes_played_today"]
minutes_played_week_copy = rs["minutes_played_week"]
minutes_played_week_copy[player_race] += game_duration
if (args):
minutes_played_today_copy[player_race] += game_duration
rs["minutes_played_today"] = minutes_played_today_copy
rs["minutes_played_week"] = minutes_played_week_copy
else:
enemy_race = player['m_race']
#Have to make a copy because nested dicts don't update for Manager dicts
copy_wins = rs["wins"]
copy_games = rs["games"]
if (player_race == "Terran"):
if (enemy_race == "Terran"):
copy_games["TvT"] += 1
if (result == "Win"):
copy_wins["TvT"] += 1
elif (enemy_race == "Zerg"):
copy_games["TvZ"] += 1
if (result == "Win"):
copy_wins["TvZ"] += 1
else:
copy_games["TvP"] += 1
if (result == "Win"):
copy_wins["TvP"] += 1
elif (player_race == "Zerg"):
if (enemy_race == "Terran"):
copy_games["ZvT"] += 1
if (result == "Win"):
copy_wins["ZvT"] += 1
elif (enemy_race == "Zerg"):
copy_games["ZvZ"] += 1
if (result == "Win"):
copy_wins["ZvZ"] += 1
else:
copy_games["ZvP"] += 1
if (result == "Win"):
copy_wins["ZvP"] += 1
elif (player_race == "Protoss"):
if (enemy_race == "Terran"):
copy_games["PvT"] += 1
if (result == "Win"):
copy_wins["PvT"] += 1
elif (enemy_race == "Zerg"):
copy_games["PvZ"] += 1
if (result == "Win"):
copy_wins["PvZ"] += 1
else:
copy_games["PvP"] += 1
if (result == "Win"):
copy_wins["PvP"] += 1
rs["wins"] = copy_wins
rs["games"] = copy_games
return rs
def s2_parse_replay(file,
try_lastest=True,
parse_events=True,
onlyBlizzard=False,
withoutRecoverEnabled=False,
return_raw=False,
return_events=False,
try_closest=False):
""" Function parsing the replay and returning a replay class
`try_lastest=False` doesn't try the lastest protocol version
`parse_events = False` prevents parsing replay events, less accurate game length
`onlyBlizzard = True` returns `None` for non-blizzard replays. Commanders have to be found.
`withoutRecoverEnabled = True` returns `None` for games with game recovery enabled
`return_raw = True` returns raw data as well
`return_events = True` returns events as well
"""
# Exit straight away if onlyBlizzard enforced
if onlyBlizzard and '[MM]' in file:
return None
# Open archive
archive = mpyq.MPQArchive(file)
contents = archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(contents)
replay_build = header['m_version']['m_baseBuild']
# If the build is in a known list of protocols that aren't included but work, replace the build by the version that works.
base_build = valid_protocols.get(replay_build, replay_build)
try:
protocol = versions.build(base_build)
used_build = base_build
except:
if try_closest:
used_build = find_closest_values(base_build, valid_protocols)[0]
protocol = versions.build(used_build)
elif try_lastest:
protocol = versions.latest()
used_build = protocol_build().split('.')[-2]
else:
return None
# Get player info
player_info = archive.read_file('replay.details')
player_info = protocol.decode_replay_details(player_info)
# Exit if onlyBlizzard maps enforced
if onlyBlizzard and not player_info['m_isBlizzardMap']:
return None
# Exit if game can be recovered
if withoutRecoverEnabled and not player_info['m_disableRecoverGame']:
return None
# Get detailed info
detailed_info = archive.read_file('replay.initData')
detailed_info = protocol.decode_replay_initdata(detailed_info)
# Get metadata
metadata = json.loads(archive.read_file('replay.gamemetadata.json'))
# Messages
messages = archive.read_file('replay.message.events')
messages = protocol.decode_replay_message_events(messages)
events = list()
if parse_events:
# Get game events
game_events = archive.read_file('replay.game.events')
game_events = protocol.decode_replay_game_events(game_events)
# Get tracker events
tracker_events = archive.read_file('replay.tracker.events')
tracker_events = protocol.decode_replay_tracker_events(tracker_events)
# Merge them together
events = list(game_events) + list(tracker_events)
events = sorted(events, key=lambda x: x['_gameloop'])
# Create output
replay = dict()
replay['file'] = file
replay['build'] = {
'replay_build': replay_build,
'protocol_build': used_build
}
replay['date'] = time.strftime('%Y:%m:%d:%H:%M:%S',
time.localtime(os.path.getmtime(file)))
if metadata['Title'] in map_names:
replay['map_name'] = map_names[metadata['Title']]['EN']
else:
replay['map_name'] = metadata['Title']
replay['isBlizzard'] = player_info['m_isBlizzardMap']
replay['extension'] = detailed_info['m_syncLobbyState'][
'm_gameDescription']['m_hasExtensionMod']
replay['brutal_plus'] = detailed_info['m_syncLobbyState']['m_lobbyState'][
'm_slots'][0].get('m_brutalPlusDifficulty', 0)
replay['length'] = metadata['Duration']
replay['start_time'] = get_start_time(events)
replay['last_deselect_event'] = get_last_deselect_event(events)
replay['last_deselect_event'] = replay['last_deselect_event'] if replay[
'last_deselect_event'] != None else replay['length']
replay['result'] = 'Victory' if metadata['Players'][0][
'Result'] == 'Win' or metadata['Players'][1][
'Result'] == 'Win' else 'Defeat'
if replay['result'] == 'Victory' and parse_events:
replay['accurate_length'] = replay['last_deselect_event'] - replay[
'start_time']
replay['end_time'] = replay['last_deselect_event']
else:
replay['accurate_length'] = replay['length'] - replay['start_time']
replay['end_time'] = replay['length']
if parse_events:
try:
replay['mutators'] = identify_mutators(
events, extension=replay['extension'], mm='[MM]' in file)
except:
replay['mutators'] = list()
logger.error(traceback.format_exc())
else:
replay['mutators'] = list()
replay['form_alength'] = time.strftime(
'%H:%M:%S', time.gmtime(replay['accurate_length']))
replay['form_alength'] = replay['form_alength'] if not replay[
'form_alength'].startswith('00:') else replay['form_alength'][3:]
# Add data to players
replay['players'] = list()
for idx, player in enumerate(metadata['Players']):
p = {
'apm': player['APM'],
'result': player['Result'],
'pid': player['PlayerID']
}
p['apm'] = round(p['apm'] * replay['length'] /
replay['accurate_length'])
replay['players'].append(p)
for idx, player in enumerate(player_info['m_playerList']):
replay['players'][idx]['name'] = player['m_name'].decode()
replay['players'][idx]['race'] = player['m_race'].decode()
replay['players'][idx][
'observer'] = False if player['m_observe'] == 0 else True
if idx == 0:
replay['region'] = region_dict.get(player['m_toon']['m_region'],
'')
commander_found = False
for idx, player in enumerate(
detailed_info['m_syncLobbyState']['m_lobbyState']['m_slots']):
if idx < len(replay['players']):
replay['players'][idx]['masteries'] = player[
'm_commanderMasteryTalents']
replay['players'][idx]['commander'] = player['m_commander'].decode(
)
replay['players'][idx]['commander_level'] = player[
'm_commanderLevel']
replay['players'][idx]['commander_mastery_level'] = player[
'm_commanderMasteryLevel']
replay['players'][idx]['prestige'] = player.get(
'm_selectedCommanderPrestige', 0)
replay['players'][idx]['prestige_name'] = prestige_names.get(
replay['players'][idx]['commander'],
dict()).get(replay['players'][idx]['prestige'], '')
replay['players'][idx]['handle'] = player['m_toonHandle'].decode()
replay['players'][idx]['difficulty'] = player['m_difficulty']
if not replay['players'][idx]['commander'] in {None, ''}:
commander_found = True
# Exit if onlyBlizzard maps enforced
if onlyBlizzard and not commander_found:
return None
# Player names
for idx, player in enumerate(
detailed_info['m_syncLobbyState']['m_userInitialData']):
_name = player['m_name'].decode()
if idx < len(replay['players']) and _name != '':
replay['players'][idx]['name'] = _name
# Insert dummy players to positions: zero & two if playing alone.
replay['players'].insert(0, {'pid': 0})
if replay['players'][2]['pid'] != 2:
replay['players'].insert(2, {'pid': 2})
# Enemy race
replay['enemy_race'] = replay['players'][3]['race']
# Difficulty
diff_1 = diff_dict.get(replay['players'][3]['difficulty'], '')
diff_2 = diff_dict.get(replay['players'][4]['difficulty'], '')
replay['difficulty'] = (diff_1, diff_2)
# Delete difficulty per player as it's not accurate for human players and we are returning map difficulty already
for player in replay['players']:
player.pop('difficulty', None)
# Extended difficulty (including B+)
if replay['brutal_plus'] > 0:
replay['ext_difficulty'] = f'B+{replay["brutal_plus"]}'
elif diff_1 == diff_2:
replay['ext_difficulty'] = diff_1
else:
replay['ext_difficulty'] = f'{diff_1}/{diff_2}'
# Messages
messages = [m for m in messages if m.get('m_string', None) != None]
replay['messages'] = [{
'text': m['m_string'].decode(),
'player': m['_userid']['m_userId'] + 1,
'time': m['_gameloop'] / 16
} for m in messages]
# Events
if return_events:
replay['events'] = events
# Raw
if return_raw:
replay['raw'] = {
'metadata': metadata,
'player_info': player_info,
'detailed_info': detailed_info
}
if replay_build != used_build and not replay_build in valid_protocols:
logger.info(
f'Succesfully replaced build {replay_build} with build {used_build}!'
)
return replay
def main(injectoutput, injectargs):
"""
Get command line arguments and invoke the command line functionality.
"""
filters = []
parser = argparse.ArgumentParser()
parser.add_argument('replay_file',
help='.SC2Replay file to load',
nargs='?')
parser.add_argument("--gameevents",
help="print game events",
action="store_true")
parser.add_argument("--messageevents",
help="print message events",
action="store_true")
parser.add_argument("--trackerevents",
help="print tracker events",
action="store_true")
parser.add_argument("--attributeevents",
help="print attributes events",
action="store_true")
parser.add_argument("--attributeparse",
help="parse attributes events",
action="store_true")
parser.add_argument("--header",
help="print protocol header",
action="store_true")
parser.add_argument("--metadata",
help="print game metadata",
action="store_true")
parser.add_argument("--details",
help="print protocol details",
action="store_true")
parser.add_argument("--details_backup",
help="print protocol anoynmized details",
action="store_true")
parser.add_argument("--initdata",
help="print protocol initdata",
action="store_true")
parser.add_argument("--all", help="print all data", action="store_true")
parser.add_argument("--quiet",
help="disable printing",
action="store_true")
parser.add_argument("--stats", help="print stats", action="store_true")
parser.add_argument("--diff",
help="diff two protocols",
default=None,
action="store")
parser.add_argument("--versions",
help="show all protocol versions",
action="store_true")
parser.add_argument("--types",
help="show type information in event output",
action="store_true")
parser.add_argument("--json",
help="print output as json",
action="store_true")
parser.add_argument("--ndjson",
help="print output as ndjson (newline delimited)",
action="store_true")
parser.add_argument("--profile",
help="Whether to profile or not",
action="store_true")
args = parser.parse_args(injectargs)
if args.profile:
pr = cProfile.Profile()
pr.enable()
# TODO: clean up the command line arguments to allow cleaner sub-command
# style commands
# List all protocol versions
if args.versions:
files = list_all()
pattern = re.compile('^protocol([0-9]+).py$')
captured = []
for f in files:
captured.append(pattern.match(f).group(1))
if len(captured) == 8:
print(captured[0:8])
captured = []
print(captured)
return
# Diff two protocols
if args.diff and args.diff is not None:
version_list = args.diff.split(',')
if len(version_list) < 2:
print(
"--diff requires two versions separated by comma e.g. --diff=1,2",
file=sys.stderr)
sys.exit(1)
diff(version_list[0], version_list[1])
return
# Check/test the replay file
if args.replay_file is None:
print(".S2Replay file not specified", file=sys.stderr)
sys.exit(1)
archive = MPQArchive(args.replay_file)
filters = []
if args.json:
filters.insert(0, JSONOutputFilter(injectoutput))
elif args.ndjson:
filters.insert(0, NDJSONOutputFilter(sys.stdout))
elif not args.quiet:
filters.insert(0, PrettyPrintFilter(sys.stdout))
if args.types:
filters.insert(0, TypeDumpFilter())
if args.stats:
filters.insert(0, StatCollectionFilter())
def process_event(event):
for f in filters:
event = f.process(event)
# Read the protocol header, this can be read with any protocol
contents = archive.header['user_data_header']['content']
header = latest().decode_replay_header(contents)
if args.header:
process_event(header)
# The header's baseBuild determines which protocol to use
baseBuild = header['m_version']['m_baseBuild']
try:
protocol = build(baseBuild)
except Exception as e:
print('Unsupported base build: {0} ({1!s})'.format(baseBuild, e),
file=sys.stderr)
sys.exit(1)
# Process game metadata
if args.all or args.metadata:
contents = read_contents(archive, 'replay.gamemetadata.json')
process_event(json.loads(contents))
# Print protocol details
if args.all or args.details:
contents = read_contents(archive, 'replay.details')
details = protocol.decode_replay_details(contents)
details = process_details_data(details)
process_event(details)
# Print protocol details
if args.all or args.details_backup:
contents = read_contents(archive, 'replay.details.backup')
details_backup = protocol.decode_replay_details(contents)
details_backup = process_details_data(details_backup)
process_event(details_backup)
# Print protocol init data
if args.all or args.initdata:
contents = read_contents(archive, 'replay.initData')
initdata = protocol.decode_replay_initdata(contents)
initdata = process_init_data(initdata)
process_event(initdata)
# Print game events and/or game events stats
if args.all or args.gameevents:
contents = read_contents(archive, 'replay.game.events')
map(process_event, protocol.decode_replay_game_events(contents))
# Print message events
if args.all or args.messageevents:
contents = read_contents(archive, 'replay.message.events')
map(process_event, protocol.decode_replay_message_events(contents))
# Print tracker events
if args.all or args.trackerevents:
if hasattr(protocol, 'decode_replay_tracker_events'):
contents = read_contents(archive, 'replay.tracker.events')
map(process_event, protocol.decode_replay_tracker_events(contents))
# Print attributes events
if args.all or args.attributeevents or args.attributeparse:
contents = read_contents(archive, 'replay.attributes.events')
attributes = protocol.decode_replay_attributes_events(contents)
# Process raw attribute events structure
if args.attributeevents:
process_event(attributes)
# Convert attributes to higher level requested data, will
# call prcess_event for each new event that it creates
if args.attributeparse:
process_scope_attributes(attributes['scopes'], process_event)
for f in filters:
f.finish()
if args.profile:
pr.disable()
print("Profiler Results")
print("----------------")
s = get_stream()
sortby = 'cumulative'
ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
ps.print_stats()
print(s.getvalue())
def compile_stats(directory, nicknames_dict):
# Using mypq, load the replay file
matcher = re.compile(r'\.SC2Replay$', re.IGNORECASE)
replays = [file for file in os.listdir(directory) if matcher.search(file)]
print("Found %d replays to scan" % len(replays))
# KEY: Name. VALUE: PlayerObject
player_dictionary = {}
race_dictionary = {
"Protoss": "P",
"Zerg": "Zerg",
"Terran": "T",
"Rand": "Random",
"Prot": "Protoss",
"Terr": "Terran",
"异虫": "Z",
"星灵": "P",
"人类": "T"
}
# Manual MMR overrides. Insert new entries if you want to manually override a player's MMR.
# ex: { "You" : 6700 }
mmr_exceptions = {}
for replay in replays:
try:
# necessary stuff from s2protocol
archive = mpyq.MPQArchive(os.path.join(directory, replay))
contents = archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(contents)
base_build = header['m_version']['m_baseBuild']
if base_build == 76811:
protocol = versions.build(76114)
else:
protocol = versions.build(base_build)
# get the general info about the replay
contents = archive.read_file('replay.details')
result = protocol.decode_replay_details(contents)
player_list = result['m_playerList']
# get the metadata info about the replay
metadata_contents = archive.read_file('replay.gamemetadata.json')
metadata_json = json.loads(metadata_contents.decode('utf-8'))
# Get MMR, APM for each player
player_mmr = [
get_metadata_key(metadata_json, 'MMR', 0),
get_metadata_key(metadata_json, 'MMR', 1)
]
player_apm = [
get_metadata_key(metadata_json, 'APM', 0),
get_metadata_key(metadata_json, 'APM', 1)
]
# ex: ["P". "Z"]
player_races = [
get_metadata_key(metadata_json, 'SelectedRace', 0),
get_metadata_key(metadata_json, 'SelectedRace', 1)
]
player_races = [race_dictionary[race] for race in player_races]
# string array with 2 player names, i.e [Feniks, DarthNoob]
player_names = [
erase_punctuation(player_list[0]['m_name']),
erase_punctuation(player_list[1]['m_name'])
]
# player result is 1 if won, 2 if not.
player_result = [
player_list[0]['m_result'] == 1,
player_list[1]['m_result'] == 1
]
# record whether this player won
for i in [0, 1]:
player_name = player_names[i]
game_object = GameObject(opponent=player_names[1 - i],
race=player_races[i],
win=player_result[i],
mmr=player_mmr[i],
apm=player_apm[i],
duration=metadata_json['Duration'])
if player_name in mmr_exceptions:
game_object.mmr = max(game_object.mmr,
mmr_exceptions[player_name])
if player_name.lower() in nicknames_dict:
player_name = nicknames_dict[player_name.lower()]
if player_name in player_dictionary:
player_dictionary[player_name].games.append(game_object)
player_dictionary[player_name].wins += player_result[i]
else:
player_dictionary[player_name] = PlayerObject(
player_name, player_result[i], [game_object])
except:
print("Error processing replay: %s" % replay)
traceback.print_exc()
return player_dictionary
def selenium_process_replay():
print("Processing One Map From Ladder Rotation")
options = Options()
options.add_argument("--headless")
options.add_argument('--disable-gpu')
options.add_argument('--no-sandbox')
options.binary_location = os.environ.get('GOOGLE_CHROME_BIN')
options.add_experimental_option(
"prefs", {
"download.default_directory": getcwd() + '\\TempReplays',
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"safebrowsing_for_trusted_sources_enabled": False,
"safebrowsing.enabled": False
})
curAllMaps = open(getcwd() +
'/PlayerMatch/ValidMapRotationBackup.txt').readlines()
remaining = len(curAllMaps)
currentLine = OverallSentiment.objects.filter(
terranSentimentCount=0).count()
if remaining <= currentLine - 1:
return
curMap = curAllMaps[currentLine + 1]
print(curMap)
# open(getcwd() + '/PlayerMatch/ValidMapRotationBackup.txt', 'w').writelines(curAllMaps[1:])
curMapLink = ""
# browser = webdriver.Chrome(getcwd() + '/PlayerMatch/chromedriver', options=options)
browser = webdriver.Chrome(executable_path=str(
os.environ.get('CHROMEDRIVER_PATH')),
options=options)
enable_download_in_headless_chrome(browser,
getcwd() + '/PlayerMatch/TempReplays')
curMapName = curMap
curMapName = curMapName.replace(" ", "%20")
curMapLink = "https://gggreplays.com/matches#?map_name=" + curMapName + "&page="
isLast = False
isLastPage = False
curPage = 1
browser.get(curMapLink + str(curPage))
sleep(1)
while not isLastPage:
fileNameList = []
TableBody = browser.find_element_by_xpath(
'//*[@id="matches"]/div[3]/div[3]/table/tbody')
AllRows = TableBody.find_elements_by_tag_name("tr")
TotalRowNum = len(AllRows)
for i in range(2, TotalRowNum + 1):
# Add this below if there is loading error on the table elements
try:
DateElement = WebDriverWait(browser, 5).until(
EC.visibility_of_element_located((
By.XPATH,
'//*[@id="matches"]/div[3]/div[3]/table/tbody/tr[{}]/td[20]'
.format(i))))
except TimeoutException as err:
continue
DateText = DateElement.text
# browser.find_element_by_xpath(
# '//*[@id="matches"]/div[3]/div[3]/table/tbody/tr[{}]/td[20]'.format(i)).text
PlayerText = browser.find_element_by_xpath(
'//*[@id="matches"]/div[3]/div[3]/table/tbody/tr[{}]/td[6]'.
format(i)).text
if "A.I." not in PlayerText and PlayerText:
if "year" in DateText:
print(PlayerText)
matchLink = browser.find_element_by_xpath(
'//*[@id="matches"]/div[3]/div[3]/table/tbody/tr[{}]/td[2]/a'
.format(i)).get_attribute('href')
print(matchLink)
# browser.find_element_by_xpath('//*[@id="matches"]/div[3]/div[3]/table/tbody/tr[{}]'.format(i)).click()
before = listdir(getcwd() + '/PlayerMatch/TempReplays')
downURL = matchLink + "/replay"
browser.get(downURL)
# sleep(2)
after = listdir(getcwd() + '/PlayerMatch/TempReplays')
change = set(after) - set(before)
file_name = ""
loopCount = 0
while loopCount < 16 and len(change) == 0:
sleep(0.25)
after = listdir(getcwd() + '/PlayerMatch/TempReplays')
change = set(after) - set(before)
if len(change) > 0:
file_name = change.pop()
if 'crdownload' not in file_name:
break
loopCount += 1
if loopCount == 16:
browser.get(curMapLink + str(curPage))
continue
# and 'crdownload'
if file_name and 'crdownload' not in file_name:
fileNameList.append(file_name)
# browser.get(curMapLink+str(j))
print(fileNameList)
for fileName in fileNameList:
archive = mpyq.MPQArchive(getcwd() + '/PlayerMatch/TempReplays/' +
fileName)
print(archive.files)
contents = archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(contents)
baseBuild = header['m_version']['m_baseBuild']
try:
protocol = versions.build(baseBuild)
analyze_sentiments(archive, protocol)
except ImportError as err:
print(err.args)
if isLast:
isLastPage = True
curPage += 1
browser.get(curMapLink + str(curPage))
sleep(1)
nextButton = browser.find_element_by_xpath(
'//*[@id="matches"]/div[3]/div[3]/unpaginate/div/ul/li[3]'
).get_attribute("style")
if 'none' in nextButton:
isLast = True
OverallSentiment.objects.create(terranSentimentCount=0,
terranSentimentOverall=0,
zergSentimentCount=0,
zergSentimentOverall=0,
protossSentimentCoun=0,
protossSentimentOverall=0)
# curRace = curRace[2: len(curRace) - 1]
archive = None
browser.close()
new_name = str(uuid4())
rename(getcwd() + '/PlayerMatch/TempReplays/',
getcwd() + '/PlayerMatch/' + new_name)
rmtree(getcwd() + '/PlayerMatch/' + new_name)
mkdir(getcwd() + '/PlayerMatch/TempReplays')
def __init__(self, sc2name, replay_path=''):
self.local_path = replay_path
self.player = None
self.opponent = None
self.UTC_timestamp = 0
self.is_comp = False
self.map = ''
self.map_code = ''
__archive = None
__baseBuild = None
__header = None
__protocol = None
__details = None
if replay_path != '':
try:
#generate MPQ archive
__archive = mpyq.MPQArchive(replay_path)
#get the replays protocol version
contents = __archive.header['user_data_header']['content']
__header = versions.latest().decode_replay_header(contents)
except Exception as ex:
raise Exception('Issue in archive unpack: {0}'.format(str(ex)))
# The header's baseBuild determines which protocol to use
#part of this code was modified from
#s2_cli.py @ https://github.com/Blizzard/s2protocol/tree/master/s2protocol
__baseBuild = __header['m_version']['m_baseBuild']
try:
__protocol = versions.build(__baseBuild)
except Exception as ex:
raise Exception('Unsupported base build: {0} ({1})'.format(
__baseBuild, str(ex)))
#replay details
try:
contents = __archive.read_file('replay.details')
__details = __protocol.decode_replay_details(contents)
self.map = __details['m_title'].decode('utf-8')
self.map_code = self.map.replace(' ', '-').lower()
self.UTC_timestamp = __details['m_timeUTC']
except Exception as ex:
raise Exception(
'Issue in extracting replay details: {0} ({1})'.format(
__baseBuild, str(ex)))
#replay initdata
try:
contents = __archive.read_file('replay.initData')
initdata = __protocol.decode_replay_initdata(contents)
game_desc = initdata['m_syncLobbyState']['m_gameDescription']
game_options = game_desc['m_gameOptions']
self.is_comp = game_options[
'm_competitive'] and not game_options[
'm_cooperative'] and not game_desc[
'm_isCoopMode'] and game_desc['m_maxUsers'] == 2
except Exception as ex:
raise Exception(
'Issue in extracting replay init data: {0} ({1})'.format(
__baseBuild, str(ex)))
try:
#pre process for matchup and names
num_players = len(__details['m_playerList'])
for i in range(num_players):
name = __details['m_playerList'][i]['m_name'].decode(
'utf-8')
race = __details['m_playerList'][i]['m_race'].decode(
'utf-8')
clan = ''
team = __details['m_playerList'][i]['m_teamId']
result = __details['m_playerList'][i]['m_result']
if name.find('<') > -1:
info = self.__beautify_name(name)
clan = info[0]
name = info[1]
if sc2name == name:
self.player = create_player(name, race, result == 1,
clan, team)
else:
self.opponent = create_player(name, race, result == 1,
clan, team)
except Exception as ex:
raise Exception('Issue in replay processing: {0} ({1})'.format(
__baseBuild, str(ex)))
def rename_file(filename, settings):
"""
Rename the replay with the format following format
XvY - Win/Loss - Duration of replay in minutes - Name of opponent / their MMR - Map name
"""
if (path.splitext(filename)[1] == '.SC2Replay'):
archive = mpyq.MPQArchive(filename)
contents = str(archive.header['user_data_header']['content'])
#figure out build version of replay
header = versions.latest().decode_replay_header(contents)
baseBuild = header['m_version']['m_baseBuild']
protocol = versions.build(baseBuild)
contents = archive.read_file('replay.details')
details = protocol.decode_replay_details(contents)
metadata = json.loads(archive.read_file('replay.gamemetadata.json'))
contents = archive.read_file('replay.initData')
initData = protocol.decode_replay_initdata(contents)
#data only matters if it's a 1v1
if (initData['m_syncLobbyState']['m_gameDescription']['m_gameOptions']
['m_competitive'] and initData['m_syncLobbyState']
['m_gameDescription']['m_maxPlayers'] == 2):
game_duration = metadata["Duration"] / 84
player_race = ""
enemy_name = ""
enemy_race = ""
enemy_mmr = ""
result = ""
for player in details['m_playerList']:
if (player['m_toon']['m_id'] == settings["[player_id]"]):
player_race = player['m_race']
for meta_player in metadata['Players']:
if meta_player['PlayerID'] == player[
'm_workingSetSlotId'] + 1:
result = meta_player["Result"]
else:
#player might not have MMR if in placements
try:
enemy_mmr = meta_player["MMR"]
except:
enemy_mmr = ""
else:
enemy_race = player['m_race']
enemy_name = player['m_name']
matchup = get_matchup_from_races(player_race, enemy_race)
#force cleanup on archive to allow file to be renamed
del archive
formatted_enemy_name = unicode(enemy_name, "utf-8")
formatted_enemy_name = formatted_enemy_name.replace("<", "(")
formatted_enemy_name = formatted_enemy_name.replace(">", ")")
formatted_enemy_name = formatted_enemy_name.replace("<sp/>", "")
name = matchup + " - " + result + " - " + str(
game_duration) + " min - " + "[" + formatted_enemy_name + (
"" if enemy_mmr == "" else
" ") + str(enemy_mmr) + "] - " + metadata["Title"]
attempt = 0
passed = False
while (not passed):
try:
if attempt == 0:
rename(
filename,
path.join(path.dirname(filename),
name + ".SC2Replay"))
else:
rename(
filename,
path.join(
path.dirname(filename),
name + " (" + str(attempt) + ").SC2Replay"))
passed = True
except:
attempt += 1
if loopCount == 16:
browser.get(curMapLink + str(j))
continue
fileNameList.append(file_name)
# browser.get(curMapLink+str(j))
print(fileNameList)
for fileName in fileNameList:
archive = mpyq.MPQArchive(getcwd() + '\\TempReplays\\' + fileName)
print(archive.files)
contents = archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(contents)
baseBuild = header['m_version']['m_baseBuild']
try:
protocol = versions.build(baseBuild)
analyze_sentiments(archive, protocol, races)
except ImportError as err:
print(err.args)
# contents = archive.read_file('replay.initData')
# lobbyDetails = protocol.decode_replay_initdata(contents)
# curRace = curRace[2: len(curRace) - 1]
archive = None
browser.close()
def test_latest(self):
p = _versions.latest()
self.assertIsNotNone(p)
def main():
parser = argparse.ArgumentParser(
prog='s2repdump',
description='''
: Dump player handles:
--players [replay_file]
: Reconstruct players .SC2Bank files
--bank [player_slot] [replay_file]
''',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('replay_file', help='.SC2Replay file to load')
parser.add_argument('--players',
help='print info about players',
action='store_true')
parser.add_argument('--chat', help='chat messages', action='store_true')
parser.add_argument('--json', help='json', action='store_true')
parser.add_argument('--bank',
help='reconstruct player\'s SC2Bank files',
type=int)
parser.add_argument('--out',
help='output directory',
type=str,
default='./out')
parser.add_argument('-v',
'--verbose',
help='verbose logging',
action='store_true')
args = parser.parse_args()
setupLogger()
if args.verbose:
logging.getLogger().setLevel(logging.DEBUG)
else:
logging.getLogger().setLevel(logging.INFO)
archive = mpyq.MPQArchive(args.replay_file)
# HEADER
contents = archive.header['user_data_header']['content']
header = versions.latest().decode_replay_header(contents)
# The header's baseBuild determines which protocol to use
baseBuild = header['m_version']['m_baseBuild']
try:
protocol = versions.build(baseBuild)
except Exception as e:
logging.warn('Unsupported base build: %s (%s)' % (baseBuild, str(e)))
protocol = versions.latest()
logging.warn('Attempting to use newest possible instead: %s' %
protocol.__name__)
details = protocol.decode_replay_details(
readArchiveContents(archive, 'replay.details'))
initd = protocol.decode_replay_initdata(
readArchiveContents(archive, 'replay.initData'))
playerList = read_players(initd, details)
userList = {}
for slotId in playerList:
if 'user_id' in playerList[slotId]:
userList[playerList[slotId]['user_id']] = playerList[slotId]
if args.players:
print(json.dumps(playerList, indent=True))
if args.chat:
messageevents = protocol.decode_replay_message_events(
readArchiveContents(archive, 'replay.message.events'))
clog = []
for ev in messageevents:
if ev['_event'] == 'NNet.Game.SChatMessage':
clog.append({
'gameloop': ev['_gameloop'],
'user_id': ev['_userid']['m_userId'],
'recipient': ev['m_recipient'],
'message': ev['m_string'],
})
if args.json:
print(json.dumps(clog, indent=True))
else:
for x in clog:
secs = x['gameloop'] / 16
print('[%d:%02d:%02d] %s: %s' %
(secs / 3600, secs % 3600 / 60, secs % 60,
userList[x['user_id']]['name'], x['message']))
if args.bank is not None:
logging.info('Processing player "%s"' % userList[args.bank]['name'])
gameevents = protocol.decode_replay_game_events(
readArchiveContents(archive, 'replay.game.events'))
banks = reconstruct_banks(gameevents, args.bank)
for b in banks:
logging.info('Reconstructed "%s.SC2Bank"' % b.name)
b.write(args.out)
