def load_supervised_data(replay_files, discretize):
valid_files = 0
start_time = time.time()
for i, f in enumerate(replay_files):
#print('loading', f)
try:
game = Game(f)
except Exception as e:
print(f, e)
continue
try:
check_valid(game)
yield from get_supervised_data(game, discretize)
valid_files += 1
except InvalidGameError as e:
print(e)
if i % 10 == 0:
elapsed_time = time.time() - start_time
time_per_file = elapsed_time / (i + 1)
remaining_time = time_per_file * (len(replay_files) - i - 1)
print(
'%d %d %.f %.2f %.f' %
(i, valid_files, elapsed_time, time_per_file, remaining_time))
#if elapsed_time > 30: break
print(valid_files, len(replay_files))
def setGame(self, slp):
try:
self.__GAME = Game(slp)
except:
return False
self.__PLAYERS.clear()
return True
def proc_file(self, src_fname,rename_only=False):
try:
self.slp_object = Game(os.path.join(self.src_dir, src_fname))
dest_fname = self.format_filename()
if rename_only == True:
os.rename(os.path.join(self.src_dir, src_fname), os.path.join(self.src_dir, dest_fname + '.slp'))
else:
dataframe_dict = initialize_dataframe_dict(SLPParser.pre_frame_attributes, SLPParser.post_frame_attributes, SLPParser.split_coord_attributes)
for frame in self.slp_object.frames:
proc_frame(dataframe_dict, frame, SLPParser.pre_frame_attributes, SLPParser.post_frame_attributes, SLPParser.split_coord_attributes)
df = pd.DataFrame.from_dict(dataframe_dict)
assert(df.shape[1] == 33)
df = data_pre_proc_mvp(df)
df.to_csv(os.path.join(self.dest_dir, dest_fname + ".csv"), index=False)
except:
print(f'Failed to parse {src_fname}')
traceback.print_exc()
def get_info(given_slippi_file):
# tries to rename a specific hard-coded slippi file based off of info from within the file
# filename =
# slippifile = Game('C:\\Users\\talen\\Desktop\\testfolder\\realgame.slp')
slippifile = Game(given_slippi_file)
slippifile.frames[0]
player1_name = slippifile.metadata.players[0].netplay.name
player2_name = slippifile.metadata.players[1].netplay.name
game_datetime = slippifile.metadata.date
stagestr = slippifile.start.stage
stage = str(stagestr).split('.')[1]
#print(game_datetime.date())
#print(game_datetime.time())
formatted_datetime = (str(game_datetime.date()) + ' ' +
str(game_datetime.time())).replace(':', '')
#print ('player1_name:', player1_name)
#print ('player2_name:', player2_name)
#print(slippifile)
#print('---')
#gets character names from both players
character_names = []
for num in range(2):
for key in slippifile.metadata.players[num].characters:
char_name = str(key).split('.')[1]
character_names.append(char_name)
# print(character_names)
player1_char = character_names[0]
player2_char = character_names[1]
#print('stage:', stage)
formatted_file_name = '{} ({}) vs {} ({}) - {} - {}'.format(
player1_name, player1_char, player2_name, player2_char, stage,
str(formatted_datetime))
return formatted_file_name
def test_game(self):
game = self._game('game')
self.assertEqual(
game.metadata,
Game.Metadata._parse({
'startAt': '2018-06-22T07:52:59Z',
'lastFrame': 5085,
'playedOn': 'dolphin',
'players': {
'0': {
'characters': {
InGameCharacter.MARTH: 5209
}
},
'1': {
'characters': {
InGameCharacter.FOX: 5209
}
}
}
}))
self.assertEqual(
game.metadata,
Game.Metadata(date=datetime.datetime(2018, 6, 22, 7, 52, 59, 0,
datetime.timezone.utc),
duration=5209,
platform=Game.Metadata.Platform.DOLPHIN,
players=(Game.Metadata.Player({
InGameCharacter.MARTH:
5209
}), Game.Metadata.Player({InGameCharacter.FOX:
5209}), None, None)))
self.assertEqual(
game.start,
Start(is_teams=False,
random_seed=3803194226,
slippi=Start.Slippi(Start.Slippi.Version(1, 0, 0, 0)),
stage=Stage.YOSHIS_STORY,
players=(Start.Player(character=CSSCharacter.MARTH,
type=Start.Player.Type.HUMAN,
stocks=4,
costume=3,
team=None,
ucf=Start.Player.UCF(False, False)),
Start.Player(character=CSSCharacter.FOX,
type=Start.Player.Type.CPU,
stocks=4,
costume=0,
team=None,
ucf=Start.Player.UCF(False, False)),
None, None),
is_pal=False))
self.assertEqual(game.end, End(End.Method.CONCLUSIVE))
self.assertEqual(game.metadata.duration, len(game.frames))
def read_game(path: str) -> Game:
if not path.endswith('.slp'):
return None
try:
log.debug(f"Reading in game: {path}")
game = Game(path)
return game if is_valid_stage(game.start.stage) else None
except ValueError:
pass
def extract(f, as_sparse=True):
'''
Extracts the istream payloads from a .slp file
Parameters
-----------
f (string) : Full path to game replay file
as_sparse (bool) : If true, return istream as a scipy csr matrix
otherwise return as numpy array
Returns
-----------
payload (tuple of dicts) : returns the player data from the replay file
for all players detected.
Each of these dicts is essentially the json
object that will be sent to mongodb in export.py
'''
# get game_id and game data using f (filename)
game_id = get_id(f)
game = Game(f)
# reject games less than a minute long
if len(game.frames)/3600 < 1:
raise GameTooShortError('Game is too short')
# get outpt payload for each active controller port
try:
payload = [
{
'game_id': game_id,
'istream': istream,
'character': character,
'name': name,
'code': code,
}
for istream, character, name, code
in zip(get_istreams(game, as_sparse=as_sparse),
get_player_characters(game),
get_player_names(game),
get_player_codes(game))
if character is not None
]
# raise parsing errors
except ParseError:
raise
# if unexpected error, raise InvalidGameError
except:
raise InvalidGameError
else:
return tuple(payload)
def read_games(folder: str):
files = [
os.path.join(folder, x) for x in os.listdir(folder)
if x.endswith('.slp')
]
for game in files:
try:
yield len(files), Game(game)
except Exception:
yield len(files), None
def validate_game(fname):
# print(folderUrl + "/" + fname)
try:
game = Game(folderUrl + "/" + fname)
# game = Game1Frame('games/' + fname)
return game
except KeyboardInterrupt:
sys.exit()
except:
print('Game ' + fname + ' contains corrupt data.')
return None
def get_info(given_slippi_file):
# tries to rename a specific hard-coded slippi file based off of info from within the file
# filename =
# slippifile = Game('C:\\Users\\talen\\Desktop\\testfolder\\realgame.slp')
slippifile = Game(given_slippi_file)
slippifile.frames[0]
player1_name = slippifile.metadata.players[0].netplay.name
player2_name = slippifile.metadata.players[1].netplay.name
# uses regular expressions to remove forbidden characters that may appear in your filename accidentally
# looking at you "ttv/tweekssb"...
player1_name = re.sub(r'[\\/*?:"<>|]', "", player1_name)
player2_name = re.sub(r'[\\/*?:"<>|]', "", player2_name)
game_datetime = slippifile.metadata.date
stagestr = slippifile.start.stage
stage = str(stagestr).split('.')[1]
#print(game_datetime.date())
#print(game_datetime.time())
formatted_datetime = (str(game_datetime.date()) + ' ' +
str(game_datetime.time())).replace(':', '')
#print ('player1_name:', player1_name)
#print ('player2_name:', player2_name)
#print(slippifile)
#print('---')
#gets character names from both players
character_names = []
#for num in range(2):
num = 0
#print ('slippifile.metadata.players:', slippifile.metadata.players)
#print ('slippifile.metadata.players[0]', slippifile.metadata.players[0].characters)
'''probably update this section to be more efficient later lol'''
for key in slippifile.metadata.players[0].characters:
#print ('key:', key)
#print(slippifile.metadata.players.characters)
char_name = str(key).split('.')[1]
character_names.append(char_name)
for key in slippifile.metadata.players[1].characters:
#print ('key:', key)
#print(slippifile.metadata.players.characters)
char_name = str(key).split('.')[1]
character_names.append(char_name)
# print(character_names)
player1_char = character_names[0]
player2_char = character_names[1]
#print('stage:', stage)
formatted_file_name = '{} ({}) vs {} ({}) - {} - {}'.format(
player1_name, player1_char, player2_name, player2_char, stage,
str(formatted_datetime))
return formatted_file_name
def is_valid(f):
try:
print(f"Scanning {f}")
game = Game(os.path.join('./data', f))
all_characters = [
p.character for p in game.start.players if p is not None
]
if game.start.stage in STAGES and \
len(set(all_characters) & set(CHARACTERS)) > 0:
return True
return False
except Exception:
return False
def flagged_set_cleanup(sets):
'''Given a list of sets, look at the flagged sets with deeper (and slower) parsing than
naive_set_cleanup. Returns a modified list of sets.'''
#iterate through the flagged sets and work additional logic to cull games
for set_ in sets:
if set_['flagged'] == True:
if debug: print(f"Looking at set {set_['set_id']}")
for index in reversed(set_['game']):
if debug: print(f"Looking at game {index['game_id']}")
if warmup_detection(Game(index['file_path'])):
set_['game'].remove(index)
if debug:
print(
f"removed game id {index['game_id']} from set {set_['set_id']}"
)
return (sets)
def __main__():
parser = ArgumentParser(description=("Generate training images from a "
"Project Slippi replay."))
parser.add_argument("--ports", action="store_true", default=True,
help=("grab the used port (stock/percent) displays "
"(default: True)"))
parser.add_argument("--filetype", default="TIF",
help="the filetype of output images (default: TIF)")
parser.add_argument("replay", metavar="match.slp", type=str,
help="a Project Slippi game data file")
parser.add_argument("video", metavar="match.avi", type=str,
help="a 643x528 Dolphin framedump of `match.slp`")
parser.add_argument("outdir", type=str,
help="the directory to save files to")
args = parser.parse_args()
game = Game(args.replay)
stream = MeleeFrameSync(args.video)
def __init__(self, file, distill_dir):
self.game = Game(file)
self.distill_dir = distill_dir
self._num_players = 4
# game data
self.duration = self.game.metadata.duration
self.players = self.game.start.players
self.tags = self._get_tags()
self.active_ports = []
# frame data
self.data = self._parse_frames()
self.rebirths = self._find_rebirths()
self.death_percent = self._find_death_percent()
self.winner = self._determine_winner_port()
# distill to json
self._package_json()
self._write_json()
def replay_parser(directory, tournament=None, stream_filter=None):
'''For a given directory, search recursively for all .slp files, parse them with the below functions,
and return a list of dictionaries with each dictionary as specified:
{game_id, tournament, stream_filter, absolute directory path,
game_start, game_end, ports, characters, stage, winner}'''
replay_list = []
game_id = 0
#walk the directory
for subdir, dirs, files in os.walk(directory):
for file in files:
#only look at .slp files, then build our dictionary
if file.endswith('.slp'):
file_path = os.path.join(subdir, file)
game = Game(file_path)
characters = characters_played(game)
print(characters)
#if the replay is corrupted such that the character list is empty, skip that file
if characters != []:
gameLength = game_length(game)
game_info = basic_game_info(game)
replay_list.append({
'game_id': game_id,
'tournament': tournament,
'stream_filter': stream_filter,
'file_path': file_path,
'game_start': gameLength[0],
'game_end': gameLength[1],
'port_1': characters[0][1],
'port_2': characters[1][1],
'char_1': characters[0][0],
'char_2': characters[1][0],
'stage': game_info['Stage'],
'winner': game_info['Winner']
})
game_id += 1
return (replay_list)
def mark(root, f):
game = Game(root + "/" + f)
for players in game.start.players:
if (players and players.type == event.Start.Player.Type.CPU):
return
frames = game.frames
found = [False, False, False, False]
endSearch = [0, 0, 0, 0]
markedFrames = []
finishedFrames = []
for i in range(len(frames)):
frame = frames[i]
ports = frame.ports
for j in range(len(ports)):
if (endSearch[j] < i):
found[j] = False
if (not ports[j]):
continue
if (ports[j].leader.pre.buttons.physical ==
event.Buttons.Physical.DPAD_DOWN):
if (found[j]):
continue
if (i < endSearch[j]):
markedFrames.append(i)
found[j] = True
continue
endSearch[j] = i + 120
for frame in markedFrames:
minute = 7 - (int(frame / 60 / 60))
second = 62 - (int(frame / 60 % 60))
frame = str(minute) + "-" + str(second)
finishedFrames.append(frame)
name = ""
if (len(finishedFrames) > 0):
for frame in finishedFrames:
name = name + str(frame) + ", "
os.rename(r'' + root + "/" + f,
r'' + root + "/" + name[:len(name) - 2] + '.slp')
def record_file_slp(slp_file, outfile):
conf = Config()
# Parse file with py-slippi to determine number of frames
slippi_game = Game(slp_file)
num_frames = slippi_game.metadata.duration + DURATION_BUFFER
if is_game_too_short(slippi_game.metadata.duration, conf.remove_short):
print(
"Warning: Game is less than 30 seconds and won't be recorded. Override in config."
)
return
# Dump frames
with DolphinRunner(conf, THIS_USER_DIR, SCRIPT_DIR,
uuid.uuid4()) as dolphin_runner:
video_file, audio_file = dolphin_runner.run(slp_file, num_frames)
# Encode
ffmpeg_runner = FfmpegRunner(conf.ffmpeg)
ffmpeg_runner.run(video_file, audio_file, outfile)
print('Created {}'.format(outfile))
def chart():
for filename in os.listdir(directory):
if filename.endswith(".slp"):
dict['slippiCount'] += 1
if dict['slippiCount'] % 5 == 0:
print(dict)
currGame = Game(directory + filename)
'Game object is created here'
for i in currGame.metadata.players:
if i is not None:
if i.netplay.name == 'FOXMASTER2000' or i.netplay.name == 'DS4_Johns':
try:
if 22 in i.characters.keys():
dict['falcoGames'] += 1
if 1 in i.characters.keys():
dict['foxGames'] += 1
if 18 in i.characters.keys():
dict['marthGames'] += 1
except:
print("exception!")
continue
else:
continue
print(dict)
import os
from slippi import Game
total_frames = 0
for dirpath, _, filenames in os.walk('replays/Gang-Steals/stream'):
for fname in filenames:
game = Game(os.path.join(dirpath, fname))
total_frames += len(game.frames)
print("total frames", total_frames)
displaycount = len(main[tournament])
else:
displaycount = 0
#if tournament doesn't exist in map/main make new dictionary for it
if not tournament in map:
map[tournament] = {}
if not tournament in main:
main[tournament] = {}
#iterate through replays, create json objects and place in main and map
for replay in replays:
print(replay)
#if replay doesn't already exist in tournament map
if not replay in map[tournament] and not '.json' in replay:
game = Game(foldername+replay)
data = {}
#generate list of players in game
players = []
portnumber = 1
for player in game.start.players:
if player is not None:
port = {}
port['number'] = portnumber
port['character'] = player.character
port['costume'] = player.costume
port['tag'] = player.tag
port['searchtag'] = player.tag
if len(port['searchtag']) is not 0:
for key, replacement in tagtranslator.items():
port['searchtag'] = port['searchtag'].replace(key, replacement)
import json
from slippi import Game
from tqdm import tqdm
game = Game('Game_20210225T173905.slp')
positions = []
minX = float('inf')
maxX = -float('inf')
minY = float('inf')
maxY = -float('inf')
for frame in tqdm(game.frames):
port_positions = {}
for i, port in enumerate(frame.ports):
if port:
data = port.leader.post
pos = data.position
x, y = pos.x, pos.y
if x < minX:
minX = x
if x > maxX:
maxX = x
if y < minY:
minY = y
if y > maxY:
maxY = y
replays = [
f for f in listdir("../../replays") if isfile(join("../../replays", f))
]
#get existing data (so we don't redo any existing replays)
with open('../../map.json', 'r') as file:
map = json.load(file)
#iterate for counting new replays
newReplays = 0
#generate dictionary for each replay
for replaypath in replays:
#if replay doesn't already exist
if not replaypath in map:
game = Game("../../replays//" + replaypath)
data = {}
#generate list of players in game
players = []
for player in game.start.players:
if player is not None:
port = {}
port['character'] = player.character
port['costume'] = player.costume
port['tag'] = player.tag
port['searchtag'] = player.tag
if len(port['searchtag']) is not 0:
for key, replacement in tagtranslator.items():
port['searchtag'] = port['searchtag'].replace(
key, replacement)
players.append(port)
i for i, player in enumerate(game.start.players) if player is not None
]
game_buffer = nest.map_structure(lambda _: [], game_paths)
for frame in game.frames[:2]:
for i, p in enumerate(player_ports):
data = frame.ports[p].leader
append_data(data, game_buffer[i])
_to_feature = lambda t, l: TYPE_FEATURE_MAP[t](l)
game_features = nest.map_structure_up_to(game_paths, _to_feature,
game_paths, game_buffer)
flat_features = {}
nest.map_structure_with_paths(write_flat, game_features, d=flat_features)
return tf.train.Example(features=tf.train.Features(feature=flat_features))
replay_files = []
for dirpath, _, filenames in os.walk('replays/Gang-Steals/stream'):
for fname in filenames:
replay_files.append(os.path.join(dirpath, fname))
test_game = Game(replay_files[100])
print(len(test_game.frames))
test_example = to_tf_example(test_game)
def _game(self, name):
return Game(path(name))
def get_frames(replay):
game = Game(replay)
frames = game.metadata.duration
return frames
def _game(self, name):
return Game(
os.path.join(os.path.dirname(__file__), 'replays', name + '.slp'))
