def merge(*args, quiet=True): import retro known_hashes = {} imported_games = 0 for game in retro.list_games(): shafile = os.path.join(retro.get_game_path(game), 'rom.sha') with open(shafile) as f: shas = f.read().strip().split('\n') for ext, platform in retro.EMU_EXTENSIONS.items(): if game.endswith('-' + platform): break for sha in shas: known_hashes[sha] = (game, ext) for rom in args: try: data, hash = groom_rom(rom) except IOError: continue if hash in known_hashes: game, ext = known_hashes[hash] if not quiet: print('Importing', game) with open(os.path.join(retro.get_game_path(game), 'rom%s' % ext), 'wb') as f: f.write(data) imported_games += 1 if not quiet: print('Imported %i games' % imported_games)
def state(game): errors = [] states = retro.list_states(game) if not states: return [], [] rom = retro.get_romfile_path(game) path = retro.get_game_path(game) emu = retro.RetroEmulator(rom) for statefile in states: try: with gzip.open(os.path.join(path, statefile + '.state'), 'rb') as fh: state = fh.read() except (IOError, zlib.error): errors.append((game, 'state failed to decode: %s' % statefile)) continue emu.set_state(state) emu.step() del emu gc.collect() return [], errors
def __init__(self, game, states, scenario=None, info=None, use_restricted_actions=retro.ACTIONS_FILTERED, record=False): super().__init__(game, retro.STATE_NONE, scenario, info, use_restricted_actions, record) # path loading self.game_path = retro.get_game_path(game) self.metadata_path = os.path.join(self.game_path, 'metadata.json') # list of states --> levels self.initial_states = {} for state_id in states: self.initial_states[state_id] = self.read_state(state_id)
def verify_json(): warnings = [] errors = [] for game in retro.list_games(): gamedir = retro.get_game_path(game) w, e = verify_data(game) warnings.extend(w) errors.extend(e) w, e = verify_scenario(game) warnings.extend(w) errors.extend(e) return warnings, errors
def verify_hash(game): errors = [] gamedir = retro.get_game_path(game) rom = retro.get_romfile_path(game) system = retro.get_romfile_system(rom) with open(os.path.join(gamedir, 'rom.sha')) as f: expected_shas = f.read().strip().split('\n') with open(rom, 'rb') as f: if system == 'Nes': # Chop off header for checksum f.read(16) real_sha = hashlib.sha1(f.read()).hexdigest() if real_sha not in expected_shas: errors.append((game, 'sha mismatch')) return [], errors
def scan_missing(): missing = [] for game in retro.list_games(): gamedir = retro.get_game_path(game) if not os.path.isfile(os.path.join(gamedir, 'data.json')): missing.append((game, 'data.json')) if not os.path.isfile(os.path.join(gamedir, 'scenario.json')): missing.append((game, 'scenario.json')) if not os.path.isfile(os.path.join(gamedir, 'metadata.json')): missing.append((game, 'metadata.json')) if not retro.list_states(game): missing.append((game, '*.state')) if not os.path.isfile(os.path.join(gamedir, 'rom.sha')): missing.append((game, 'rom.sha')) return missing
def verify_data(game, raw=None): file = os.path.join(game, 'data.json') try: if not raw: with open(os.path.join(retro.get_game_path(file))) as f: data = json.load(f) else: data = json.loads(raw) except json.JSONDecodeError: return [], [(file, 'fail decode')] except IOError: return [], [] data = data.get('info') warnings = [] errors = [] if not data: return [], [(file, 'missing info')] for variable, definition in data.items(): if 'address' not in definition: errors.append((file, 'missing address for %s' % variable)) if 'type' not in definition: errors.append((file, 'missing type for %s' % variable)) else: if not re.match(r'\|[dinu]1|(>[<=]?|<[>=]?|=[><]?)[dinu][2-8]', definition['type']): errors.append((file, 'invalid type %s for %s' % (definition['type'], variable))) elif re.match( r'([><=]{2}|=[><]|<[>=]|>[<=])[dinu][2-8]|[><=]{1,2}d[5-8]', definition['type']): warnings.append((file, 'suspicious type %s for %s' % (definition['type'], variable))) if 'lives' in data and data['lives'].get('type', '') not in ('|u1', '|i1', '|d1'): warnings.append( (file, 'suspicious type %s for lives' % data['lives']['type'])) if 'score' in data and (data['score'].get( 'type', '??')[1:] in ('u1', 'd1', 'n1', 'n2') or 'i' in data['score'].get('type', '')): warnings.append( (file, 'suspicious type %s for score' % data['score']['type'])) warnings = [(file, w) for (file, w) in warnings if w not in whitelist.get(file, [])] return warnings, errors
def verify_hash_collisions(): errors = [] seen_hashes = {} for game in retro.list_games(): gamedir = retro.get_game_path(game) try: with open(os.path.join(gamedir, 'rom.sha')) as f: expected_shas = f.read().strip().split('\n') except IOError: continue for expected_sha in expected_shas: seen = seen_hashes.get(expected_sha, []) seen.append(game) seen_hashes[expected_sha] = seen for sha, games in seen_hashes.items(): if len(games) < 2: continue for game in games: errors.append((game, 'sha duplicate')) return [], errors
def verify_default_state(game, raw=None): file = os.path.join(game, 'metadata.json') try: if not raw: with open(retro.get_game_path(file)) as f: metadata = json.load(f) else: metadata = json.loads(raw) except json.JSONDecodeError: return [], [(file, 'fail decode')] except IOError: return [], [] errors = [] state = metadata.get('default_state') if not state: return [], [(file, 'default state missing')] if state not in retro.list_states(game): errors.append((file, 'invalid default state %s' % state)) return [], errors
def reset(self): # Reset to a random level (but don't change the game) try: game = self.env.unwrapped.gamename except AttributeError: logger.warning('no game name') pass else: game_path = retro.get_game_path(game) # pick a random state that's in the same game game_states = train_states[train_states.game == game] # if self.state: # game_states = game_states[game_states.state.str.contains(self.state)] # Load choice = game_states.sample().iloc[0] state = choice.state + '.state' logger.info('reseting env %s to %s %s', self.unwrapped.rank, game, state) with gzip.open(os.path.join(game_path, state), 'rb') as fh: self.env.unwrapped.initial_state = fh.read() return self.env.reset()
def __init__(self, game, state=retro.STATE_DEFAULT, scenario=None, info=None, use_restricted_actions=retro.ACTIONS_FILTERED, record=False): if not hasattr(self, 'spec'): self.spec = None self.img = None self.viewer = None self.gamename = game self.statename = state game_path = retro.get_game_path(game) rom_path = retro.get_romfile_path(game) metadata_path = os.path.join(game_path, 'metadata.json') if state == retro.STATE_NONE: self.initial_state = None elif state == retro.STATE_DEFAULT: self.initial_state = None try: with open(metadata_path) as f: metadata = json.load(f) if 'default_state' in metadata: with gzip.open( os.path.join(game_path, metadata['default_state']) + '.state', 'rb') as fh: self.initial_state = fh.read() except (IOError, json.JSONDecodeError): pass else: if not state.endswith('.state'): state += '.state' with gzip.open(os.path.join(game_path, state), 'rb') as fh: self.initial_state = fh.read() self.data = GameData() if info is None: info = 'data' if info.endswith('.json'): # assume it's a path info_path = info else: info_path = os.path.join(game_path, info + '.json') if scenario is None: scenario = 'scenario' if scenario.endswith('.json'): # assume it's a path scenario_path = scenario else: scenario_path = os.path.join(game_path, scenario + '.json') system = retro.get_romfile_system(rom_path) # We can't have more than one emulator per process. Before creating an # emulator, ensure that unused ones are garbage-collected gc.collect() self.em = retro.RetroEmulator(rom_path) self.em.configure_data(self.data) self.em.step() img = self.em.get_screen() core = retro.get_system_info(system) self.BUTTONS = core['buttons'] self.NUM_BUTTONS = len(self.BUTTONS) self.BUTTON_COMBOS = self.data.valid_actions() try: assert self.data.load( info_path, scenario_path), 'Failed to load info (%s) or scenario (%s)' % ( info_path, scenario_path) except Exception: del self.em raise if use_restricted_actions == retro.ACTIONS_DISCRETE: combos = 1 for combo in self.BUTTON_COMBOS: combos *= len(combo) self.action_space = gym.spaces.Discrete(combos) elif use_restricted_actions == retro.ACTIONS_MULTI_DISCRETE: self.action_space = gym.spaces.MultiDiscrete([ len(combos) if gym_version >= (0, 9, 6) else (0, len(combos) - 1) for combos in self.BUTTON_COMBOS ]) else: self.action_space = gym.spaces.MultiBinary(self.NUM_BUTTONS) kwargs = {} if gym_version >= (0, 9, 6): kwargs['dtype'] = np.uint8 self.observation_space = gym.spaces.Box(low=0, high=255, shape=img.shape, **kwargs) self.use_restricted_actions = use_restricted_actions self.movie = None self.movie_id = 0 self.movie_path = None if record is True: self.auto_record() elif record is not False: self.auto_record(record) self.seed() if gym_version < (0, 9, 6): self._seed = self.seed self._step = self.step self._reset = self.reset self._render = self.render self._close = self.close
def verify_scenario(game, scenario='scenario', raw=None, dataraw=None): file = os.path.join(game, '%s.json' % 'scenario') try: if not raw: with open(retro.get_game_path(file)) as f: scen = json.load(f) else: scen = json.loads(raw) except json.JSONDecodeError: return [], [(file, 'fail decode')] except IOError: return [], [] warnings = [] errors = [] if 'reward' not in scen or ('variables' not in scen['reward'] and 'script' not in scen['reward']): warnings.append((file, 'missing reward')) if 'done' not in scen or ('variables' not in scen['done'] and 'script' not in scen['done'] and 'nodes' not in scen['done']): warnings.append((file, 'missing done')) try: if not dataraw: datafile = os.path.join(retro.get_game_path(game), 'data.json') with open(datafile) as f: data = json.load(f) else: data = json.loads(dataraw) data = data.get('info') reward = scen.get('reward') done = scen.get('done') if reward and 'variables' in reward: for variable, definition in reward['variables'].items(): if variable not in data: errors.append((file, 'invalid variable %s' % variable)) if not definition: errors.append((file, 'invalid definition %s' % variable)) continue if 'reward' not in definition and 'penalty' not in definition: errors.append((file, 'blank reward %s' % variable)) if done and 'variables' in done: if 'score' in done['variables']: warnings.append( (file, 'suspicious variable in done condition: score')) if 'health' in done['variables'] and 'lives' in done[ 'variables'] and 'condition' not in done: warnings.append( (file, 'suspicious done condition: health OR lives')) if done.get('condition', 'any') == 'all' and ( len(done['variables']) + len(done.get('nodes', {}))) < 2: errors.append( (file, 'incorrect done condition all with only 1 check')) if done.get('condition', 'any') == 'any' and ( len(done['variables']) + len(done.get('nodes', {}))) > 2: warnings.append( (file, 'suspicious done condition any with more than 2 checks')) for variable, definition in done['variables'].items(): if 'op' not in definition: errors.append( (file, 'invalid done condition %s' % variable)) elif definition.get('reference', 0) == 0: if 'op' in ('equal', 'negative-equal'): warnings.append( (file, 'incorrect op: zero for %s' % variable)) elif 'op' == 'not-equal': warnings.append( (file, 'incorrect op: nonzero for %s' % variable)) elif 'op' == 'less-than': warnings.append( (file, 'incorrect op: negative for %s' % variable)) elif 'op' == 'greater-than': warnings.append( (file, 'incorrect op: positive for %s' % variable)) if data: if variable not in data: errors.append((file, 'invalid variable %s' % variable)) else: if 'i' not in data[variable].get( 'type', '') and definition.get( 'op', '') == 'negative' and definition.get( 'measurement') != 'delta': errors.append( (file, 'op: negative on unsigned %s' % variable)) except (json.JSONDecodeError, IOError): pass warnings = [(file, w) for (file, w) in warnings if w not in whitelist.get(file, [])] return warnings, errors