def merge(*args, quiet=True):
import retro
known_hashes = {}
imported_games = 0
for game in retro.list_games():
shafile = os.path.join(retro.get_game_path(game), 'rom.sha')
with open(shafile) as f:
shas = f.read().strip().split('\n')
for ext, platform in retro.EMU_EXTENSIONS.items():
if game.endswith('-' + platform):
break
for sha in shas:
known_hashes[sha] = (game, ext)
for rom in args:
try:
data, hash = groom_rom(rom)
except IOError:
continue
if hash in known_hashes:
game, ext = known_hashes[hash]
if not quiet:
print('Importing', game)
with open(os.path.join(retro.get_game_path(game), 'rom%s' % ext),
'wb') as f:
f.write(data)
imported_games += 1
if not quiet:
print('Imported %i games' % imported_games)
def state(game):
errors = []
states = retro.list_states(game)
if not states:
return [], []
rom = retro.get_romfile_path(game)
path = retro.get_game_path(game)
emu = retro.RetroEmulator(rom)
for statefile in states:
try:
with gzip.open(os.path.join(path, statefile + '.state'),
'rb') as fh:
state = fh.read()
except (IOError, zlib.error):
errors.append((game, 'state failed to decode: %s' % statefile))
continue
emu.set_state(state)
emu.step()
del emu
gc.collect()
return [], errors
def __init__(self, game, states, scenario=None, info=None,
use_restricted_actions=retro.ACTIONS_FILTERED, record=False):
super().__init__(game, retro.STATE_NONE, scenario, info, use_restricted_actions, record)
# path loading
self.game_path = retro.get_game_path(game)
self.metadata_path = os.path.join(self.game_path, 'metadata.json')
# list of states --> levels
self.initial_states = {}
for state_id in states:
self.initial_states[state_id] = self.read_state(state_id)
def verify_json():
warnings = []
errors = []
for game in retro.list_games():
gamedir = retro.get_game_path(game)
w, e = verify_data(game)
warnings.extend(w)
errors.extend(e)
w, e = verify_scenario(game)
warnings.extend(w)
errors.extend(e)
return warnings, errors
def verify_hash(game):
errors = []
gamedir = retro.get_game_path(game)
rom = retro.get_romfile_path(game)
system = retro.get_romfile_system(rom)
with open(os.path.join(gamedir, 'rom.sha')) as f:
expected_shas = f.read().strip().split('\n')
with open(rom, 'rb') as f:
if system == 'Nes':
# Chop off header for checksum
f.read(16)
real_sha = hashlib.sha1(f.read()).hexdigest()
if real_sha not in expected_shas:
errors.append((game, 'sha mismatch'))
return [], errors
def scan_missing():
missing = []
for game in retro.list_games():
gamedir = retro.get_game_path(game)
if not os.path.isfile(os.path.join(gamedir, 'data.json')):
missing.append((game, 'data.json'))
if not os.path.isfile(os.path.join(gamedir, 'scenario.json')):
missing.append((game, 'scenario.json'))
if not os.path.isfile(os.path.join(gamedir, 'metadata.json')):
missing.append((game, 'metadata.json'))
if not retro.list_states(game):
missing.append((game, '*.state'))
if not os.path.isfile(os.path.join(gamedir, 'rom.sha')):
missing.append((game, 'rom.sha'))
return missing
def verify_data(game, raw=None):
file = os.path.join(game, 'data.json')
try:
if not raw:
with open(os.path.join(retro.get_game_path(file))) as f:
data = json.load(f)
else:
data = json.loads(raw)
except json.JSONDecodeError:
return [], [(file, 'fail decode')]
except IOError:
return [], []
data = data.get('info')
warnings = []
errors = []
if not data:
return [], [(file, 'missing info')]
for variable, definition in data.items():
if 'address' not in definition:
errors.append((file, 'missing address for %s' % variable))
if 'type' not in definition:
errors.append((file, 'missing type for %s' % variable))
else:
if not re.match(r'\|[dinu]1|(>[<=]?|<[>=]?|=[><]?)[dinu][2-8]',
definition['type']):
errors.append((file, 'invalid type %s for %s' %
(definition['type'], variable)))
elif re.match(
r'([><=]{2}|=[><]|<[>=]|>[<=])[dinu][2-8]|[><=]{1,2}d[5-8]',
definition['type']):
warnings.append((file, 'suspicious type %s for %s' %
(definition['type'], variable)))
if 'lives' in data and data['lives'].get('type',
'') not in ('|u1', '|i1', '|d1'):
warnings.append(
(file, 'suspicious type %s for lives' % data['lives']['type']))
if 'score' in data and (data['score'].get(
'type', '??')[1:] in ('u1', 'd1', 'n1', 'n2')
or 'i' in data['score'].get('type', '')):
warnings.append(
(file, 'suspicious type %s for score' % data['score']['type']))
warnings = [(file, w) for (file, w) in warnings
if w not in whitelist.get(file, [])]
return warnings, errors
def verify_hash_collisions():
errors = []
seen_hashes = {}
for game in retro.list_games():
gamedir = retro.get_game_path(game)
try:
with open(os.path.join(gamedir, 'rom.sha')) as f:
expected_shas = f.read().strip().split('\n')
except IOError:
continue
for expected_sha in expected_shas:
seen = seen_hashes.get(expected_sha, [])
seen.append(game)
seen_hashes[expected_sha] = seen
for sha, games in seen_hashes.items():
if len(games) < 2:
continue
for game in games:
errors.append((game, 'sha duplicate'))
return [], errors
def verify_default_state(game, raw=None):
file = os.path.join(game, 'metadata.json')
try:
if not raw:
with open(retro.get_game_path(file)) as f:
metadata = json.load(f)
else:
metadata = json.loads(raw)
except json.JSONDecodeError:
return [], [(file, 'fail decode')]
except IOError:
return [], []
errors = []
state = metadata.get('default_state')
if not state:
return [], [(file, 'default state missing')]
if state not in retro.list_states(game):
errors.append((file, 'invalid default state %s' % state))
return [], errors
def reset(self):
# Reset to a random level (but don't change the game)
try:
game = self.env.unwrapped.gamename
except AttributeError:
logger.warning('no game name')
pass
else:
game_path = retro.get_game_path(game)
# pick a random state that's in the same game
game_states = train_states[train_states.game == game]
# if self.state:
# game_states = game_states[game_states.state.str.contains(self.state)]
# Load
choice = game_states.sample().iloc[0]
state = choice.state + '.state'
logger.info('reseting env %s to %s %s', self.unwrapped.rank, game, state)
with gzip.open(os.path.join(game_path, state), 'rb') as fh:
self.env.unwrapped.initial_state = fh.read()
return self.env.reset()
def __init__(self,
game,
state=retro.STATE_DEFAULT,
scenario=None,
info=None,
use_restricted_actions=retro.ACTIONS_FILTERED,
record=False):
if not hasattr(self, 'spec'):
self.spec = None
self.img = None
self.viewer = None
self.gamename = game
self.statename = state
game_path = retro.get_game_path(game)
rom_path = retro.get_romfile_path(game)
metadata_path = os.path.join(game_path, 'metadata.json')
if state == retro.STATE_NONE:
self.initial_state = None
elif state == retro.STATE_DEFAULT:
self.initial_state = None
try:
with open(metadata_path) as f:
metadata = json.load(f)
if 'default_state' in metadata:
with gzip.open(
os.path.join(game_path, metadata['default_state'])
+ '.state', 'rb') as fh:
self.initial_state = fh.read()
except (IOError, json.JSONDecodeError):
pass
else:
if not state.endswith('.state'):
state += '.state'
with gzip.open(os.path.join(game_path, state), 'rb') as fh:
self.initial_state = fh.read()
self.data = GameData()
if info is None:
info = 'data'
if info.endswith('.json'):
# assume it's a path
info_path = info
else:
info_path = os.path.join(game_path, info + '.json')
if scenario is None:
scenario = 'scenario'
if scenario.endswith('.json'):
# assume it's a path
scenario_path = scenario
else:
scenario_path = os.path.join(game_path, scenario + '.json')
system = retro.get_romfile_system(rom_path)
# We can't have more than one emulator per process. Before creating an
# emulator, ensure that unused ones are garbage-collected
gc.collect()
self.em = retro.RetroEmulator(rom_path)
self.em.configure_data(self.data)
self.em.step()
img = self.em.get_screen()
core = retro.get_system_info(system)
self.BUTTONS = core['buttons']
self.NUM_BUTTONS = len(self.BUTTONS)
self.BUTTON_COMBOS = self.data.valid_actions()
try:
assert self.data.load(
info_path,
scenario_path), 'Failed to load info (%s) or scenario (%s)' % (
info_path, scenario_path)
except Exception:
del self.em
raise
if use_restricted_actions == retro.ACTIONS_DISCRETE:
combos = 1
for combo in self.BUTTON_COMBOS:
combos *= len(combo)
self.action_space = gym.spaces.Discrete(combos)
elif use_restricted_actions == retro.ACTIONS_MULTI_DISCRETE:
self.action_space = gym.spaces.MultiDiscrete([
len(combos) if gym_version >= (0, 9, 6) else
(0, len(combos) - 1) for combos in self.BUTTON_COMBOS
])
else:
self.action_space = gym.spaces.MultiBinary(self.NUM_BUTTONS)
kwargs = {}
if gym_version >= (0, 9, 6):
kwargs['dtype'] = np.uint8
self.observation_space = gym.spaces.Box(low=0,
high=255,
shape=img.shape,
**kwargs)
self.use_restricted_actions = use_restricted_actions
self.movie = None
self.movie_id = 0
self.movie_path = None
if record is True:
self.auto_record()
elif record is not False:
self.auto_record(record)
self.seed()
if gym_version < (0, 9, 6):
self._seed = self.seed
self._step = self.step
self._reset = self.reset
self._render = self.render
self._close = self.close
def verify_scenario(game, scenario='scenario', raw=None, dataraw=None):
file = os.path.join(game, '%s.json' % 'scenario')
try:
if not raw:
with open(retro.get_game_path(file)) as f:
scen = json.load(f)
else:
scen = json.loads(raw)
except json.JSONDecodeError:
return [], [(file, 'fail decode')]
except IOError:
return [], []
warnings = []
errors = []
if 'reward' not in scen or ('variables' not in scen['reward']
and 'script' not in scen['reward']):
warnings.append((file, 'missing reward'))
if 'done' not in scen or ('variables' not in scen['done']
and 'script' not in scen['done']
and 'nodes' not in scen['done']):
warnings.append((file, 'missing done'))
try:
if not dataraw:
datafile = os.path.join(retro.get_game_path(game), 'data.json')
with open(datafile) as f:
data = json.load(f)
else:
data = json.loads(dataraw)
data = data.get('info')
reward = scen.get('reward')
done = scen.get('done')
if reward and 'variables' in reward:
for variable, definition in reward['variables'].items():
if variable not in data:
errors.append((file, 'invalid variable %s' % variable))
if not definition:
errors.append((file, 'invalid definition %s' % variable))
continue
if 'reward' not in definition and 'penalty' not in definition:
errors.append((file, 'blank reward %s' % variable))
if done and 'variables' in done:
if 'score' in done['variables']:
warnings.append(
(file, 'suspicious variable in done condition: score'))
if 'health' in done['variables'] and 'lives' in done[
'variables'] and 'condition' not in done:
warnings.append(
(file, 'suspicious done condition: health OR lives'))
if done.get('condition', 'any') == 'all' and (
len(done['variables']) + len(done.get('nodes', {}))) < 2:
errors.append(
(file, 'incorrect done condition all with only 1 check'))
if done.get('condition', 'any') == 'any' and (
len(done['variables']) + len(done.get('nodes', {}))) > 2:
warnings.append(
(file,
'suspicious done condition any with more than 2 checks'))
for variable, definition in done['variables'].items():
if 'op' not in definition:
errors.append(
(file, 'invalid done condition %s' % variable))
elif definition.get('reference', 0) == 0:
if 'op' in ('equal', 'negative-equal'):
warnings.append(
(file, 'incorrect op: zero for %s' % variable))
elif 'op' == 'not-equal':
warnings.append(
(file, 'incorrect op: nonzero for %s' % variable))
elif 'op' == 'less-than':
warnings.append(
(file, 'incorrect op: negative for %s' % variable))
elif 'op' == 'greater-than':
warnings.append(
(file, 'incorrect op: positive for %s' % variable))
if data:
if variable not in data:
errors.append((file, 'invalid variable %s' % variable))
else:
if 'i' not in data[variable].get(
'type', '') and definition.get(
'op', '') == 'negative' and definition.get(
'measurement') != 'delta':
errors.append(
(file,
'op: negative on unsigned %s' % variable))
except (json.JSONDecodeError, IOError):
pass
warnings = [(file, w) for (file, w) in warnings
if w not in whitelist.get(file, [])]
return warnings, errors
