def save_playthrough_to_redis(gamename,
gamedir=None,
redisbasekey=REDIS_FTWC_PLAYTHROUGHS,
randseed=DEFAULT_PTHRU_SEED,
goal_type=GOAL_MEAL,
do_write=False,
redis=None,
skip_existing=True):
if gamedir is None:
if redis:
gamedir = get_dir_for_game(redis, gamename)
else:
gamedir = TW_TRAINING_DIR # best guess, we'll see later if it's there or not
if gamename.endswith(".z8") or gamefile.endswith(".ulx"):
_gamefile = f"{gamedir}/{gamename}"
else:
_gamefile = f"{gamedir}/{gamename}.z8"
if not os.path.exists(_gamefile):
_gamefile = f"{gamedir}/{gamename}.ulx"
redis_ops = 0
num_steps = 0
step_array = [
] # convert json dict data (with redundant keys) to an array for convenience
ptid = playthrough_id(
objective_name=goal_type, seed=randseed
) # playtrough ID (which of potentially different) for this gamename
if not redis:
do_write = False
if do_write:
if not redis.exists(f'{redisbasekey}:{gamename}'):
redis_ops += 1
redis.jsonset(f'{redisbasekey}:{gamename}', Path.rootPath(),
{ptid: {}})
elif skip_existing:
if redis.jsonobjlen(f'{redisbasekey}:{gamename}',
Path('.' + ptid)): # if exists and non-empty
print(f"SKIPPED EXISTING playthrough {gamename}")
return num_steps, redis_ops
_dones = [0]
_rewards = [0]
next_cmds = ['start']
gymenv, _obs, _infos = start_game_for_playthrough(_gamefile)
redis_ops, playthru_step_data = save_playthrough_step_info_to_redis(
gamename,
num_steps,
_obs,
_rewards,
_dones,
_infos,
next_cmds,
redisbasekey=redisbasekey,
ptid=ptid,
redis=redis,
do_write=do_write,
redis_ops=redis_ops)
step_array.append(playthru_step_data[format_stepkey(num_steps)])
next_cmds = _infos['tw_o_step']
while not _dones[0] and num_steps < MAX_PLAYTHROUGH_STEPS + 1:
num_steps += 1
_obs, _rewards, _dones, _infos = step_game_for_playthrough(
gymenv, next_cmds)
redis_ops, playthru_step_data = save_playthrough_step_info_to_redis(
gamename,
num_steps,
_obs,
_rewards,
_dones,
_infos,
next_cmds,
redisbasekey=redisbasekey,
ptid=ptid,
redis=redis,
do_write=do_write,
redis_ops=redis_ops)
step_array.append(playthru_step_data[format_stepkey(num_steps)])
next_cmds = _infos['tw_o_step']
gymenv.close()
print(
f"----------------- {gamename} playthrough steps: {num_steps} Redis writes {redis_ops} ----------------"
)
return num_steps, redis_ops, step_array
def extracted_data_to_redis(game_names=None,
xtract_dir=XTRACT_DIR,
redis=None):
do_write = False
if not game_names:
game_names = []
itercount = 0
room_count = 0
desc_count = 0
failed_count = 0
max_desc_wlen = 0
max_desc_wlen = 0
max_desc_str = ''
redis_ops = 0
for bn in game_names: #[0:10]:
if do_write:
if not redis.exists(f'{REDIS_EXTRACTED_DATA}:{bn}'):
redis_ops += 1
redis.jsonset(f'{REDIS_EXTRACTED_DATA}:{bn}', Path.rootPath(),
{'room': {}})
xdir = xtract_dir + '/' + bn
itercount += 1
data_files = os.listdir(xdir)
# suffixes = ['.room_state', '.desc']
suffix = '.room_state'
filtered_files = list(
filter(lambda fname: fname.endswith(suffix), data_files))
sorted_files = list(
map(lambda fname: fname.split('_', maxsplit=2), filtered_files))
for i, tup in enumerate(sorted_files):
tup[1] = int(tup[1]) # convert room numbers from ascii to int
tup[2] = tup[2].split('.')[0] # the room name
tup.append(
filtered_files[i]
) # add the original file name (NOTE: tup is a list, not a tuple)
# sort by room number, not alphabetically
sorted_files = sorted(sorted_files)
# print(sorted_files[0:3])
for (_, i, room, f) in sorted_files:
obj_lines = []
room_count += 1
with open(xdir + '/' + f, 'r') as room_state_file:
state_str = room_state_file.read()
room_name = f.split('.')[0]
with open(xdir + '/' + room_name + '.desc', 'r') as desc_file:
desc_lines = [line.strip() for line in desc_file]
desc_str = ' '.join(desc_lines)
if len(desc_str):
desc_count += 1
if len(desc_str) > len(max_desc_str):
if len(desc_str.split(' ')) > max_desc_wlen:
max_desc_wlen = len(desc_str.split(' '))
max_desc_str = desc_str
jsonobj = {
'room_name': room,
'room_id': i,
'init_descr': desc_str,
'init_state': state_str
}
if do_write:
redis_ops += 1
redis.jsonset(f'{REDIS_EXTRACTED_DATA}:{bn}',
Path(f'.room.{room}'), jsonobj)
#print(bn, room.upper(), f)
print("Processed", itercount, "games, ", room_count, "rooms.")
print(redis_ops, "redis write ops")
print("max room description length =", len(max_desc_str), "# words=",
max_desc_wlen)
assert desc_count == room_count