def save_playthrough_to_redis(gamename, gamedir=None, redisbasekey=REDIS_FTWC_PLAYTHROUGHS, randseed=DEFAULT_PTHRU_SEED, goal_type=GOAL_MEAL, do_write=False, redis=None, skip_existing=True): if gamedir is None: if redis: gamedir = get_dir_for_game(redis, gamename) else: gamedir = TW_TRAINING_DIR # best guess, we'll see later if it's there or not if gamename.endswith(".z8") or gamefile.endswith(".ulx"): _gamefile = f"{gamedir}/{gamename}" else: _gamefile = f"{gamedir}/{gamename}.z8" if not os.path.exists(_gamefile): _gamefile = f"{gamedir}/{gamename}.ulx" redis_ops = 0 num_steps = 0 step_array = [ ] # convert json dict data (with redundant keys) to an array for convenience ptid = playthrough_id( objective_name=goal_type, seed=randseed ) # playtrough ID (which of potentially different) for this gamename if not redis: do_write = False if do_write: if not redis.exists(f'{redisbasekey}:{gamename}'): redis_ops += 1 redis.jsonset(f'{redisbasekey}:{gamename}', Path.rootPath(), {ptid: {}}) elif skip_existing: if redis.jsonobjlen(f'{redisbasekey}:{gamename}', Path('.' + ptid)): # if exists and non-empty print(f"SKIPPED EXISTING playthrough {gamename}") return num_steps, redis_ops _dones = [0] _rewards = [0] next_cmds = ['start'] gymenv, _obs, _infos = start_game_for_playthrough(_gamefile) redis_ops, playthru_step_data = save_playthrough_step_info_to_redis( gamename, num_steps, _obs, _rewards, _dones, _infos, next_cmds, redisbasekey=redisbasekey, ptid=ptid, redis=redis, do_write=do_write, redis_ops=redis_ops) step_array.append(playthru_step_data[format_stepkey(num_steps)]) next_cmds = _infos['tw_o_step'] while not _dones[0] and num_steps < MAX_PLAYTHROUGH_STEPS + 1: num_steps += 1 _obs, _rewards, _dones, _infos = step_game_for_playthrough( gymenv, next_cmds) redis_ops, playthru_step_data = save_playthrough_step_info_to_redis( gamename, num_steps, _obs, _rewards, _dones, _infos, next_cmds, redisbasekey=redisbasekey, ptid=ptid, redis=redis, do_write=do_write, redis_ops=redis_ops) step_array.append(playthru_step_data[format_stepkey(num_steps)]) next_cmds = _infos['tw_o_step'] gymenv.close() print( f"----------------- {gamename} playthrough steps: {num_steps} Redis writes {redis_ops} ----------------" ) return num_steps, redis_ops, step_array
def extracted_data_to_redis(game_names=None, xtract_dir=XTRACT_DIR, redis=None): do_write = False if not game_names: game_names = [] itercount = 0 room_count = 0 desc_count = 0 failed_count = 0 max_desc_wlen = 0 max_desc_wlen = 0 max_desc_str = '' redis_ops = 0 for bn in game_names: #[0:10]: if do_write: if not redis.exists(f'{REDIS_EXTRACTED_DATA}:{bn}'): redis_ops += 1 redis.jsonset(f'{REDIS_EXTRACTED_DATA}:{bn}', Path.rootPath(), {'room': {}}) xdir = xtract_dir + '/' + bn itercount += 1 data_files = os.listdir(xdir) # suffixes = ['.room_state', '.desc'] suffix = '.room_state' filtered_files = list( filter(lambda fname: fname.endswith(suffix), data_files)) sorted_files = list( map(lambda fname: fname.split('_', maxsplit=2), filtered_files)) for i, tup in enumerate(sorted_files): tup[1] = int(tup[1]) # convert room numbers from ascii to int tup[2] = tup[2].split('.')[0] # the room name tup.append( filtered_files[i] ) # add the original file name (NOTE: tup is a list, not a tuple) # sort by room number, not alphabetically sorted_files = sorted(sorted_files) # print(sorted_files[0:3]) for (_, i, room, f) in sorted_files: obj_lines = [] room_count += 1 with open(xdir + '/' + f, 'r') as room_state_file: state_str = room_state_file.read() room_name = f.split('.')[0] with open(xdir + '/' + room_name + '.desc', 'r') as desc_file: desc_lines = [line.strip() for line in desc_file] desc_str = ' '.join(desc_lines) if len(desc_str): desc_count += 1 if len(desc_str) > len(max_desc_str): if len(desc_str.split(' ')) > max_desc_wlen: max_desc_wlen = len(desc_str.split(' ')) max_desc_str = desc_str jsonobj = { 'room_name': room, 'room_id': i, 'init_descr': desc_str, 'init_state': state_str } if do_write: redis_ops += 1 redis.jsonset(f'{REDIS_EXTRACTED_DATA}:{bn}', Path(f'.room.{room}'), jsonobj) #print(bn, room.upper(), f) print("Processed", itercount, "games, ", room_count, "rooms.") print(redis_ops, "redis write ops") print("max room description length =", len(max_desc_str), "# words=", max_desc_wlen) assert desc_count == room_count