def create_difficulty_map(redserv=None, difficulty_map_key=REDIS_GATA_DIFFICULTY_MAP, game_dirs=None): if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: _rj = redserv if game_dirs is None: training_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TRAINING_DIR') validation_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_VALIDATION_DIR') test_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TEST_DIR') game_dirs = (training_dir, validation_dir, test_dir) print( f"create_difficulty_map({difficulty_map_key}, game_dirs={game_dirs})") for game_dir in game_dirs: print("create_difficulty_map -- GAME DIR:", game_dir) game_names_ = [] for level in range(1, 11): difficulty = f"difficulty_level_{level}" print("\n-------------------", difficulty) games_list = count_game_files(game_dir + difficulty) game_names_.extend(games_list) _rj.sadd(difficulty_map_key + str(level), *games_list) print( f"total games in {game_dir}: {len(game_names_)} {len(set(game_names_))}" ) assert len(game_names_) == len( set(game_names_)) # they should all be unique if redserv is None: _rj.close()
def get_max_score(gn, redserv=None): if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: assert isinstance(redserv, Client) _rj = redserv dirpath = get_dir_for_game(_rj, gn) if redserv is None: _rj.close() with open(f"{dirpath}/{gn}.json", "r") as f: metadata = json.load(f)['metadata'] return int(metadata['max_score'])
def create_nsteps_map(redserv=None, nsteps_map_key=REDIS_FTWC_NSTEPS_MAP, nsteps_index_key=REDIS_FTWC_NSTEPS_INDEX, gameset_keys=FTWC_GAME_SETS): """ after all playthroughs have been save to redis, index number of steps <-> game names """ if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: _rj = redserv if nsteps_map_key == REDIS_GATA_NSTEPS_MAP: redisbasekey = REDIS_GATA_PLAYTHROUGHS elif nsteps_map_key == REDIS_FTWC_NSTEPS_MAP: redisbasekey = REDIS_FTWC_PLAYTHROUGHS else: assert False, "Unknown Redis nsteps_map_key " + nsteps_map_key for key in _rj.keys(nsteps_index_key + "*"): print("Will delete:", key) _rj.delete(key) print(_rj.hlen(nsteps_map_key)) _rj.delete(nsteps_map_key) for setkey in gameset_keys: game_names_ = _rj.smembers(setkey) for _gn in game_names_: nsteps = retrieve_playthrough_nsteps(_gn, redis=_rj, redisbasekey=redisbasekey) if nsteps > 0: print(nsteps, _gn) _rj.hset(nsteps_map_key, _gn, nsteps) _rj.sadd(f"{nsteps_index_key}{nsteps}", _gn) print(len(_rj.keys(nsteps_index_key + "*")), _rj.hlen(nsteps_map_key)) total = 0 sort_list = [] for key in _rj.keys(nsteps_index_key + "*"): nsteps = int(key.split(':')[-1]) num_games = _rj.scard(key) total += num_games sort_list.append((nsteps, num_games, key)) # print(key, "has", num_games, "game names") sort_list.sort() for nsteps, num_games, setkey in sort_list: print(f"[{nsteps}]\t {num_games}\t {setkey}") if redserv is None: _rj.close()
def lookup_difficulty_level(gamename, redserv=None, difficulty_map_key=REDIS_GATA_DIFFICULTY_MAP): # assert gamename , "Missing required argument: gamename" level = -1 if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: _rj = redserv for i in range(1, 11): if _rj.sismember(difficulty_map_key + str(i), gamename): level = i break if redserv is None: _rj.close() return level
def select_games_for_mingpt(redserv=None): if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: _rj = redserv # for n in range(70): # rkey = f"{REDIS_FTWC_NSTEPS_INDEX}{n}" # if rj.exists(rkey): # n_nsteps = rj.scard(f"{REDIS_FTWC_NSTEPS_INDEX}{n}") # n_nodrop = rj.sdiff(f"{REDIS_FTWC_NSTEPS_INDEX}{n}", "ftwc:cog2019:skills:drop") # # maxtoks, mintoks = get_max_token_length(n_nodrop) # if n <= 32 and len(n_nodrop): # rj.sadd(REDIS_MINGPT_ALL, *n_nodrop) # print(f"{n:2d}:__{n_nsteps:3d}__{len(n_nodrop):3d}") #" {maxtoks:4d} {mintoks:4d}") if redserv is None: _rj.close()
def create_skills_map(redserv=None, skillsmap_key=REDIS_FTWC_SKILLS_MAP, gameset_keys=FTWC_GAME_SETS): """ after all game names have been added to redis, we map skills to game names""" if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: _rj = redserv skills_index = {} # maps skillname to a set of game names all_mapped_skills = set() # all game names that are in the skills map for setkey in gameset_keys: game_names = _rj.smembers(setkey) print(f"{setkey} has {len(game_names)} members") for g in game_names: gid, sklist = split_gamename(g) # print(g, gid, sklist) for skill in sklist: if skill not in skills_index: skills_index[skill] = set() skills_index[skill].add(g) _rj.sadd(skillsmap_key + skill, g) # print(len(skills_index), skills_index.keys()) # for key in skills_index.keys(): # print(key, len(skills_index[key])) # all_mapped_skills = all_mapped_skills.union(skills_index[key]) skillsmap_keys = _rj.keys(skillsmap_key + "*") for k in skillsmap_keys: print(k, _rj.scard(k)) all_mapped_skills = all_mapped_skills.union(_rj.smembers(k)) print( f"TOTAL # of game files for which skills have been mapped: {len(all_mapped_skills)}" ) if redserv is None: _rj.close()
def retrieve_playthrough_json( gamename, redis=None, # redis server: expected to be a RedisJSON client connection ptid=playthrough_id(), # default playthrough, can optionally specify redisbasekey=REDIS_FTWC_PLAYTHROUGHS, randseed=DEFAULT_PTHRU_SEED): if redis is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: assert isinstance(redis, Client) _rj = redis jsonobj = _rj.jsonget(f'{redisbasekey}:{gamename}', Path('.' + ptid)) if redis is None: _rj.close() step_array = [ ] # convert json dict data (with redundant keys) to an array for convenience for i, step_key in enumerate(list(jsonobj.keys())): assert step_key == format_stepkey( i) # format_stepkey() from twutils.playthroughs step_array.append(jsonobj[step_key]) assert len(step_array) == len(jsonobj.keys()) return step_array
def main(args): total_redis_ops = 0 total_files = 0 if args.which == 'extra': print("Generating playthrough data for games from", args.extra_games_dir) assert args.files_only, "Extra game playthroughs currently require '--files-only' option" if args.files_only: rj = None print("** Not using Redis, all output goes to file system. **") if args.export_files: print("++ Also saving generated playthough data to files ++") else: rj = Client(host='localhost', port=6379, decode_responses=True) # redisJSON API if args.export_files: print("Exporting generated playthough data to files") else: print("Importing playrhroughs to Redis...") if not args.which: assert False, "Expected which= one of [extra, train, valid, test, miniset, gata_train, gata_valid, gata_test]" exit(1) rediskey = gamesets[args.which] if rediskey: if (args.which).startswith("gata_"): redisbasekey = REDIS_GATA_PLAYTHROUGHS else: redisbasekey = REDIS_FTWC_PLAYTHROUGHS #num_games = rj.scard(rediskey) gamenames = rj.smembers(rediskey) if args.which == 'miniset': # gamenames = list(gamenames)[0:3] # just the first 3 gamenames = [ 'tw-cooking-recipe3+take3+cut+go6-Z7L8CvEPsO53iKDg' ] else: gamenames = _list_games(args.extra_games_dir) #num_games = len(gamenames) print("num_games:", len(gamenames), gamenames[0:5]) if args.start_idx is not None: gamenames = gamenames[args.start_idx:args.start_idx + 1000] #small slice FOR TESTING: gamenames = gamenames[0:5] if args.export_files: if (args.which).startswith("gata_"): destdir = f'./playthru_data/{args.which}' elif args.which != 'extra': destdir = f'./playthru_data/{args.which}' elif args.which == 'extra': destdir = f'./playthru_extra/' else: assert False, f"UNEXPECTED: invalid combination of args? {str(args)}" for i, gname in enumerate(tqdm(gamenames)): if args.which == 'extra': print(f"[{i}] {gname}") num_steps, redis_ops, playthru = save_playthrough_to_redis( gname, gamedir=args.extra_games_dir, redis=None, redisbasekey=None, do_write=False) for s in playthru: print(s, '\n') if args.export_files: total_files += export_playthru(gname.split('.')[0], playthru, destdir=destdir) else: if not args.export_files: print(f"[{i}] BEGIN PLAYTHROUGH: {gname}") num_steps, redis_ops, _ = save_playthrough_to_redis( gname, redis=rj, redisbasekey=redisbasekey, do_write=args.do_write) print( f"[{i}] PLAYTHROUGH {gname}: steps:{num_steps} to redis: {redis_ops}" ) total_redis_ops += redis_ops else: playthru = retrieve_playthrough_json( gname, redis=redis, redisbasekey=redisbasekey) total_files += export_playthru(gname, playthru, destdir=destdir) print("Total redis writes:", total_redis_ops) print("Total files exported:", total_files) if rj: if args.do_write and not args.export_files: rj.save() rj.close()