def create_difficulty_map(redserv=None,
difficulty_map_key=REDIS_GATA_DIFFICULTY_MAP,
game_dirs=None):
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
if game_dirs is None:
training_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TRAINING_DIR')
validation_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_VALIDATION_DIR')
test_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TEST_DIR')
game_dirs = (training_dir, validation_dir, test_dir)
print(
f"create_difficulty_map({difficulty_map_key}, game_dirs={game_dirs})")
for game_dir in game_dirs:
print("create_difficulty_map -- GAME DIR:", game_dir)
game_names_ = []
for level in range(1, 11):
difficulty = f"difficulty_level_{level}"
print("\n-------------------", difficulty)
games_list = count_game_files(game_dir + difficulty)
game_names_.extend(games_list)
_rj.sadd(difficulty_map_key + str(level), *games_list)
print(
f"total games in {game_dir}: {len(game_names_)} {len(set(game_names_))}"
)
assert len(game_names_) == len(
set(game_names_)) # they should all be unique
if redserv is None:
_rj.close()
def get_max_score(gn, redserv=None):
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
assert isinstance(redserv, Client)
_rj = redserv
dirpath = get_dir_for_game(_rj, gn)
if redserv is None:
_rj.close()
with open(f"{dirpath}/{gn}.json", "r") as f:
metadata = json.load(f)['metadata']
return int(metadata['max_score'])
def create_nsteps_map(redserv=None,
nsteps_map_key=REDIS_FTWC_NSTEPS_MAP,
nsteps_index_key=REDIS_FTWC_NSTEPS_INDEX,
gameset_keys=FTWC_GAME_SETS):
""" after all playthroughs have been save to redis, index number of steps <-> game names """
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
if nsteps_map_key == REDIS_GATA_NSTEPS_MAP:
redisbasekey = REDIS_GATA_PLAYTHROUGHS
elif nsteps_map_key == REDIS_FTWC_NSTEPS_MAP:
redisbasekey = REDIS_FTWC_PLAYTHROUGHS
else:
assert False, "Unknown Redis nsteps_map_key " + nsteps_map_key
for key in _rj.keys(nsteps_index_key + "*"):
print("Will delete:", key)
_rj.delete(key)
print(_rj.hlen(nsteps_map_key))
_rj.delete(nsteps_map_key)
for setkey in gameset_keys:
game_names_ = _rj.smembers(setkey)
for _gn in game_names_:
nsteps = retrieve_playthrough_nsteps(_gn,
redis=_rj,
redisbasekey=redisbasekey)
if nsteps > 0:
print(nsteps, _gn)
_rj.hset(nsteps_map_key, _gn, nsteps)
_rj.sadd(f"{nsteps_index_key}{nsteps}", _gn)
print(len(_rj.keys(nsteps_index_key + "*")), _rj.hlen(nsteps_map_key))
total = 0
sort_list = []
for key in _rj.keys(nsteps_index_key + "*"):
nsteps = int(key.split(':')[-1])
num_games = _rj.scard(key)
total += num_games
sort_list.append((nsteps, num_games, key))
# print(key, "has", num_games, "game names")
sort_list.sort()
for nsteps, num_games, setkey in sort_list:
print(f"[{nsteps}]\t {num_games}\t {setkey}")
if redserv is None:
_rj.close()
def lookup_difficulty_level(gamename,
redserv=None,
difficulty_map_key=REDIS_GATA_DIFFICULTY_MAP):
# assert gamename , "Missing required argument: gamename"
level = -1
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
for i in range(1, 11):
if _rj.sismember(difficulty_map_key + str(i), gamename):
level = i
break
if redserv is None:
_rj.close()
return level
def select_games_for_mingpt(redserv=None):
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
# for n in range(70):
# rkey = f"{REDIS_FTWC_NSTEPS_INDEX}{n}"
# if rj.exists(rkey):
# n_nsteps = rj.scard(f"{REDIS_FTWC_NSTEPS_INDEX}{n}")
# n_nodrop = rj.sdiff(f"{REDIS_FTWC_NSTEPS_INDEX}{n}", "ftwc:cog2019:skills:drop")
# # maxtoks, mintoks = get_max_token_length(n_nodrop)
# if n <= 32 and len(n_nodrop):
# rj.sadd(REDIS_MINGPT_ALL, *n_nodrop)
# print(f"{n:2d}:__{n_nsteps:3d}__{len(n_nodrop):3d}") #" {maxtoks:4d} {mintoks:4d}")
if redserv is None:
_rj.close()
def create_skills_map(redserv=None,
skillsmap_key=REDIS_FTWC_SKILLS_MAP,
gameset_keys=FTWC_GAME_SETS):
""" after all game names have been added to redis, we map skills to game names"""
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
skills_index = {} # maps skillname to a set of game names
all_mapped_skills = set() # all game names that are in the skills map
for setkey in gameset_keys:
game_names = _rj.smembers(setkey)
print(f"{setkey} has {len(game_names)} members")
for g in game_names:
gid, sklist = split_gamename(g)
# print(g, gid, sklist)
for skill in sklist:
if skill not in skills_index:
skills_index[skill] = set()
skills_index[skill].add(g)
_rj.sadd(skillsmap_key + skill, g)
# print(len(skills_index), skills_index.keys())
# for key in skills_index.keys():
# print(key, len(skills_index[key]))
# all_mapped_skills = all_mapped_skills.union(skills_index[key])
skillsmap_keys = _rj.keys(skillsmap_key + "*")
for k in skillsmap_keys:
print(k, _rj.scard(k))
all_mapped_skills = all_mapped_skills.union(_rj.smembers(k))
print(
f"TOTAL # of game files for which skills have been mapped: {len(all_mapped_skills)}"
)
if redserv is None:
_rj.close()
def retrieve_playthrough_json(
gamename,
redis=None, # redis server: expected to be a RedisJSON client connection
ptid=playthrough_id(), # default playthrough, can optionally specify
redisbasekey=REDIS_FTWC_PLAYTHROUGHS,
randseed=DEFAULT_PTHRU_SEED):
if redis is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
assert isinstance(redis, Client)
_rj = redis
jsonobj = _rj.jsonget(f'{redisbasekey}:{gamename}', Path('.' + ptid))
if redis is None:
_rj.close()
step_array = [
] # convert json dict data (with redundant keys) to an array for convenience
for i, step_key in enumerate(list(jsonobj.keys())):
assert step_key == format_stepkey(
i) # format_stepkey() from twutils.playthroughs
step_array.append(jsonobj[step_key])
assert len(step_array) == len(jsonobj.keys())
return step_array
def main(args):
total_redis_ops = 0
total_files = 0
if args.which == 'extra':
print("Generating playthrough data for games from",
args.extra_games_dir)
assert args.files_only, "Extra game playthroughs currently require '--files-only' option"
if args.files_only:
rj = None
print("** Not using Redis, all output goes to file system. **")
if args.export_files:
print("++ Also saving generated playthough data to files ++")
else:
rj = Client(host='localhost', port=6379,
decode_responses=True) # redisJSON API
if args.export_files:
print("Exporting generated playthough data to files")
else:
print("Importing playrhroughs to Redis...")
if not args.which:
assert False, "Expected which= one of [extra, train, valid, test, miniset, gata_train, gata_valid, gata_test]"
exit(1)
rediskey = gamesets[args.which]
if rediskey:
if (args.which).startswith("gata_"):
redisbasekey = REDIS_GATA_PLAYTHROUGHS
else:
redisbasekey = REDIS_FTWC_PLAYTHROUGHS
#num_games = rj.scard(rediskey)
gamenames = rj.smembers(rediskey)
if args.which == 'miniset':
# gamenames = list(gamenames)[0:3] # just the first 3
gamenames = [
'tw-cooking-recipe3+take3+cut+go6-Z7L8CvEPsO53iKDg'
]
else:
gamenames = _list_games(args.extra_games_dir)
#num_games = len(gamenames)
print("num_games:", len(gamenames), gamenames[0:5])
if args.start_idx is not None:
gamenames = gamenames[args.start_idx:args.start_idx + 1000]
#small slice FOR TESTING: gamenames = gamenames[0:5]
if args.export_files:
if (args.which).startswith("gata_"):
destdir = f'./playthru_data/{args.which}'
elif args.which != 'extra':
destdir = f'./playthru_data/{args.which}'
elif args.which == 'extra':
destdir = f'./playthru_extra/'
else:
assert False, f"UNEXPECTED: invalid combination of args? {str(args)}"
for i, gname in enumerate(tqdm(gamenames)):
if args.which == 'extra':
print(f"[{i}] {gname}")
num_steps, redis_ops, playthru = save_playthrough_to_redis(
gname,
gamedir=args.extra_games_dir,
redis=None,
redisbasekey=None,
do_write=False)
for s in playthru:
print(s, '\n')
if args.export_files:
total_files += export_playthru(gname.split('.')[0],
playthru,
destdir=destdir)
else:
if not args.export_files:
print(f"[{i}] BEGIN PLAYTHROUGH: {gname}")
num_steps, redis_ops, _ = save_playthrough_to_redis(
gname,
redis=rj,
redisbasekey=redisbasekey,
do_write=args.do_write)
print(
f"[{i}] PLAYTHROUGH {gname}: steps:{num_steps} to redis: {redis_ops}"
)
total_redis_ops += redis_ops
else:
playthru = retrieve_playthrough_json(
gname, redis=redis, redisbasekey=redisbasekey)
total_files += export_playthru(gname,
playthru,
destdir=destdir)
print("Total redis writes:", total_redis_ops)
print("Total files exported:", total_files)
if rj:
if args.do_write and not args.export_files:
rj.save()
rj.close()