def create_skills_map(redserv=None,
skillsmap_key=REDIS_FTWC_SKILLS_MAP,
gameset_keys=FTWC_GAME_SETS):
""" after all game names have been added to redis, we map skills to game names"""
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
skills_index = {} # maps skillname to a set of game names
all_mapped_skills = set() # all game names that are in the skills map
for setkey in gameset_keys:
game_names = _rj.smembers(setkey)
print(f"{setkey} has {len(game_names)} members")
for g in game_names:
gid, sklist = split_gamename(g)
# print(g, gid, sklist)
for skill in sklist:
if skill not in skills_index:
skills_index[skill] = set()
skills_index[skill].add(g)
_rj.sadd(skillsmap_key + skill, g)
# print(len(skills_index), skills_index.keys())
# for key in skills_index.keys():
# print(key, len(skills_index[key]))
# all_mapped_skills = all_mapped_skills.union(skills_index[key])
skillsmap_keys = _rj.keys(skillsmap_key + "*")
for k in skillsmap_keys:
print(k, _rj.scard(k))
all_mapped_skills = all_mapped_skills.union(_rj.smembers(k))
print(
f"TOTAL # of game files for which skills have been mapped: {len(all_mapped_skills)}"
)
if redserv is None:
_rj.close()
def create_nsteps_map(redserv=None,
nsteps_map_key=REDIS_FTWC_NSTEPS_MAP,
nsteps_index_key=REDIS_FTWC_NSTEPS_INDEX,
gameset_keys=FTWC_GAME_SETS):
""" after all playthroughs have been save to redis, index number of steps <-> game names """
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
if nsteps_map_key == REDIS_GATA_NSTEPS_MAP:
redisbasekey = REDIS_GATA_PLAYTHROUGHS
elif nsteps_map_key == REDIS_FTWC_NSTEPS_MAP:
redisbasekey = REDIS_FTWC_PLAYTHROUGHS
else:
assert False, "Unknown Redis nsteps_map_key " + nsteps_map_key
for key in _rj.keys(nsteps_index_key + "*"):
print("Will delete:", key)
_rj.delete(key)
print(_rj.hlen(nsteps_map_key))
_rj.delete(nsteps_map_key)
for setkey in gameset_keys:
game_names_ = _rj.smembers(setkey)
for _gn in game_names_:
nsteps = retrieve_playthrough_nsteps(_gn,
redis=_rj,
redisbasekey=redisbasekey)
if nsteps > 0:
print(nsteps, _gn)
_rj.hset(nsteps_map_key, _gn, nsteps)
_rj.sadd(f"{nsteps_index_key}{nsteps}", _gn)
print(len(_rj.keys(nsteps_index_key + "*")), _rj.hlen(nsteps_map_key))
total = 0
sort_list = []
for key in _rj.keys(nsteps_index_key + "*"):
nsteps = int(key.split(':')[-1])
num_games = _rj.scard(key)
total += num_games
sort_list.append((nsteps, num_games, key))
# print(key, "has", num_games, "game names")
sort_list.sort()
for nsteps, num_games, setkey in sort_list:
print(f"[{nsteps}]\t {num_games}\t {setkey}")
if redserv is None:
_rj.close()
class RedisDriver(DatabaseDriver):
_host = "localhost"
_port = 6379
_client = None
def __init__(self, host: str = "localhost", port: int = 6379):
self._host = host
self._port = port
self._client = Client(host=host,
port=port,
decode_responses=True,
encoder=JSONSchemaObject.JSONSchemaEncoder())
def find_by_ref(self, ref: str):
return self._client.jsonget(ref)
def find_id_by(self, idx: str, value: str, version: str):
result = []
for member in self._client.smembers("{}:{}".format(idx, value)):
if version == "all":
result.append(member)
continue
# we split the index to check against the version
idxs = str(member).split(":")
# the _version is the second token of idxs
if idxs[1] == version:
result.append(member)
return result
def save(self, obj_list: list, indexed_attrs: list):
# First cycle is just to verify if we do not have any
# index integrity violation
for obj in indexed_attrs:
# We do not store neither _id or _version
if obj[1] == "_id" or obj[1] == "_version":
continue
if obj[2] is None or obj[2] == "":
raise ValueError("Indexed value {} must not be empty".format(
obj[1]))
# the indexed is composed by schema path:indexes:attr_name
indexed_key = store_name = "{}:indexes:{}:{}".format(
obj[0], obj[1], obj[2])
# we already have this key let's get any value and make
# sure we this belongs to the same id
for member in self._client.smembers(indexed_key):
# we only need to use one element since the _id MUST be equal
idxs = str(member).split(":")
# the _id is the first token of idxs, check if we recieved the same
# id, if not this is a index violation
if not str(obj[3]).startswith(idxs[0]):
raise ValueError(
"{}:{} not unique, another object already have that value"
.format(obj[1], obj[2]))
# we just need one iteration
break
# this cycle we just store the indexes
for obj in indexed_attrs:
if obj[2] is None or obj[2] == "" or obj[1] == "_id" or obj[
1] == "_version":
continue
# Set the store name and store data
store_name = "{}:indexes:{}:{}".format(obj[0], obj[1], obj[2])
store_data = obj[3]
self._client.sadd(store_name, store_data)
# We now store the actual objects, and return the added ids
ids = []
for obj in obj_list:
# Set the store name and store data
store_name = "{}:{}".format(obj[0], obj[1])
store_data = obj[2]
self._client.jsonset(store_name, Path.rootPath(), store_data)
ids.append(obj[1])
return ids
def main(args):
total_redis_ops = 0
total_files = 0
if args.which == 'extra':
print("Generating playthrough data for games from",
args.extra_games_dir)
assert args.files_only, "Extra game playthroughs currently require '--files-only' option"
if args.files_only:
rj = None
print("** Not using Redis, all output goes to file system. **")
if args.export_files:
print("++ Also saving generated playthough data to files ++")
else:
rj = Client(host='localhost', port=6379,
decode_responses=True) # redisJSON API
if args.export_files:
print("Exporting generated playthough data to files")
else:
print("Importing playrhroughs to Redis...")
if not args.which:
assert False, "Expected which= one of [extra, train, valid, test, miniset, gata_train, gata_valid, gata_test]"
exit(1)
rediskey = gamesets[args.which]
if rediskey:
if (args.which).startswith("gata_"):
redisbasekey = REDIS_GATA_PLAYTHROUGHS
else:
redisbasekey = REDIS_FTWC_PLAYTHROUGHS
#num_games = rj.scard(rediskey)
gamenames = rj.smembers(rediskey)
if args.which == 'miniset':
# gamenames = list(gamenames)[0:3] # just the first 3
gamenames = [
'tw-cooking-recipe3+take3+cut+go6-Z7L8CvEPsO53iKDg'
]
else:
gamenames = _list_games(args.extra_games_dir)
#num_games = len(gamenames)
print("num_games:", len(gamenames), gamenames[0:5])
if args.start_idx is not None:
gamenames = gamenames[args.start_idx:args.start_idx + 1000]
#small slice FOR TESTING: gamenames = gamenames[0:5]
if args.export_files:
if (args.which).startswith("gata_"):
destdir = f'./playthru_data/{args.which}'
elif args.which != 'extra':
destdir = f'./playthru_data/{args.which}'
elif args.which == 'extra':
destdir = f'./playthru_extra/'
else:
assert False, f"UNEXPECTED: invalid combination of args? {str(args)}"
for i, gname in enumerate(tqdm(gamenames)):
if args.which == 'extra':
print(f"[{i}] {gname}")
num_steps, redis_ops, playthru = save_playthrough_to_redis(
gname,
gamedir=args.extra_games_dir,
redis=None,
redisbasekey=None,
do_write=False)
for s in playthru:
print(s, '\n')
if args.export_files:
total_files += export_playthru(gname.split('.')[0],
playthru,
destdir=destdir)
else:
if not args.export_files:
print(f"[{i}] BEGIN PLAYTHROUGH: {gname}")
num_steps, redis_ops, _ = save_playthrough_to_redis(
gname,
redis=rj,
redisbasekey=redisbasekey,
do_write=args.do_write)
print(
f"[{i}] PLAYTHROUGH {gname}: steps:{num_steps} to redis: {redis_ops}"
)
total_redis_ops += redis_ops
else:
playthru = retrieve_playthrough_json(
gname, redis=redis, redisbasekey=redisbasekey)
total_files += export_playthru(gname,
playthru,
destdir=destdir)
print("Total redis writes:", total_redis_ops)
print("Total files exported:", total_files)
if rj:
if args.do_write and not args.export_files:
rj.save()
rj.close()
