def create_difficulty_map(redserv=None,
difficulty_map_key=REDIS_GATA_DIFFICULTY_MAP,
game_dirs=None):
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
if game_dirs is None:
training_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TRAINING_DIR')
validation_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_VALIDATION_DIR')
test_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TEST_DIR')
game_dirs = (training_dir, validation_dir, test_dir)
print(
f"create_difficulty_map({difficulty_map_key}, game_dirs={game_dirs})")
for game_dir in game_dirs:
print("create_difficulty_map -- GAME DIR:", game_dir)
game_names_ = []
for level in range(1, 11):
difficulty = f"difficulty_level_{level}"
print("\n-------------------", difficulty)
games_list = count_game_files(game_dir + difficulty)
game_names_.extend(games_list)
_rj.sadd(difficulty_map_key + str(level), *games_list)
print(
f"total games in {game_dir}: {len(game_names_)} {len(set(game_names_))}"
)
assert len(game_names_) == len(
set(game_names_)) # they should all be unique
if redserv is None:
_rj.close()
def create_nsteps_map(redserv=None,
nsteps_map_key=REDIS_FTWC_NSTEPS_MAP,
nsteps_index_key=REDIS_FTWC_NSTEPS_INDEX,
gameset_keys=FTWC_GAME_SETS):
""" after all playthroughs have been save to redis, index number of steps <-> game names """
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
if nsteps_map_key == REDIS_GATA_NSTEPS_MAP:
redisbasekey = REDIS_GATA_PLAYTHROUGHS
elif nsteps_map_key == REDIS_FTWC_NSTEPS_MAP:
redisbasekey = REDIS_FTWC_PLAYTHROUGHS
else:
assert False, "Unknown Redis nsteps_map_key " + nsteps_map_key
for key in _rj.keys(nsteps_index_key + "*"):
print("Will delete:", key)
_rj.delete(key)
print(_rj.hlen(nsteps_map_key))
_rj.delete(nsteps_map_key)
for setkey in gameset_keys:
game_names_ = _rj.smembers(setkey)
for _gn in game_names_:
nsteps = retrieve_playthrough_nsteps(_gn,
redis=_rj,
redisbasekey=redisbasekey)
if nsteps > 0:
print(nsteps, _gn)
_rj.hset(nsteps_map_key, _gn, nsteps)
_rj.sadd(f"{nsteps_index_key}{nsteps}", _gn)
print(len(_rj.keys(nsteps_index_key + "*")), _rj.hlen(nsteps_map_key))
total = 0
sort_list = []
for key in _rj.keys(nsteps_index_key + "*"):
nsteps = int(key.split(':')[-1])
num_games = _rj.scard(key)
total += num_games
sort_list.append((nsteps, num_games, key))
# print(key, "has", num_games, "game names")
sort_list.sort()
for nsteps, num_games, setkey in sort_list:
print(f"[{nsteps}]\t {num_games}\t {setkey}")
if redserv is None:
_rj.close()
def create_skills_map(redserv=None,
skillsmap_key=REDIS_FTWC_SKILLS_MAP,
gameset_keys=FTWC_GAME_SETS):
""" after all game names have been added to redis, we map skills to game names"""
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
skills_index = {} # maps skillname to a set of game names
all_mapped_skills = set() # all game names that are in the skills map
for setkey in gameset_keys:
game_names = _rj.smembers(setkey)
print(f"{setkey} has {len(game_names)} members")
for g in game_names:
gid, sklist = split_gamename(g)
# print(g, gid, sklist)
for skill in sklist:
if skill not in skills_index:
skills_index[skill] = set()
skills_index[skill].add(g)
_rj.sadd(skillsmap_key + skill, g)
# print(len(skills_index), skills_index.keys())
# for key in skills_index.keys():
# print(key, len(skills_index[key]))
# all_mapped_skills = all_mapped_skills.union(skills_index[key])
skillsmap_keys = _rj.keys(skillsmap_key + "*")
for k in skillsmap_keys:
print(k, _rj.scard(k))
all_mapped_skills = all_mapped_skills.union(_rj.smembers(k))
print(
f"TOTAL # of game files for which skills have been mapped: {len(all_mapped_skills)}"
)
if redserv is None:
_rj.close()
from datetime import datetime
from pprint import pprint
from rejson import Client, Path
rj = Client(host='localhost', port=6379, decode_responses=True)
# Get single key:value pair
pprint(rj.jsonget("redis_club_urls:item73", Path.rootPath()))
# Convert timestamp from iso to datetime
timestamp = rj.jsonget("redis_club_urls:item73",
Path.rootPath())['last_modified']
timestamp = datetime.fromisoformat(timestamp)
# Get all keys
print(rj.keys())
# Get all values matching pattern from keys
for key in rj.scan_iter("redis_club_urls:item:*"):
print(rj.jsonget(key, Path('.club_page')))
# Add all values matching pattern to new key
for key in rj.scan_iter("redis_club_urls:item:*"):
val = rj.jsonget(key, Path('.club_page'))
rj.sadd('all_urls', val)
class RedisDriver(DatabaseDriver):
_host = "localhost"
_port = 6379
_client = None
def __init__(self, host: str = "localhost", port: int = 6379):
self._host = host
self._port = port
self._client = Client(host=host,
port=port,
decode_responses=True,
encoder=JSONSchemaObject.JSONSchemaEncoder())
def find_by_ref(self, ref: str):
return self._client.jsonget(ref)
def find_id_by(self, idx: str, value: str, version: str):
result = []
for member in self._client.smembers("{}:{}".format(idx, value)):
if version == "all":
result.append(member)
continue
# we split the index to check against the version
idxs = str(member).split(":")
# the _version is the second token of idxs
if idxs[1] == version:
result.append(member)
return result
def save(self, obj_list: list, indexed_attrs: list):
# First cycle is just to verify if we do not have any
# index integrity violation
for obj in indexed_attrs:
# We do not store neither _id or _version
if obj[1] == "_id" or obj[1] == "_version":
continue
if obj[2] is None or obj[2] == "":
raise ValueError("Indexed value {} must not be empty".format(
obj[1]))
# the indexed is composed by schema path:indexes:attr_name
indexed_key = store_name = "{}:indexes:{}:{}".format(
obj[0], obj[1], obj[2])
# we already have this key let's get any value and make
# sure we this belongs to the same id
for member in self._client.smembers(indexed_key):
# we only need to use one element since the _id MUST be equal
idxs = str(member).split(":")
# the _id is the first token of idxs, check if we recieved the same
# id, if not this is a index violation
if not str(obj[3]).startswith(idxs[0]):
raise ValueError(
"{}:{} not unique, another object already have that value"
.format(obj[1], obj[2]))
# we just need one iteration
break
# this cycle we just store the indexes
for obj in indexed_attrs:
if obj[2] is None or obj[2] == "" or obj[1] == "_id" or obj[
1] == "_version":
continue
# Set the store name and store data
store_name = "{}:indexes:{}:{}".format(obj[0], obj[1], obj[2])
store_data = obj[3]
self._client.sadd(store_name, store_data)
# We now store the actual objects, and return the added ids
ids = []
for obj in obj_list:
# Set the store name and store data
store_name = "{}:{}".format(obj[0], obj[1])
store_data = obj[2]
self._client.jsonset(store_name, Path.rootPath(), store_data)
ids.append(obj[1])
return ids
