def create_difficulty_map(redserv=None, difficulty_map_key=REDIS_GATA_DIFFICULTY_MAP, game_dirs=None): if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: _rj = redserv if game_dirs is None: training_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TRAINING_DIR') validation_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_VALIDATION_DIR') test_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TEST_DIR') game_dirs = (training_dir, validation_dir, test_dir) print( f"create_difficulty_map({difficulty_map_key}, game_dirs={game_dirs})") for game_dir in game_dirs: print("create_difficulty_map -- GAME DIR:", game_dir) game_names_ = [] for level in range(1, 11): difficulty = f"difficulty_level_{level}" print("\n-------------------", difficulty) games_list = count_game_files(game_dir + difficulty) game_names_.extend(games_list) _rj.sadd(difficulty_map_key + str(level), *games_list) print( f"total games in {game_dir}: {len(game_names_)} {len(set(game_names_))}" ) assert len(game_names_) == len( set(game_names_)) # they should all be unique if redserv is None: _rj.close()
def create_nsteps_map(redserv=None, nsteps_map_key=REDIS_FTWC_NSTEPS_MAP, nsteps_index_key=REDIS_FTWC_NSTEPS_INDEX, gameset_keys=FTWC_GAME_SETS): """ after all playthroughs have been save to redis, index number of steps <-> game names """ if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: _rj = redserv if nsteps_map_key == REDIS_GATA_NSTEPS_MAP: redisbasekey = REDIS_GATA_PLAYTHROUGHS elif nsteps_map_key == REDIS_FTWC_NSTEPS_MAP: redisbasekey = REDIS_FTWC_PLAYTHROUGHS else: assert False, "Unknown Redis nsteps_map_key " + nsteps_map_key for key in _rj.keys(nsteps_index_key + "*"): print("Will delete:", key) _rj.delete(key) print(_rj.hlen(nsteps_map_key)) _rj.delete(nsteps_map_key) for setkey in gameset_keys: game_names_ = _rj.smembers(setkey) for _gn in game_names_: nsteps = retrieve_playthrough_nsteps(_gn, redis=_rj, redisbasekey=redisbasekey) if nsteps > 0: print(nsteps, _gn) _rj.hset(nsteps_map_key, _gn, nsteps) _rj.sadd(f"{nsteps_index_key}{nsteps}", _gn) print(len(_rj.keys(nsteps_index_key + "*")), _rj.hlen(nsteps_map_key)) total = 0 sort_list = [] for key in _rj.keys(nsteps_index_key + "*"): nsteps = int(key.split(':')[-1]) num_games = _rj.scard(key) total += num_games sort_list.append((nsteps, num_games, key)) # print(key, "has", num_games, "game names") sort_list.sort() for nsteps, num_games, setkey in sort_list: print(f"[{nsteps}]\t {num_games}\t {setkey}") if redserv is None: _rj.close()
def create_skills_map(redserv=None, skillsmap_key=REDIS_FTWC_SKILLS_MAP, gameset_keys=FTWC_GAME_SETS): """ after all game names have been added to redis, we map skills to game names""" if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: _rj = redserv skills_index = {} # maps skillname to a set of game names all_mapped_skills = set() # all game names that are in the skills map for setkey in gameset_keys: game_names = _rj.smembers(setkey) print(f"{setkey} has {len(game_names)} members") for g in game_names: gid, sklist = split_gamename(g) # print(g, gid, sklist) for skill in sklist: if skill not in skills_index: skills_index[skill] = set() skills_index[skill].add(g) _rj.sadd(skillsmap_key + skill, g) # print(len(skills_index), skills_index.keys()) # for key in skills_index.keys(): # print(key, len(skills_index[key])) # all_mapped_skills = all_mapped_skills.union(skills_index[key]) skillsmap_keys = _rj.keys(skillsmap_key + "*") for k in skillsmap_keys: print(k, _rj.scard(k)) all_mapped_skills = all_mapped_skills.union(_rj.smembers(k)) print( f"TOTAL # of game files for which skills have been mapped: {len(all_mapped_skills)}" ) if redserv is None: _rj.close()
from datetime import datetime from pprint import pprint from rejson import Client, Path rj = Client(host='localhost', port=6379, decode_responses=True) # Get single key:value pair pprint(rj.jsonget("redis_club_urls:item73", Path.rootPath())) # Convert timestamp from iso to datetime timestamp = rj.jsonget("redis_club_urls:item73", Path.rootPath())['last_modified'] timestamp = datetime.fromisoformat(timestamp) # Get all keys print(rj.keys()) # Get all values matching pattern from keys for key in rj.scan_iter("redis_club_urls:item:*"): print(rj.jsonget(key, Path('.club_page'))) # Add all values matching pattern to new key for key in rj.scan_iter("redis_club_urls:item:*"): val = rj.jsonget(key, Path('.club_page')) rj.sadd('all_urls', val)
class RedisDriver(DatabaseDriver): _host = "localhost" _port = 6379 _client = None def __init__(self, host: str = "localhost", port: int = 6379): self._host = host self._port = port self._client = Client(host=host, port=port, decode_responses=True, encoder=JSONSchemaObject.JSONSchemaEncoder()) def find_by_ref(self, ref: str): return self._client.jsonget(ref) def find_id_by(self, idx: str, value: str, version: str): result = [] for member in self._client.smembers("{}:{}".format(idx, value)): if version == "all": result.append(member) continue # we split the index to check against the version idxs = str(member).split(":") # the _version is the second token of idxs if idxs[1] == version: result.append(member) return result def save(self, obj_list: list, indexed_attrs: list): # First cycle is just to verify if we do not have any # index integrity violation for obj in indexed_attrs: # We do not store neither _id or _version if obj[1] == "_id" or obj[1] == "_version": continue if obj[2] is None or obj[2] == "": raise ValueError("Indexed value {} must not be empty".format( obj[1])) # the indexed is composed by schema path:indexes:attr_name indexed_key = store_name = "{}:indexes:{}:{}".format( obj[0], obj[1], obj[2]) # we already have this key let's get any value and make # sure we this belongs to the same id for member in self._client.smembers(indexed_key): # we only need to use one element since the _id MUST be equal idxs = str(member).split(":") # the _id is the first token of idxs, check if we recieved the same # id, if not this is a index violation if not str(obj[3]).startswith(idxs[0]): raise ValueError( "{}:{} not unique, another object already have that value" .format(obj[1], obj[2])) # we just need one iteration break # this cycle we just store the indexes for obj in indexed_attrs: if obj[2] is None or obj[2] == "" or obj[1] == "_id" or obj[ 1] == "_version": continue # Set the store name and store data store_name = "{}:indexes:{}:{}".format(obj[0], obj[1], obj[2]) store_data = obj[3] self._client.sadd(store_name, store_data) # We now store the actual objects, and return the added ids ids = [] for obj in obj_list: # Set the store name and store data store_name = "{}:{}".format(obj[0], obj[1]) store_data = obj[2] self._client.jsonset(store_name, Path.rootPath(), store_data) ids.append(obj[1]) return ids