def create_nsteps_map(redserv=None,
nsteps_map_key=REDIS_FTWC_NSTEPS_MAP,
nsteps_index_key=REDIS_FTWC_NSTEPS_INDEX,
gameset_keys=FTWC_GAME_SETS):
""" after all playthroughs have been save to redis, index number of steps <-> game names """
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
if nsteps_map_key == REDIS_GATA_NSTEPS_MAP:
redisbasekey = REDIS_GATA_PLAYTHROUGHS
elif nsteps_map_key == REDIS_FTWC_NSTEPS_MAP:
redisbasekey = REDIS_FTWC_PLAYTHROUGHS
else:
assert False, "Unknown Redis nsteps_map_key " + nsteps_map_key
for key in _rj.keys(nsteps_index_key + "*"):
print("Will delete:", key)
_rj.delete(key)
print(_rj.hlen(nsteps_map_key))
_rj.delete(nsteps_map_key)
for setkey in gameset_keys:
game_names_ = _rj.smembers(setkey)
for _gn in game_names_:
nsteps = retrieve_playthrough_nsteps(_gn,
redis=_rj,
redisbasekey=redisbasekey)
if nsteps > 0:
print(nsteps, _gn)
_rj.hset(nsteps_map_key, _gn, nsteps)
_rj.sadd(f"{nsteps_index_key}{nsteps}", _gn)
print(len(_rj.keys(nsteps_index_key + "*")), _rj.hlen(nsteps_map_key))
total = 0
sort_list = []
for key in _rj.keys(nsteps_index_key + "*"):
nsteps = int(key.split(':')[-1])
num_games = _rj.scard(key)
total += num_games
sort_list.append((nsteps, num_games, key))
# print(key, "has", num_games, "game names")
sort_list.sort()
for nsteps, num_games, setkey in sort_list:
print(f"[{nsteps}]\t {num_games}\t {setkey}")
if redserv is None:
_rj.close()
def get_answers(self, rj: RedisClient, clear: bool = True) -> List[Answer]:
"""
Get all answers the frontend has received.
"""
if not clear:
raise NotImplementedError
key = f"alg-{self.ident}-answers"
if key in rj.keys():
pipe = rj.pipeline()
pipe.jsonget(key, Path("."))
pipe.jsonset(key, Path("."), [])
answers, success = pipe.execute()
return answers
return []
def create_skills_map(redserv=None,
skillsmap_key=REDIS_FTWC_SKILLS_MAP,
gameset_keys=FTWC_GAME_SETS):
""" after all game names have been added to redis, we map skills to game names"""
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
skills_index = {} # maps skillname to a set of game names
all_mapped_skills = set() # all game names that are in the skills map
for setkey in gameset_keys:
game_names = _rj.smembers(setkey)
print(f"{setkey} has {len(game_names)} members")
for g in game_names:
gid, sklist = split_gamename(g)
# print(g, gid, sklist)
for skill in sklist:
if skill not in skills_index:
skills_index[skill] = set()
skills_index[skill].add(g)
_rj.sadd(skillsmap_key + skill, g)
# print(len(skills_index), skills_index.keys())
# for key in skills_index.keys():
# print(key, len(skills_index[key]))
# all_mapped_skills = all_mapped_skills.union(skills_index[key])
skillsmap_keys = _rj.keys(skillsmap_key + "*")
for k in skillsmap_keys:
print(k, _rj.scard(k))
all_mapped_skills = all_mapped_skills.union(_rj.smembers(k))
print(
f"TOTAL # of game files for which skills have been mapped: {len(all_mapped_skills)}"
)
if redserv is None:
_rj.close()
class ReJson:
"""Facade for ReJson"""
def __init__(self, host: str, port: Union[str, int]) -> None:
"""Instantiate a connection to ReJson.
:param host: The hostname/ip of the Redis instance.
:type host: str
:param port: The port of the Redis instance.
:type port: int
"""
self._client = Client(host=host, port=port, decode_responses=True)
def keys(self) -> Json:
"""Get all keys"""
return self._client.keys()
def post(self, key: str, obj: Json) -> None:
"""Post a new Json object to the store.
:param key: The key to store the Json at.
:type key: str
:param obj: What to store.
:type obj: Json
"""
self._client.jsonset(key, Path.rootPath(), obj)
def get(self, key: str) -> Json:
"""[summary]
:param key: The key that the Json object was stored at.
:type key: str
:return: The Json stored at `key`.
:rtype: Json
"""
return self._client.jsonget(key, Path.rootPath())
def update(self, key: str, path: str, value: Json) -> None:
"""[summary]
:param key: The key that the Json object was stored at.
:type key: str
:param path: A period seperated string of keys to traverse the Json.
:type path: str
:param value: The new value.
:type value: Json
"""
self._client.jsonset(key, Path(f".{path}"), value)
def append(self, key: str, path: str, *values: Json) -> None:
"""Append to some array within a Json obejct.
:param key: The key that the Json object was stored at.
:type key: str
:param path: A period seperated string of keys to traverse the Json.
:type path: str
"""
self._client.jsonarrappend(key, Path(f".{path}"), *values)
def pop(self, key: str, path: str) -> Json:
"""Pop from from array within a Json object.
:param key: The key that the Json object was stored at.
:type key: str
:param path: A period seperated string of keys to traverse the Json.
:type path: str
:return: The Json value popped from the array.
:rtype: Json
"""
return self._client.jsonarrpop(key, f".{path}")
def remove(self, key: str, path: str, value: Json) -> None:
"""Remove something from some array within a Json object.
:param key: The key that the Json object was stored at.
:type key: str
:param path: A period seperated string of keys to travers the Json.
:type path: str
:param value: The value to remove from the array.
:type value: Json
"""
index = self._client.jsonarrindex(key, f".{path}", value)
self._client.jsondel(key, f"{path}[{index}]")
from datetime import datetime
from pprint import pprint
from rejson import Client, Path
rj = Client(host='localhost', port=6379, decode_responses=True)
# Get single key:value pair
pprint(rj.jsonget("redis_club_urls:item73", Path.rootPath()))
# Convert timestamp from iso to datetime
timestamp = rj.jsonget("redis_club_urls:item73",
Path.rootPath())['last_modified']
timestamp = datetime.fromisoformat(timestamp)
# Get all keys
print(rj.keys())
# Get all values matching pattern from keys
for key in rj.scan_iter("redis_club_urls:item:*"):
print(rj.jsonget(key, Path('.club_page')))
# Add all values matching pattern to new key
for key in rj.scan_iter("redis_club_urls:item:*"):
val = rj.jsonget(key, Path('.club_page'))
rj.sadd('all_urls', val)
class JobsAPI(Resource):
def __init__(self):
self.redis = Client(host='127.0.0.1', port=6379, decode_responses=True)
def get(self, **kwargs):
if kwargs.get('job_id'):
job_id = kwargs.get('job_id')
if self.redis.exists(job_id):
parser = reqparse.RequestParser()
if request.url_rule.rule == '/jobs/<string:job_id>/next':
parser.add_argument('expired_duration',
type=int,
default=300)
args = parser.parse_args(strict=True)
if self.redis.jsonget(job_id, Path('.items')):
ttl = args.get('expired_duration')
items = self.redis.jsonget(job_id, Path('.items'))
for item in items:
if not self.redis.exists(f'hold_{item}'):
self.redis.execute_command(
'SET', f'hold_{item}', job_id)
self.redis.execute_command(
'EXPIRE', f'hold_{item}', ttl)
return output_json(
{
'status': 'ok',
'job_id': job_id,
'ttl': ttl,
'index': items.index(item),
'item': item
}, 200)
return output_json(
{
'status': 'error',
'job_id': job_id,
'description': 'Items list is empty.'
}, 400)
if request.url_rule.rule == '/jobs/<string:job_id>/items':
parser.add_argument('active',
default='true',
choices=('true', 'false'))
args = parser.parse_args(strict=True)
items = self.redis.jsonget(job_id, Path('.items'))
done_items = self.redis.jsonget(job_id, Path('.done'))
if args.get('active') == 'true':
active_items = []
for item in items:
if not self.redis.exists(f'hold_{item}') and \
items.index(item) not in done_items:
active_items.append(item)
return output_json(
{
'status': 'ok',
'job_id': job_id,
'items': active_items
}, 200)
return output_json(
{
'status': 'ok',
'job_id': job_id,
'items': items + done_items
}, 200)
else:
return output_json(
{
'status': 'error',
'job_id': job_id,
'description': 'The job is not in the queue.'
}, 400)
return output_json(
{
'status': 'ok',
'jobs': [i for i in self.redis.keys() if i[:5] != 'hold_']
}, 200)
def post(self, **kwargs):
if request.url_rule.rule == '/jobs/<string:job_id>/items/<int:item_index>/done':
job_id = kwargs.get('job_id')
item_index = kwargs.get('item_index')
done_item = self.redis.jsonget(job_id, Path('.items'))[item_index]
if item_index in self.redis.jsonget(job_id, Path('.done')):
return output_json(
{
'status': 'error',
'description': 'The item already was marked as done.',
'job_id': job_id,
'index': item_index,
'item': done_item
}, 400)
self.redis.delete(f'hold_{done_item}')
self.redis.jsonarrappend(job_id, Path('.done'), item_index)
return output_json(
{
'status': 'ok',
'description': 'The item is marked as done.',
'job_id': job_id,
'index': item_index,
'item': done_item
}, 200)
if request.url_rule.rule == '/jobs/<string:job_id>/items/<int:item_index>/error':
job_id = kwargs.get('job_id')
item_index = kwargs.get('item_index')
error_item = self.redis.jsonget(job_id, Path('.items'))[item_index]
if item_index in self.redis.jsonget(job_id, Path('.error')):
return output_json(
{
'status': 'error',
'description': 'The item already was marked as error.',
'job_id': job_id,
'index': item_index,
'item': error_item
}, 400)
self.redis.delete(f'hold_{error_item}')
self.redis.jsonarrappend(job_id, Path('.error'), item_index)
return output_json(
{
'status': 'ok',
'description': 'The item is marked as error.',
'job_id': job_id,
'index': item_index,
'item': error_item
}, 200)
if isinstance(request.json, list) and request.json:
job_id = str(uuid.uuid4())
data = {'items': request.json, 'done': [], 'error': []}
if self.redis.jsonset(job_id, Path.rootPath(), data):
return output_json(
{
'status': 'ok',
'description': 'Job is added to queue.',
'job_id': job_id
}, 201)
else:
return output_json(
{
'status': 'error',
'description': 'Wrong request!'
}, 400)
def delete(self, job_id):
if self.redis.exists(job_id):
self.redis.delete(job_id)
return output_json(
{
'status': 'ok',
'description': 'Job is deleted.'
}, 200)
else:
return output_json(
{
'status': 'error',
'description': 'The job is not in the queue.'
}, 400)