def reset(self, client, rj, futures=None): """ Stop the algorithm. The algorithm will be deleted shortly after this function is called. """ reset = rj.jsonget("reset", root) logger.warning("reset=%s for %s", reset, self.ident) if not reset: return False logger.warning(f"Deleting various keys for {self.ident}") rj2 = RedisClient(host="redis", port=6379, decode_responses=False) rj2.delete(f"state-{self.ident}") rj2.delete(f"model-{self.ident}") rj.jsondel(f"alg-perf-{self.ident}", root) rj.delete(f"alg-perf-{self.ident}") # Clear answers logger.warning(f"Clearing answers for {self.ident}") self.get_answers(rj, clear=True) # Clear queries (twice) logger.warning(f"Clearing queries for {self.ident}") key = f"alg-{self.ident}-queries" for k in range(4, 18): limit = 2**k rj.zremrangebyscore(key, -limit, limit) sleep(0.1) n_queries = rj.zcard(key) logger.warning(f"n_queries={n_queries}") if not n_queries: break logger.warning(f"Clearing queries again for {self.ident}") self.clear_queries(rj) if futures: for future in futures: if future: client.cancel(future, force=True) logger.warning(f"Restarting Dask client for {self.ident}") f = client.restart(timeout="5s") try: client.sync(f) except: pass client.run(garbage_collect) logger.warning(f"Setting stopped-{self.ident}") rj.jsonset(f"stopped-{self.ident}", Path("."), True) logger.warning(f"All done stopping {self.ident}") return True
def create_nsteps_map(redserv=None, nsteps_map_key=REDIS_FTWC_NSTEPS_MAP, nsteps_index_key=REDIS_FTWC_NSTEPS_INDEX, gameset_keys=FTWC_GAME_SETS): """ after all playthroughs have been save to redis, index number of steps <-> game names """ if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: _rj = redserv if nsteps_map_key == REDIS_GATA_NSTEPS_MAP: redisbasekey = REDIS_GATA_PLAYTHROUGHS elif nsteps_map_key == REDIS_FTWC_NSTEPS_MAP: redisbasekey = REDIS_FTWC_PLAYTHROUGHS else: assert False, "Unknown Redis nsteps_map_key " + nsteps_map_key for key in _rj.keys(nsteps_index_key + "*"): print("Will delete:", key) _rj.delete(key) print(_rj.hlen(nsteps_map_key)) _rj.delete(nsteps_map_key) for setkey in gameset_keys: game_names_ = _rj.smembers(setkey) for _gn in game_names_: nsteps = retrieve_playthrough_nsteps(_gn, redis=_rj, redisbasekey=redisbasekey) if nsteps > 0: print(nsteps, _gn) _rj.hset(nsteps_map_key, _gn, nsteps) _rj.sadd(f"{nsteps_index_key}{nsteps}", _gn) print(len(_rj.keys(nsteps_index_key + "*")), _rj.hlen(nsteps_map_key)) total = 0 sort_list = [] for key in _rj.keys(nsteps_index_key + "*"): nsteps = int(key.split(':')[-1]) num_games = _rj.scard(key) total += num_games sort_list.append((nsteps, num_games, key)) # print(key, "has", num_games, "game names") sort_list.sort() for nsteps, num_games, setkey in sort_list: print(f"[{nsteps}]\t {num_games}\t {setkey}") if redserv is None: _rj.close()
def clear_queries(self, rj: RedisClient) -> bool: """ Clear all queries that this sampler has posted from the database. """ rj.delete(f"alg-{self.ident}-queries") return True
class JobsAPI(Resource): def __init__(self): self.redis = Client(host='127.0.0.1', port=6379, decode_responses=True) def get(self, **kwargs): if kwargs.get('job_id'): job_id = kwargs.get('job_id') if self.redis.exists(job_id): parser = reqparse.RequestParser() if request.url_rule.rule == '/jobs/<string:job_id>/next': parser.add_argument('expired_duration', type=int, default=300) args = parser.parse_args(strict=True) if self.redis.jsonget(job_id, Path('.items')): ttl = args.get('expired_duration') items = self.redis.jsonget(job_id, Path('.items')) for item in items: if not self.redis.exists(f'hold_{item}'): self.redis.execute_command( 'SET', f'hold_{item}', job_id) self.redis.execute_command( 'EXPIRE', f'hold_{item}', ttl) return output_json( { 'status': 'ok', 'job_id': job_id, 'ttl': ttl, 'index': items.index(item), 'item': item }, 200) return output_json( { 'status': 'error', 'job_id': job_id, 'description': 'Items list is empty.' }, 400) if request.url_rule.rule == '/jobs/<string:job_id>/items': parser.add_argument('active', default='true', choices=('true', 'false')) args = parser.parse_args(strict=True) items = self.redis.jsonget(job_id, Path('.items')) done_items = self.redis.jsonget(job_id, Path('.done')) if args.get('active') == 'true': active_items = [] for item in items: if not self.redis.exists(f'hold_{item}') and \ items.index(item) not in done_items: active_items.append(item) return output_json( { 'status': 'ok', 'job_id': job_id, 'items': active_items }, 200) return output_json( { 'status': 'ok', 'job_id': job_id, 'items': items + done_items }, 200) else: return output_json( { 'status': 'error', 'job_id': job_id, 'description': 'The job is not in the queue.' }, 400) return output_json( { 'status': 'ok', 'jobs': [i for i in self.redis.keys() if i[:5] != 'hold_'] }, 200) def post(self, **kwargs): if request.url_rule.rule == '/jobs/<string:job_id>/items/<int:item_index>/done': job_id = kwargs.get('job_id') item_index = kwargs.get('item_index') done_item = self.redis.jsonget(job_id, Path('.items'))[item_index] if item_index in self.redis.jsonget(job_id, Path('.done')): return output_json( { 'status': 'error', 'description': 'The item already was marked as done.', 'job_id': job_id, 'index': item_index, 'item': done_item }, 400) self.redis.delete(f'hold_{done_item}') self.redis.jsonarrappend(job_id, Path('.done'), item_index) return output_json( { 'status': 'ok', 'description': 'The item is marked as done.', 'job_id': job_id, 'index': item_index, 'item': done_item }, 200) if request.url_rule.rule == '/jobs/<string:job_id>/items/<int:item_index>/error': job_id = kwargs.get('job_id') item_index = kwargs.get('item_index') error_item = self.redis.jsonget(job_id, Path('.items'))[item_index] if item_index in self.redis.jsonget(job_id, Path('.error')): return output_json( { 'status': 'error', 'description': 'The item already was marked as error.', 'job_id': job_id, 'index': item_index, 'item': error_item }, 400) self.redis.delete(f'hold_{error_item}') self.redis.jsonarrappend(job_id, Path('.error'), item_index) return output_json( { 'status': 'ok', 'description': 'The item is marked as error.', 'job_id': job_id, 'index': item_index, 'item': error_item }, 200) if isinstance(request.json, list) and request.json: job_id = str(uuid.uuid4()) data = {'items': request.json, 'done': [], 'error': []} if self.redis.jsonset(job_id, Path.rootPath(), data): return output_json( { 'status': 'ok', 'description': 'Job is added to queue.', 'job_id': job_id }, 201) else: return output_json( { 'status': 'error', 'description': 'Wrong request!' }, 400) def delete(self, job_id): if self.redis.exists(job_id): self.redis.delete(job_id) return output_json( { 'status': 'ok', 'description': 'Job is deleted.' }, 200) else: return output_json( { 'status': 'error', 'description': 'The job is not in the queue.' }, 400)
def _reset(timeout: float = 5): _save(rj) # Stop background jobs (ie adaptive algs) rj.jsonset("reset", root, True) rj2 = Client(host="redis", port=6379, decode_responses=False) if "samplers" in rj.keys(): samplers = rj.jsonget("samplers") stopped = {name: False for name in samplers} __deadline = time() + timeout for k in itertools.count(): rj.jsonset("reset", root, True) for name in stopped: if f"stopped-{name}" in rj.keys(): stopped[name] = rj.jsonget(f"stopped-{name}", root) if all(stopped.values()): logger.warning(f"stopped={stopped}") break sleep(1) logger.warning( f"Waited {k + 1} seconds algorithms... stopped? {stopped}" f" (rj.keys() == {rj.keys()}" ) if timeout and time() >= __deadline: logger.warning(f"Hit timeout={timeout} w/ stopped={stopped}. Breaking!") break logger.warning(" starting with clearing queries...") for ident in samplers: rj2.delete(f"alg-{ident}-queries") httpx.post(f"http://localhost:8400/reset/") logger.warning("Trying to completely flush database...") for _rj in [rj, rj2]: _rj.memory_purge() _rj.flushall(asynchronous=True) _rj.flushdb(asynchronous=True) _rj.memory_purge() for k in _rj.keys(): _rj.delete(k) _rj.flushdb(asynchronous=False) _rj.flushall(asynchronous=False) now = datetime.now().isoformat()[: 10 + 6] save_dir = ROOT_DIR / "out" files = [f.name for f in save_dir.glob("*")] logger.warning(f"dump_rdb in files? {'dump.rdb' in files}") if "dump.rdb" in files: logger.warning(f"Moving dump.rdb to dump-{now}.rdb") shutil.move(str(save_dir / "dump.rdb"), str(save_dir / f"dump-{now}.rdb")) files = [f.name for f in save_dir.glob("*")] logger.warning(f"after moving, dump_rdb in files? {'dump.rdb' in files}") files = [f.name for f in save_dir.glob("*")] assert "dump.rdb" not in files logger.warning("After reset, rj.keys=%s", rj.keys()) rj.jsonset("responses", root, {}) rj.jsonset("start_time", root, -1) rj.jsonset("start_datetime", root, "-1") rj.jsonset("exp_config", root, {}) return {"success": True}