async def receive(attr_dict): redis_path_param_default = '_d' uri = "ws://tasks.logstash:3232" async with websockets.connect(uri, ping_interval=None) as websocket: data = await websocket.recv() print(f"> {data}") # Parse this message and extract the resource-group res_data = json.loads(data) res_id = res_data['id'].replace('/', '__') print('> res_id-- ' + res_id) rg = res_id.split('__')[3] print('> rg-- ' + rg) rejson_key = rg print('> attr_dict-- ' + str(attr_dict)) # Check if the resource-group is present in the HashMap and it's attribute value if rg in attr_dict.keys(): print('> true') attribute = attr_dict[rg] redis_path_param = res_id + '_' + res_data[attribute] else: redis_path_param = res_id + redis_path_param_default # use the rejson module to insert the data into Redis db rj = Client(host='redis-server', port=6379, decode_responses=True) print('> redis_path_param-- ' + redis_path_param) redis_path_param = redis_path_param.replace('.', '$') redis_path_param = redis_path_param.replace('-', '_') #rj.jsonset(rejson_key,Path.rootPath(),json.dumps({})) rj.jsonset(rejson_key, Path('.' + redis_path_param), res_data)
def from_settings(cls, settings): kwargs = { 'persist': settings.getbool('SCHEDULER_PERSIST'), 'flush_on_start': settings.getbool('SCHEDULER_FLUSH_ON_START'), 'idle_before_close': settings.getint('SCHEDULER_IDLE_BEFORE_CLOSE'), } # If these values are missing, it means we want to use the defaults. optional = { # TODO: Use custom prefixes for this settings to note that are # specific to scrapy-redis. 'queue_key': 'SCHEDULER_QUEUE_KEY', 'queue_cls': 'SCHEDULER_QUEUE_CLASS', 'dupefilter_key': 'SCHEDULER_DUPEFILTER_KEY', # We use the default setting name to keep compatibility. 'dupefilter_cls': 'DUPEFILTER_CLASS', 'serializer': 'SCHEDULER_SERIALIZER', } for name, setting_name in optional.items(): val = settings.get(setting_name) if val: kwargs[name] = val # Support serializer as a path to a module. if isinstance(kwargs.get('serializer'), six.string_types): kwargs['serializer'] = importlib.import_module( kwargs['serializer']) server = Client(host='localhost', port=6379, decode_responses=True) # Ensure the connection is working. server.ping() return cls(server=server, **kwargs)
def __init__(self): self.cfg = read_cfg("redis") self.rj = Client(host=self.cfg.get("ip"), port=self.cfg.get("port"), decoder=RedisJsonDecoder(), decode_responses=True) self.logger = logger.myLogger("Redis")
def __init__(self, host: str = "localhost", port: int = 6379): self._host = host self._port = port self._client = Client(host=host, port=port, decode_responses=True, encoder=JSONSchemaObject.JSONSchemaEncoder())
def find_by_state(self): try: rj = Client( host='redis', port=6379, decode_responses=True) return rj.jsonget(self.name) except: return None
def __init__(self, host: str, port: Union[str, int]) -> None: """Instantiate a connection to ReJson. :param host: The hostname/ip of the Redis instance. :type host: str :param port: The port of the Redis instance. :type port: int """ self._client = Client(host=host, port=port, decode_responses=True)
def webhook_view(timestamp=None, alert_type=None, reciever=None, key_name=None): if request.method == 'GET': conn = Client(host=REDIS_SERVER, port=6379, db=0, password=REDIS_PWD) params = timestamp + '_' + alert_type + '_' + reciever data = json.dumps(conn.jsonget(params)) return render_template('info.html', data=data)
def save_simgrids_setup_data_rejson(id, setupJson, rejson_host, rejson_key, rejson_db): saveJson = {**get_default_json(), **setupJson} rj = Client(host=rejson_host, port=rejson_key, db=rejson_db) redis_key = rj.incr('simgrids_key') saveJson["id"] = redis_key json_key = "simgrids_key_{0}".format(redis_key) status = rj.jsonset(json_key, Path.rootPath(), saveJson) return redis_key, saveJson
def main(): rj = Client(host='localhost', port=6379, decode_responses=True) obj = { 'answer': 42, 'arr': [None, True, 3.14], 'truth': { 'coord': 'out there' } } rj.jsonset('obj', Path.rootPath(), obj)
def __init__(self): self.tests = ['ru.4', '5k4', 'u;4', 'ji3', '5', '2l4', 'xk7'] # no need bracket here # the redis server self.rj = Client(host='localhost', port=6379, decode_responses=True) # phonetic table, radical to han characters self.fn = 'phone.json' self.objname = 'obj' self.data = None self.check_and_store()
def create_difficulty_map(redserv=None, difficulty_map_key=REDIS_GATA_DIFFICULTY_MAP, game_dirs=None): if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: _rj = redserv if game_dirs is None: training_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TRAINING_DIR') validation_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_VALIDATION_DIR') test_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TEST_DIR') game_dirs = (training_dir, validation_dir, test_dir) print( f"create_difficulty_map({difficulty_map_key}, game_dirs={game_dirs})") for game_dir in game_dirs: print("create_difficulty_map -- GAME DIR:", game_dir) game_names_ = [] for level in range(1, 11): difficulty = f"difficulty_level_{level}" print("\n-------------------", difficulty) games_list = count_game_files(game_dir + difficulty) game_names_.extend(games_list) _rj.sadd(difficulty_map_key + str(level), *games_list) print( f"total games in {game_dir}: {len(game_names_)} {len(set(game_names_))}" ) assert len(game_names_) == len( set(game_names_)) # they should all be unique if redserv is None: _rj.close()
class Connector(): def __init__(self): self._rj = Client(host='localhost', port=6379, decode_responses=True) def saveLog(self, logJson, logId): print("saveLog") self._rj.jsonset(logId, Path.rootPath(), logJson) def getLog(self, logId): return self._rj.jsonget(logId, Path.rootPath())
def reset(self, client, rj, futures=None): """ Stop the algorithm. The algorithm will be deleted shortly after this function is called. """ reset = rj.jsonget("reset", root) logger.warning("reset=%s for %s", reset, self.ident) if not reset: return False logger.warning(f"Deleting various keys for {self.ident}") rj2 = RedisClient(host="redis", port=6379, decode_responses=False) rj2.delete(f"state-{self.ident}") rj2.delete(f"model-{self.ident}") rj.jsondel(f"alg-perf-{self.ident}", root) rj.delete(f"alg-perf-{self.ident}") # Clear answers logger.warning(f"Clearing answers for {self.ident}") self.get_answers(rj, clear=True) # Clear queries (twice) logger.warning(f"Clearing queries for {self.ident}") key = f"alg-{self.ident}-queries" for k in range(4, 18): limit = 2**k rj.zremrangebyscore(key, -limit, limit) sleep(0.1) n_queries = rj.zcard(key) logger.warning(f"n_queries={n_queries}") if not n_queries: break logger.warning(f"Clearing queries again for {self.ident}") self.clear_queries(rj) if futures: for future in futures: if future: client.cancel(future, force=True) logger.warning(f"Restarting Dask client for {self.ident}") f = client.restart(timeout="5s") try: client.sync(f) except: pass client.run(garbage_collect) logger.warning(f"Setting stopped-{self.ident}") rj.jsonset(f"stopped-{self.ident}", Path("."), True) logger.warning(f"All done stopping {self.ident}") return True
def get_max_score(gn, redserv=None): if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: assert isinstance(redserv, Client) _rj = redserv dirpath = get_dir_for_game(_rj, gn) if redserv is None: _rj.close() with open(f"{dirpath}/{gn}.json", "r") as f: metadata = json.load(f)['metadata'] return int(metadata['max_score'])
def _create_client(self, *args, **kwargs): if REDIS_CLUSTER_HOST is None or \ REDIS_CLUSTER_PORT is None: return Client(*args, **kwargs) else: from rediscluster import RedisCluster startup_nodes = [{ "host": REDIS_CLUSTER_HOST, "port": REDIS_CLUSTER_PORT }] conn = RedisCluster(startup_nodes=startup_nodes, decode_responses=True) return Client(client=conn, *args, **kwargs)
def get_answers(self, rj: RedisClient, clear: bool = True) -> List[Answer]: """ Get all answers the frontend has received. """ if not clear: raise NotImplementedError key = f"alg-{self.ident}-answers" if key in rj.keys(): pipe = rj.pipeline() pipe.jsonget(key, Path(".")) pipe.jsonset(key, Path("."), []) answers, success = pipe.execute() return answers return []
def post(self, request): validated_data = request.serializer.validated_data task = Task.objects.create(title=validated_data['title'], description=validated_data['description'], status=Task.CREATED, user_created=request.user, date_create_task=datetime.now()) if validated_data['user_assigned']: task.user_assigned = validated_data['user_assigned'] task.save() if validated_data['user_assigned'] and validated_data[ 'user_assigned'] != request.user: AddNotificationTaskStatus(task.user_assigned, task, "created") # add in ReJSON database rj = Client( host='localhost', port=6379, ) rj.jsonset('task:' + str(task.id), Path.rootPath(), TaskSerializer(task).data) rj.execute_command('JSON.NUMINCRBY acc .total 1') rj.execute_command('JSON.SET acc .maxId ' + str(task.id)) return Response(status=201)
def main(): rj = Client(host='localhost', port=6379, decode_responses=True) obj = { 'answer': 42, 'arr': [None, True, 3.14], 'truth': { 'coord': 'out there' } } rj.jsonset('obj', Path.rootPath(), obj) # Get something print 'Is there anybody... {}?'.format( rj.jsonget('obj', Path('.truth.coord')))
async def get_model(sampler: str): samplers = rj.jsonget("samplers") if sampler not in samplers: raise ServerException( f"Can't find model for sampler='{sampler}'. " f"Valid choices for sampler are {samplers}" ) if f"model-{sampler}" not in rj.keys(): logger.warning("rj.keys() = %s", rj.keys()) flush_logger(logger) raise ServerException(f"Model has not been created for sampler='{sampler}'") rj2 = Client(host="redis", port=6379, decode_responses=False) ir = rj2.get(f"model-{sampler}") model = cloudpickle.loads(ir) return model
def lookup_difficulty_level(gamename, redserv=None, difficulty_map_key=REDIS_GATA_DIFFICULTY_MAP): # assert gamename , "Missing required argument: gamename" level = -1 if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: _rj = redserv for i in range(1, 11): if _rj.sismember(difficulty_map_key + str(i), gamename): level = i break if redserv is None: _rj.close() return level
def main(action, filepath, search): rj = Client(host='localhost', port=6379, decode_responses=True) # update your redis settings if action == 'dump': out = {} for key in r.scan_iter(search): out.update({key: r.get(key)}) if len(out) > 0: try: with open(filepath, 'w') as outfile: json.dump(out, outfile) print('Dump Successful') except Exception as e: print(e) else: print("Keys not found") elif action == 'load': try: print('here......') usr = r.json.get('user') print(usr) except Exception as e: print('here is the error: ' + str(e))
def select_games_for_mingpt(redserv=None): if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: _rj = redserv # for n in range(70): # rkey = f"{REDIS_FTWC_NSTEPS_INDEX}{n}" # if rj.exists(rkey): # n_nsteps = rj.scard(f"{REDIS_FTWC_NSTEPS_INDEX}{n}") # n_nodrop = rj.sdiff(f"{REDIS_FTWC_NSTEPS_INDEX}{n}", "ftwc:cog2019:skills:drop") # # maxtoks, mintoks = get_max_token_length(n_nodrop) # if n <= 32 and len(n_nodrop): # rj.sadd(REDIS_MINGPT_ALL, *n_nodrop) # print(f"{n:2d}:__{n_nsteps:3d}__{len(n_nodrop):3d}") #" {maxtoks:4d} {mintoks:4d}") if redserv is None: _rj.close()
def get_instance(self): if not RedisClient.SINGLETON_INSTANCE: RedisClient.SINGLETON_INSTANCE = Client( host=os.environ['redis_host'], port=os.environ['redis_port'], decode_responses=True, password=os.environ['redis_password']) return RedisClient.SINGLETON_INSTANCE
class Database: """Database handler""" def __init__(self, database_name, port=6379): self.database = Client(host=database_name, port=port, decode_responses=True) def flush(self): """Flushing a certain table of the database""" return self.database.flushdb() def import_data(self, keyname, data): """Load data into the database""" return self.database.jsonset(keyname, Path.rootPath(), data) def ping(self): """Tests the connection""" return self.database.ping()
def pull_from_redis(keys, client=Client(decode_responses=True), path=Path.rootPath()): insert_queue = list() for key in client.scan_iter(keys): insert_queue.append(client.jsonget(key, path)) return insert_queue
def create_skills_map(redserv=None, skillsmap_key=REDIS_FTWC_SKILLS_MAP, gameset_keys=FTWC_GAME_SETS): """ after all game names have been added to redis, we map skills to game names""" if redserv is None: _rj = Client(host='localhost', port=6379, decode_responses=True) else: _rj = redserv skills_index = {} # maps skillname to a set of game names all_mapped_skills = set() # all game names that are in the skills map for setkey in gameset_keys: game_names = _rj.smembers(setkey) print(f"{setkey} has {len(game_names)} members") for g in game_names: gid, sklist = split_gamename(g) # print(g, gid, sklist) for skill in sklist: if skill not in skills_index: skills_index[skill] = set() skills_index[skill].add(g) _rj.sadd(skillsmap_key + skill, g) # print(len(skills_index), skills_index.keys()) # for key in skills_index.keys(): # print(key, len(skills_index[key])) # all_mapped_skills = all_mapped_skills.union(skills_index[key]) skillsmap_keys = _rj.keys(skillsmap_key + "*") for k in skillsmap_keys: print(k, _rj.scard(k)) all_mapped_skills = all_mapped_skills.union(_rj.smembers(k)) print( f"TOTAL # of game files for which skills have been mapped: {len(all_mapped_skills)}" ) if redserv is None: _rj.close()
def testCustomEncoderDecoderShouldSucceed(self): "Test a custom encoder and decoder" class CustomClass(object): key = '' val = '' def __init__(self, k='', v=''): self.key = k self.val = v class TestEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, CustomClass): return 'CustomClass:{}:{}'.format(obj.key, obj.val) return json.JSONEncoder.encode(self, obj) class TestDecoder(json.JSONDecoder): def decode(self, obj): d = json.JSONDecoder.decode(self, obj) if isinstance(d, six.string_types) and \ d.startswith('CustomClass:'): s = d.split(':') return CustomClass(k=s[1], v=s[2]) return d rj = Client(encoder=TestEncoder(), decoder=TestDecoder(), port=port, decode_responses=True) rj.flushdb() # Check a regular string self.assertTrue(rj.jsonset('foo', Path.rootPath(), 'bar')) self.assertEqual('string', rj.jsontype('foo', Path.rootPath())) self.assertEqual('bar', rj.jsonget('foo', Path.rootPath())) # Check the custom encoder self.assertTrue( rj.jsonset('cus', Path.rootPath(), CustomClass('foo', 'bar'))) obj = rj.jsonget('cus', Path.rootPath()) self.assertIsNotNone(obj) self.assertEqual(CustomClass, obj.__class__) self.assertEqual('foo', obj.key) self.assertEqual('bar', obj.val)
def webhook_save(): if request.method == 'POST': timestamp_human = datetime.datetime.now() timestamp = int(time.time()) nowDatetime = timestamp_human.strftime('%Y-%m-%d(%H:%M:%S)') req_data = request.get_json() alertname = req_data['commonLabels']['alertname'] severity = '' receiver = req_data['receiver'] key_name = str(timestamp) + "_" + alertname + "_" + receiver try: # conn = redis.Redis(host=REDIS_SERVER, port=6379, db=0, password=REDIS_PWD) conn = Client(host=REDIS_SERVER, port=6379, db=0, password=REDIS_PWD) conn.ping() print 'Redis connected %s' % (REDIS_SERVER) except Exception as e: print 'Error:', e exit('Failed to connecting') conn = Client(host=REDIS_SERVER, port=6379) conn.jsonset(key_name, Path.rootPath(), req_data) data = json.dumps(conn.jsonget(key_name)) print data # Redis : SCAN 0 match 1527911[1-9][1-9]* else: abort(400) if not conn.exists(key_name): print "Error: %s is doesn't exist" % (key_name) return jsonify({'status': 'success'}), 200
class RedisClient: def __init__(self): logger.debug(os.environ.get("REDIS_HOST")) self.client = Client(host=os.environ.get("REDIS_HOST", "localhost"), decode_responses=True) def getAnyShape(self, index): try: shape = self.client.jsonget(index) return shape except Exception as e: logger.warning('Fail to get the {} shape from Redis {}'.format( index, e))
def from_settings(cls, settings): params = { 'server': Client(host='localhost', port=6379, decode_responses=True), } if settings.get('REDIS_ITEMS_KEY'): params['key'] = settings['REDIS_ITEMS_KEY'] if settings.get('REDIS_ITEMS_SERIALIZER'): params['serialize_func'] = load_object( settings['REDIS_ITEMS_SERIALIZER']) return cls(**params)