async def receive(attr_dict):
redis_path_param_default = '_d'
uri = "ws://tasks.logstash:3232"
async with websockets.connect(uri, ping_interval=None) as websocket:
data = await websocket.recv()
print(f"> {data}")
# Parse this message and extract the resource-group
res_data = json.loads(data)
res_id = res_data['id'].replace('/', '__')
print('> res_id-- ' + res_id)
rg = res_id.split('__')[3]
print('> rg-- ' + rg)
rejson_key = rg
print('> attr_dict-- ' + str(attr_dict))
# Check if the resource-group is present in the HashMap and it's attribute value
if rg in attr_dict.keys():
print('> true')
attribute = attr_dict[rg]
redis_path_param = res_id + '_' + res_data[attribute]
else:
redis_path_param = res_id + redis_path_param_default
# use the rejson module to insert the data into Redis db
rj = Client(host='redis-server', port=6379, decode_responses=True)
print('> redis_path_param-- ' + redis_path_param)
redis_path_param = redis_path_param.replace('.', '$')
redis_path_param = redis_path_param.replace('-', '_')
#rj.jsonset(rejson_key,Path.rootPath(),json.dumps({}))
rj.jsonset(rejson_key, Path('.' + redis_path_param), res_data)
def from_settings(cls, settings):
kwargs = {
'persist': settings.getbool('SCHEDULER_PERSIST'),
'flush_on_start': settings.getbool('SCHEDULER_FLUSH_ON_START'),
'idle_before_close':
settings.getint('SCHEDULER_IDLE_BEFORE_CLOSE'),
}
# If these values are missing, it means we want to use the defaults.
optional = {
# TODO: Use custom prefixes for this settings to note that are
# specific to scrapy-redis.
'queue_key': 'SCHEDULER_QUEUE_KEY',
'queue_cls': 'SCHEDULER_QUEUE_CLASS',
'dupefilter_key': 'SCHEDULER_DUPEFILTER_KEY',
# We use the default setting name to keep compatibility.
'dupefilter_cls': 'DUPEFILTER_CLASS',
'serializer': 'SCHEDULER_SERIALIZER',
}
for name, setting_name in optional.items():
val = settings.get(setting_name)
if val:
kwargs[name] = val
# Support serializer as a path to a module.
if isinstance(kwargs.get('serializer'), six.string_types):
kwargs['serializer'] = importlib.import_module(
kwargs['serializer'])
server = Client(host='localhost', port=6379, decode_responses=True)
# Ensure the connection is working.
server.ping()
return cls(server=server, **kwargs)
def __init__(self):
self.cfg = read_cfg("redis")
self.rj = Client(host=self.cfg.get("ip"),
port=self.cfg.get("port"),
decoder=RedisJsonDecoder(),
decode_responses=True)
self.logger = logger.myLogger("Redis")
def __init__(self, host: str = "localhost", port: int = 6379):
self._host = host
self._port = port
self._client = Client(host=host,
port=port,
decode_responses=True,
encoder=JSONSchemaObject.JSONSchemaEncoder())
def find_by_state(self):
try:
rj = Client(
host='redis',
port=6379,
decode_responses=True)
return rj.jsonget(self.name)
except:
return None
def __init__(self, host: str, port: Union[str, int]) -> None:
"""Instantiate a connection to ReJson.
:param host: The hostname/ip of the Redis instance.
:type host: str
:param port: The port of the Redis instance.
:type port: int
"""
self._client = Client(host=host, port=port, decode_responses=True)
def webhook_view(timestamp=None,
alert_type=None,
reciever=None,
key_name=None):
if request.method == 'GET':
conn = Client(host=REDIS_SERVER, port=6379, db=0, password=REDIS_PWD)
params = timestamp + '_' + alert_type + '_' + reciever
data = json.dumps(conn.jsonget(params))
return render_template('info.html', data=data)
def save_simgrids_setup_data_rejson(id, setupJson, rejson_host, rejson_key,
rejson_db):
saveJson = {**get_default_json(), **setupJson}
rj = Client(host=rejson_host, port=rejson_key, db=rejson_db)
redis_key = rj.incr('simgrids_key')
saveJson["id"] = redis_key
json_key = "simgrids_key_{0}".format(redis_key)
status = rj.jsonset(json_key, Path.rootPath(), saveJson)
return redis_key, saveJson
def main():
rj = Client(host='localhost', port=6379, decode_responses=True)
obj = {
'answer': 42,
'arr': [None, True, 3.14],
'truth': {
'coord': 'out there'
}
}
rj.jsonset('obj', Path.rootPath(), obj)
def __init__(self):
self.tests = ['ru.4', '5k4', 'u;4', 'ji3', '5', '2l4',
'xk7'] # no need bracket here
# the redis server
self.rj = Client(host='localhost', port=6379, decode_responses=True)
# phonetic table, radical to han characters
self.fn = 'phone.json'
self.objname = 'obj'
self.data = None
self.check_and_store()
def create_difficulty_map(redserv=None,
difficulty_map_key=REDIS_GATA_DIFFICULTY_MAP,
game_dirs=None):
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
if game_dirs is None:
training_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TRAINING_DIR')
validation_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_VALIDATION_DIR')
test_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TEST_DIR')
game_dirs = (training_dir, validation_dir, test_dir)
print(
f"create_difficulty_map({difficulty_map_key}, game_dirs={game_dirs})")
for game_dir in game_dirs:
print("create_difficulty_map -- GAME DIR:", game_dir)
game_names_ = []
for level in range(1, 11):
difficulty = f"difficulty_level_{level}"
print("\n-------------------", difficulty)
games_list = count_game_files(game_dir + difficulty)
game_names_.extend(games_list)
_rj.sadd(difficulty_map_key + str(level), *games_list)
print(
f"total games in {game_dir}: {len(game_names_)} {len(set(game_names_))}"
)
assert len(game_names_) == len(
set(game_names_)) # they should all be unique
if redserv is None:
_rj.close()
class Connector():
def __init__(self):
self._rj = Client(host='localhost', port=6379, decode_responses=True)
def saveLog(self, logJson, logId):
print("saveLog")
self._rj.jsonset(logId, Path.rootPath(), logJson)
def getLog(self, logId):
return self._rj.jsonget(logId, Path.rootPath())
def reset(self, client, rj, futures=None):
"""
Stop the algorithm. The algorithm will be deleted shortly after
this function is called.
"""
reset = rj.jsonget("reset", root)
logger.warning("reset=%s for %s", reset, self.ident)
if not reset:
return False
logger.warning(f"Deleting various keys for {self.ident}")
rj2 = RedisClient(host="redis", port=6379, decode_responses=False)
rj2.delete(f"state-{self.ident}")
rj2.delete(f"model-{self.ident}")
rj.jsondel(f"alg-perf-{self.ident}", root)
rj.delete(f"alg-perf-{self.ident}")
# Clear answers
logger.warning(f"Clearing answers for {self.ident}")
self.get_answers(rj, clear=True)
# Clear queries (twice)
logger.warning(f"Clearing queries for {self.ident}")
key = f"alg-{self.ident}-queries"
for k in range(4, 18):
limit = 2**k
rj.zremrangebyscore(key, -limit, limit)
sleep(0.1)
n_queries = rj.zcard(key)
logger.warning(f"n_queries={n_queries}")
if not n_queries:
break
logger.warning(f"Clearing queries again for {self.ident}")
self.clear_queries(rj)
if futures:
for future in futures:
if future:
client.cancel(future, force=True)
logger.warning(f"Restarting Dask client for {self.ident}")
f = client.restart(timeout="5s")
try:
client.sync(f)
except:
pass
client.run(garbage_collect)
logger.warning(f"Setting stopped-{self.ident}")
rj.jsonset(f"stopped-{self.ident}", Path("."), True)
logger.warning(f"All done stopping {self.ident}")
return True
def get_max_score(gn, redserv=None):
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
assert isinstance(redserv, Client)
_rj = redserv
dirpath = get_dir_for_game(_rj, gn)
if redserv is None:
_rj.close()
with open(f"{dirpath}/{gn}.json", "r") as f:
metadata = json.load(f)['metadata']
return int(metadata['max_score'])
def _create_client(self, *args, **kwargs):
if REDIS_CLUSTER_HOST is None or \
REDIS_CLUSTER_PORT is None:
return Client(*args, **kwargs)
else:
from rediscluster import RedisCluster
startup_nodes = [{
"host": REDIS_CLUSTER_HOST,
"port": REDIS_CLUSTER_PORT
}]
conn = RedisCluster(startup_nodes=startup_nodes,
decode_responses=True)
return Client(client=conn, *args, **kwargs)
def get_answers(self, rj: RedisClient, clear: bool = True) -> List[Answer]:
"""
Get all answers the frontend has received.
"""
if not clear:
raise NotImplementedError
key = f"alg-{self.ident}-answers"
if key in rj.keys():
pipe = rj.pipeline()
pipe.jsonget(key, Path("."))
pipe.jsonset(key, Path("."), [])
answers, success = pipe.execute()
return answers
return []
def post(self, request):
validated_data = request.serializer.validated_data
task = Task.objects.create(title=validated_data['title'],
description=validated_data['description'],
status=Task.CREATED,
user_created=request.user,
date_create_task=datetime.now())
if validated_data['user_assigned']:
task.user_assigned = validated_data['user_assigned']
task.save()
if validated_data['user_assigned'] and validated_data[
'user_assigned'] != request.user:
AddNotificationTaskStatus(task.user_assigned, task, "created")
# add in ReJSON database
rj = Client(
host='localhost',
port=6379,
)
rj.jsonset('task:' + str(task.id), Path.rootPath(),
TaskSerializer(task).data)
rj.execute_command('JSON.NUMINCRBY acc .total 1')
rj.execute_command('JSON.SET acc .maxId ' + str(task.id))
return Response(status=201)
def main():
rj = Client(host='localhost', port=6379, decode_responses=True)
obj = {
'answer': 42,
'arr': [None, True, 3.14],
'truth': {
'coord': 'out there'
}
}
rj.jsonset('obj', Path.rootPath(), obj)
# Get something
print 'Is there anybody... {}?'.format(
rj.jsonget('obj', Path('.truth.coord')))
async def get_model(sampler: str):
samplers = rj.jsonget("samplers")
if sampler not in samplers:
raise ServerException(
f"Can't find model for sampler='{sampler}'. "
f"Valid choices for sampler are {samplers}"
)
if f"model-{sampler}" not in rj.keys():
logger.warning("rj.keys() = %s", rj.keys())
flush_logger(logger)
raise ServerException(f"Model has not been created for sampler='{sampler}'")
rj2 = Client(host="redis", port=6379, decode_responses=False)
ir = rj2.get(f"model-{sampler}")
model = cloudpickle.loads(ir)
return model
def lookup_difficulty_level(gamename,
redserv=None,
difficulty_map_key=REDIS_GATA_DIFFICULTY_MAP):
# assert gamename , "Missing required argument: gamename"
level = -1
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
for i in range(1, 11):
if _rj.sismember(difficulty_map_key + str(i), gamename):
level = i
break
if redserv is None:
_rj.close()
return level
def main(action, filepath, search):
rj = Client(host='localhost', port=6379,
decode_responses=True) # update your redis settings
if action == 'dump':
out = {}
for key in r.scan_iter(search):
out.update({key: r.get(key)})
if len(out) > 0:
try:
with open(filepath, 'w') as outfile:
json.dump(out, outfile)
print('Dump Successful')
except Exception as e:
print(e)
else:
print("Keys not found")
elif action == 'load':
try:
print('here......')
usr = r.json.get('user')
print(usr)
except Exception as e:
print('here is the error: ' + str(e))
def select_games_for_mingpt(redserv=None):
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
# for n in range(70):
# rkey = f"{REDIS_FTWC_NSTEPS_INDEX}{n}"
# if rj.exists(rkey):
# n_nsteps = rj.scard(f"{REDIS_FTWC_NSTEPS_INDEX}{n}")
# n_nodrop = rj.sdiff(f"{REDIS_FTWC_NSTEPS_INDEX}{n}", "ftwc:cog2019:skills:drop")
# # maxtoks, mintoks = get_max_token_length(n_nodrop)
# if n <= 32 and len(n_nodrop):
# rj.sadd(REDIS_MINGPT_ALL, *n_nodrop)
# print(f"{n:2d}:__{n_nsteps:3d}__{len(n_nodrop):3d}") #" {maxtoks:4d} {mintoks:4d}")
if redserv is None:
_rj.close()
def get_instance(self):
if not RedisClient.SINGLETON_INSTANCE:
RedisClient.SINGLETON_INSTANCE = Client(
host=os.environ['redis_host'],
port=os.environ['redis_port'],
decode_responses=True,
password=os.environ['redis_password'])
return RedisClient.SINGLETON_INSTANCE
class Database:
"""Database handler"""
def __init__(self, database_name, port=6379):
self.database = Client(host=database_name,
port=port,
decode_responses=True)
def flush(self):
"""Flushing a certain table of the database"""
return self.database.flushdb()
def import_data(self, keyname, data):
"""Load data into the database"""
return self.database.jsonset(keyname, Path.rootPath(), data)
def ping(self):
"""Tests the connection"""
return self.database.ping()
def pull_from_redis(keys,
client=Client(decode_responses=True),
path=Path.rootPath()):
insert_queue = list()
for key in client.scan_iter(keys):
insert_queue.append(client.jsonget(key, path))
return insert_queue
def create_skills_map(redserv=None,
skillsmap_key=REDIS_FTWC_SKILLS_MAP,
gameset_keys=FTWC_GAME_SETS):
""" after all game names have been added to redis, we map skills to game names"""
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
skills_index = {} # maps skillname to a set of game names
all_mapped_skills = set() # all game names that are in the skills map
for setkey in gameset_keys:
game_names = _rj.smembers(setkey)
print(f"{setkey} has {len(game_names)} members")
for g in game_names:
gid, sklist = split_gamename(g)
# print(g, gid, sklist)
for skill in sklist:
if skill not in skills_index:
skills_index[skill] = set()
skills_index[skill].add(g)
_rj.sadd(skillsmap_key + skill, g)
# print(len(skills_index), skills_index.keys())
# for key in skills_index.keys():
# print(key, len(skills_index[key]))
# all_mapped_skills = all_mapped_skills.union(skills_index[key])
skillsmap_keys = _rj.keys(skillsmap_key + "*")
for k in skillsmap_keys:
print(k, _rj.scard(k))
all_mapped_skills = all_mapped_skills.union(_rj.smembers(k))
print(
f"TOTAL # of game files for which skills have been mapped: {len(all_mapped_skills)}"
)
if redserv is None:
_rj.close()
def testCustomEncoderDecoderShouldSucceed(self):
"Test a custom encoder and decoder"
class CustomClass(object):
key = ''
val = ''
def __init__(self, k='', v=''):
self.key = k
self.val = v
class TestEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, CustomClass):
return 'CustomClass:{}:{}'.format(obj.key, obj.val)
return json.JSONEncoder.encode(self, obj)
class TestDecoder(json.JSONDecoder):
def decode(self, obj):
d = json.JSONDecoder.decode(self, obj)
if isinstance(d, six.string_types) and \
d.startswith('CustomClass:'):
s = d.split(':')
return CustomClass(k=s[1], v=s[2])
return d
rj = Client(encoder=TestEncoder(),
decoder=TestDecoder(),
port=port,
decode_responses=True)
rj.flushdb()
# Check a regular string
self.assertTrue(rj.jsonset('foo', Path.rootPath(), 'bar'))
self.assertEqual('string', rj.jsontype('foo', Path.rootPath()))
self.assertEqual('bar', rj.jsonget('foo', Path.rootPath()))
# Check the custom encoder
self.assertTrue(
rj.jsonset('cus', Path.rootPath(), CustomClass('foo', 'bar')))
obj = rj.jsonget('cus', Path.rootPath())
self.assertIsNotNone(obj)
self.assertEqual(CustomClass, obj.__class__)
self.assertEqual('foo', obj.key)
self.assertEqual('bar', obj.val)
def webhook_save():
if request.method == 'POST':
timestamp_human = datetime.datetime.now()
timestamp = int(time.time())
nowDatetime = timestamp_human.strftime('%Y-%m-%d(%H:%M:%S)')
req_data = request.get_json()
alertname = req_data['commonLabels']['alertname']
severity = ''
receiver = req_data['receiver']
key_name = str(timestamp) + "_" + alertname + "_" + receiver
try:
# conn = redis.Redis(host=REDIS_SERVER, port=6379, db=0, password=REDIS_PWD)
conn = Client(host=REDIS_SERVER,
port=6379,
db=0,
password=REDIS_PWD)
conn.ping()
print
'Redis connected %s' % (REDIS_SERVER)
except Exception as e:
print
'Error:', e
exit('Failed to connecting')
conn = Client(host=REDIS_SERVER, port=6379)
conn.jsonset(key_name, Path.rootPath(), req_data)
data = json.dumps(conn.jsonget(key_name))
print
data
# Redis : SCAN 0 match 1527911[1-9][1-9]*
else:
abort(400)
if not conn.exists(key_name):
print
"Error: %s is doesn't exist" % (key_name)
return jsonify({'status': 'success'}), 200
class RedisClient:
def __init__(self):
logger.debug(os.environ.get("REDIS_HOST"))
self.client = Client(host=os.environ.get("REDIS_HOST", "localhost"),
decode_responses=True)
def getAnyShape(self, index):
try:
shape = self.client.jsonget(index)
return shape
except Exception as e:
logger.warning('Fail to get the {} shape from Redis {}'.format(
index, e))
def from_settings(cls, settings):
params = {
'server': Client(host='localhost',
port=6379,
decode_responses=True),
}
if settings.get('REDIS_ITEMS_KEY'):
params['key'] = settings['REDIS_ITEMS_KEY']
if settings.get('REDIS_ITEMS_SERIALIZER'):
params['serialize_func'] = load_object(
settings['REDIS_ITEMS_SERIALIZER'])
return cls(**params)
