def _create_client(self, *args, **kwargs):
if REDIS_CLUSTER_HOST is None or \
REDIS_CLUSTER_PORT is None:
return Client(*args, **kwargs)
else:
from rediscluster import RedisCluster
startup_nodes = [{
"host": REDIS_CLUSTER_HOST,
"port": REDIS_CLUSTER_PORT
}]
conn = RedisCluster(startup_nodes=startup_nodes,
decode_responses=True)
return Client(client=conn, *args, **kwargs)
def create_difficulty_map(redserv=None,
difficulty_map_key=REDIS_GATA_DIFFICULTY_MAP,
game_dirs=None):
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
if game_dirs is None:
training_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TRAINING_DIR')
validation_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_VALIDATION_DIR')
test_dir = _rj.hget(REDIS_DIR_MAP, 'GATA_TEST_DIR')
game_dirs = (training_dir, validation_dir, test_dir)
print(
f"create_difficulty_map({difficulty_map_key}, game_dirs={game_dirs})")
for game_dir in game_dirs:
print("create_difficulty_map -- GAME DIR:", game_dir)
game_names_ = []
for level in range(1, 11):
difficulty = f"difficulty_level_{level}"
print("\n-------------------", difficulty)
games_list = count_game_files(game_dir + difficulty)
game_names_.extend(games_list)
_rj.sadd(difficulty_map_key + str(level), *games_list)
print(
f"total games in {game_dir}: {len(game_names_)} {len(set(game_names_))}"
)
assert len(game_names_) == len(
set(game_names_)) # they should all be unique
if redserv is None:
_rj.close()
def post(self, request):
validated_data = request.serializer.validated_data
task = Task.objects.create(title=validated_data['title'],
description=validated_data['description'],
status=Task.CREATED,
user_created=request.user,
date_create_task=datetime.now())
if validated_data['user_assigned']:
task.user_assigned = validated_data['user_assigned']
task.save()
if validated_data['user_assigned'] and validated_data[
'user_assigned'] != request.user:
AddNotificationTaskStatus(task.user_assigned, task, "created")
# add in ReJSON database
rj = Client(
host='localhost',
port=6379,
)
rj.jsonset('task:' + str(task.id), Path.rootPath(),
TaskSerializer(task).data)
rj.execute_command('JSON.NUMINCRBY acc .total 1')
rj.execute_command('JSON.SET acc .maxId ' + str(task.id))
return Response(status=201)
def __init__(self):
self.cfg = read_cfg("redis")
self.rj = Client(host=self.cfg.get("ip"),
port=self.cfg.get("port"),
decoder=RedisJsonDecoder(),
decode_responses=True)
self.logger = logger.myLogger("Redis")
async def receive(attr_dict):
redis_path_param_default = '_d'
uri = "ws://tasks.logstash:3232"
async with websockets.connect(uri, ping_interval=None) as websocket:
data = await websocket.recv()
print(f"> {data}")
# Parse this message and extract the resource-group
res_data = json.loads(data)
res_id = res_data['id'].replace('/', '__')
print('> res_id-- ' + res_id)
rg = res_id.split('__')[3]
print('> rg-- ' + rg)
rejson_key = rg
print('> attr_dict-- ' + str(attr_dict))
# Check if the resource-group is present in the HashMap and it's attribute value
if rg in attr_dict.keys():
print('> true')
attribute = attr_dict[rg]
redis_path_param = res_id + '_' + res_data[attribute]
else:
redis_path_param = res_id + redis_path_param_default
# use the rejson module to insert the data into Redis db
rj = Client(host='redis-server', port=6379, decode_responses=True)
print('> redis_path_param-- ' + redis_path_param)
redis_path_param = redis_path_param.replace('.', '$')
redis_path_param = redis_path_param.replace('-', '_')
#rj.jsonset(rejson_key,Path.rootPath(),json.dumps({}))
rj.jsonset(rejson_key, Path('.' + redis_path_param), res_data)
def redis_test():
rj = Client(host='localhost', port=6379)
# Set the key `obj` to some object
obj = {
'answer': 42,
'arr': [None, True, 3.14],
'truth': {
'coord': 'out there'
}
}
rj.jsonset('obj', Path.rootPath(), obj)
# Get something
print ('Is there anybody... {}?'.format(
rj.jsonget('obj', Path('.truth.coord'))
))
# Delete something (or perhaps nothing), append something and pop it
rj.jsondel('obj', Path('.arr[0]'))
rj.jsonarrappend('obj', Path('.arr'), 'something')
print ('{} popped!'.format(rj.jsonarrpop('obj', Path('.arr'))))
# Update something else
rj.jsonset('obj', Path('.answer'), 2.17)
# And use just like the regular redis-py client
jp = rj.pipeline()
jp.set('foo', 'bar')
jp.jsonset('baz', Path.rootPath(), 'qaz')
jp.execute()
def testCustomEncoderDecoderShouldSucceed(self):
"Test a custom encoder and decoder"
class CustomClass(object):
key = ''
val = ''
def __init__(self, k='', v=''):
self.key = k
self.val = v
class TestEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, CustomClass):
return 'CustomClass:{}:{}'.format(obj.key, obj.val)
return json.JSONEncoder.encode(self, obj)
class TestDecoder(json.JSONDecoder):
def decode(self, obj):
d = json.JSONDecoder.decode(self, obj)
if isinstance(d, six.string_types) and \
d.startswith('CustomClass:'):
s = d.split(':')
return CustomClass(k=s[1], v=s[2])
return d
rj = Client(encoder=TestEncoder(),
decoder=TestDecoder(),
port=port,
decode_responses=True)
rj.flushdb()
# Check a regular string
self.assertTrue(rj.jsonset('foo', Path.rootPath(), 'bar'))
self.assertEqual('string', rj.jsontype('foo', Path.rootPath()))
self.assertEqual('bar', rj.jsonget('foo', Path.rootPath()))
# Check the custom encoder
self.assertTrue(
rj.jsonset('cus', Path.rootPath(), CustomClass('foo', 'bar')))
# Check the custom decoder
obj = rj.jsonget('cus', Path.rootPath())
self.assertIsNotNone(obj)
self.assertEqual(CustomClass, obj.__class__)
self.assertEqual('foo', obj.key)
self.assertEqual('bar', obj.val)
# Test resetting the decoder after the client have been created
rj.setDecoder(json.JSONDecoder())
obj = rj.jsonget('cus', Path.rootPath())
self.assertIsNotNone(obj)
self.assertNotEqual(CustomClass, obj.__class__)
# Test setting the decoder after the client have been created
rj.setDecoder(TestDecoder())
obj = rj.jsonget('cus', Path.rootPath())
self.assertIsNotNone(obj)
self.assertEqual(CustomClass, obj.__class__)
self.assertEqual('foo', obj.key)
self.assertEqual('bar', obj.val)
def __init__(self, host: str = "localhost", port: int = 6379):
self._host = host
self._port = port
self._client = Client(host=host,
port=port,
decode_responses=True,
encoder=JSONSchemaObject.JSONSchemaEncoder())
def main(action, filepath, search):
rj = Client(host='localhost', port=6379,
decode_responses=True) # update your redis settings
if action == 'dump':
out = {}
for key in r.scan_iter(search):
out.update({key: r.get(key)})
if len(out) > 0:
try:
with open(filepath, 'w') as outfile:
json.dump(out, outfile)
print('Dump Successful')
except Exception as e:
print(e)
else:
print("Keys not found")
elif action == 'load':
try:
print('here......')
usr = r.json.get('user')
print(usr)
except Exception as e:
print('here is the error: ' + str(e))
def from_settings(cls, settings):
kwargs = {
'persist': settings.getbool('SCHEDULER_PERSIST'),
'flush_on_start': settings.getbool('SCHEDULER_FLUSH_ON_START'),
'idle_before_close':
settings.getint('SCHEDULER_IDLE_BEFORE_CLOSE'),
}
# If these values are missing, it means we want to use the defaults.
optional = {
# TODO: Use custom prefixes for this settings to note that are
# specific to scrapy-redis.
'queue_key': 'SCHEDULER_QUEUE_KEY',
'queue_cls': 'SCHEDULER_QUEUE_CLASS',
'dupefilter_key': 'SCHEDULER_DUPEFILTER_KEY',
# We use the default setting name to keep compatibility.
'dupefilter_cls': 'DUPEFILTER_CLASS',
'serializer': 'SCHEDULER_SERIALIZER',
}
for name, setting_name in optional.items():
val = settings.get(setting_name)
if val:
kwargs[name] = val
# Support serializer as a path to a module.
if isinstance(kwargs.get('serializer'), six.string_types):
kwargs['serializer'] = importlib.import_module(
kwargs['serializer'])
server = Client(host='localhost', port=6379, decode_responses=True)
# Ensure the connection is working.
server.ping()
return cls(server=server, **kwargs)
def webhook_view(timestamp=None,
alert_type=None,
reciever=None,
key_name=None):
if request.method == 'GET':
conn = Client(host=REDIS_SERVER, port=6379, db=0, password=REDIS_PWD)
params = timestamp + '_' + alert_type + '_' + reciever
data = json.dumps(conn.jsonget(params))
return render_template('info.html', data=data)
def __init__(self, host: str, port: Union[str, int]) -> None:
"""Instantiate a connection to ReJson.
:param host: The hostname/ip of the Redis instance.
:type host: str
:param port: The port of the Redis instance.
:type port: int
"""
self._client = Client(host=host, port=port, decode_responses=True)
def find_by_state(self):
try:
rj = Client(
host='redis',
port=6379,
decode_responses=True)
return rj.jsonget(self.name)
except:
return None
def get_instance(self):
if not RedisClient.SINGLETON_INSTANCE:
RedisClient.SINGLETON_INSTANCE = Client(
host=os.environ['redis_host'],
port=os.environ['redis_port'],
decode_responses=True,
password=os.environ['redis_password'])
return RedisClient.SINGLETON_INSTANCE
def save_simgrids_setup_data_rejson(id, setupJson, rejson_host, rejson_key,
rejson_db):
saveJson = {**get_default_json(), **setupJson}
rj = Client(host=rejson_host, port=rejson_key, db=rejson_db)
redis_key = rj.incr('simgrids_key')
saveJson["id"] = redis_key
json_key = "simgrids_key_{0}".format(redis_key)
status = rj.jsonset(json_key, Path.rootPath(), saveJson)
return redis_key, saveJson
def main():
rj = Client(host='localhost', port=6379, decode_responses=True)
obj = {
'answer': 42,
'arr': [None, True, 3.14],
'truth': {
'coord': 'out there'
}
}
def __init__(self):
self.tests = ['ru.4', '5k4', 'u;4', 'ji3', '5', '2l4',
'xk7'] # no need bracket here
# the redis server
self.rj = Client(host='localhost', port=6379, decode_responses=True)
# phonetic table, radical to han characters
self.fn = 'phone.json'
self.objname = 'obj'
self.data = None
self.check_and_store()
def pull_from_redis(keys,
client=Client(decode_responses=True),
path=Path.rootPath()):
insert_queue = list()
for key in client.scan_iter(keys):
insert_queue.append(client.jsonget(key, path))
return insert_queue
def get_max_score(gn, redserv=None):
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
assert isinstance(redserv, Client)
_rj = redserv
dirpath = get_dir_for_game(_rj, gn)
if redserv is None:
_rj.close()
with open(f"{dirpath}/{gn}.json", "r") as f:
metadata = json.load(f)['metadata']
return int(metadata['max_score'])
def from_settings(cls, settings):
params = {
'server': Client(host='localhost',
port=6379,
decode_responses=True),
}
if settings.get('REDIS_ITEMS_KEY'):
params['key'] = settings['REDIS_ITEMS_KEY']
if settings.get('REDIS_ITEMS_SERIALIZER'):
params['serialize_func'] = load_object(
settings['REDIS_ITEMS_SERIALIZER'])
return cls(**params)
def webhook_save():
if request.method == 'POST':
timestamp_human = datetime.datetime.now()
timestamp = int(time.time())
nowDatetime = timestamp_human.strftime('%Y-%m-%d(%H:%M:%S)')
req_data = request.get_json()
alertname = req_data['commonLabels']['alertname']
severity = ''
receiver = req_data['receiver']
key_name = str(timestamp) + "_" + alertname + "_" + receiver
try:
# conn = redis.Redis(host=REDIS_SERVER, port=6379, db=0, password=REDIS_PWD)
conn = Client(host=REDIS_SERVER,
port=6379,
db=0,
password=REDIS_PWD)
conn.ping()
print
'Redis connected %s' % (REDIS_SERVER)
except Exception as e:
print
'Error:', e
exit('Failed to connecting')
conn = Client(host=REDIS_SERVER, port=6379)
conn.jsonset(key_name, Path.rootPath(), req_data)
data = json.dumps(conn.jsonget(key_name))
print
data
# Redis : SCAN 0 match 1527911[1-9][1-9]*
else:
abort(400)
if not conn.exists(key_name):
print
"Error: %s is doesn't exist" % (key_name)
return jsonify({'status': 'success'}), 200
def init_db(app, args):
if args.redis is not None:
app.db = Client(host=args.redis,
charset="utf-8",
decode_responses=True)
if not is_redis_available(app):
raise Exception(
'Cannot launch application without connection to redis')
app.persistent = True
else:
app.db = dict()
app.manifests = dict()
app.persistent = False
logging.info(
"Launching Registry UI without database, data won't be persisted.")
def main():
rj = Client(host='localhost', port=6379, decode_responses=True)
obj = {
'answer': 42,
'arr': [None, True, 3.14],
'truth': {
'coord': 'out there'
}
}
rj.jsonset('obj', Path.rootPath(), obj)
# Get something
print 'Is there anybody... {}?'.format(
rj.jsonget('obj', Path('.truth.coord')))
async def get_model(sampler: str):
samplers = rj.jsonget("samplers")
if sampler not in samplers:
raise ServerException(
f"Can't find model for sampler='{sampler}'. "
f"Valid choices for sampler are {samplers}"
)
if f"model-{sampler}" not in rj.keys():
logger.warning("rj.keys() = %s", rj.keys())
flush_logger(logger)
raise ServerException(f"Model has not been created for sampler='{sampler}'")
rj2 = Client(host="redis", port=6379, decode_responses=False)
ir = rj2.get(f"model-{sampler}")
model = cloudpickle.loads(ir)
return model
def create_nsteps_map(redserv=None,
nsteps_map_key=REDIS_FTWC_NSTEPS_MAP,
nsteps_index_key=REDIS_FTWC_NSTEPS_INDEX,
gameset_keys=FTWC_GAME_SETS):
""" after all playthroughs have been save to redis, index number of steps <-> game names """
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
if nsteps_map_key == REDIS_GATA_NSTEPS_MAP:
redisbasekey = REDIS_GATA_PLAYTHROUGHS
elif nsteps_map_key == REDIS_FTWC_NSTEPS_MAP:
redisbasekey = REDIS_FTWC_PLAYTHROUGHS
else:
assert False, "Unknown Redis nsteps_map_key " + nsteps_map_key
for key in _rj.keys(nsteps_index_key + "*"):
print("Will delete:", key)
_rj.delete(key)
print(_rj.hlen(nsteps_map_key))
_rj.delete(nsteps_map_key)
for setkey in gameset_keys:
game_names_ = _rj.smembers(setkey)
for _gn in game_names_:
nsteps = retrieve_playthrough_nsteps(_gn,
redis=_rj,
redisbasekey=redisbasekey)
if nsteps > 0:
print(nsteps, _gn)
_rj.hset(nsteps_map_key, _gn, nsteps)
_rj.sadd(f"{nsteps_index_key}{nsteps}", _gn)
print(len(_rj.keys(nsteps_index_key + "*")), _rj.hlen(nsteps_map_key))
total = 0
sort_list = []
for key in _rj.keys(nsteps_index_key + "*"):
nsteps = int(key.split(':')[-1])
num_games = _rj.scard(key)
total += num_games
sort_list.append((nsteps, num_games, key))
# print(key, "has", num_games, "game names")
sort_list.sort()
for nsteps, num_games, setkey in sort_list:
print(f"[{nsteps}]\t {num_games}\t {setkey}")
if redserv is None:
_rj.close()
def lookup_difficulty_level(gamename,
redserv=None,
difficulty_map_key=REDIS_GATA_DIFFICULTY_MAP):
# assert gamename , "Missing required argument: gamename"
level = -1
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
for i in range(1, 11):
if _rj.sismember(difficulty_map_key + str(i), gamename):
level = i
break
if redserv is None:
_rj.close()
return level
def select_games_for_mingpt(redserv=None):
if redserv is None:
_rj = Client(host='localhost', port=6379, decode_responses=True)
else:
_rj = redserv
# for n in range(70):
# rkey = f"{REDIS_FTWC_NSTEPS_INDEX}{n}"
# if rj.exists(rkey):
# n_nsteps = rj.scard(f"{REDIS_FTWC_NSTEPS_INDEX}{n}")
# n_nodrop = rj.sdiff(f"{REDIS_FTWC_NSTEPS_INDEX}{n}", "ftwc:cog2019:skills:drop")
# # maxtoks, mintoks = get_max_token_length(n_nodrop)
# if n <= 32 and len(n_nodrop):
# rj.sadd(REDIS_MINGPT_ALL, *n_nodrop)
# print(f"{n:2d}:__{n_nsteps:3d}__{len(n_nodrop):3d}") #" {maxtoks:4d} {mintoks:4d}")
if redserv is None:
_rj.close()
async def main_loop(loop):
global redis_client
try:
# Connect to Redis
# retrieve config using docker configs
redis_client = Client(host='0.0.0.0',
port=28734,
decode_responses=True)
# Connect to RabbitMQ
# Can use connection pool instead of two separate clients
# rmq_sub_client = await connect_robust(host='0.0.0.0',port=29042,login='******',
# password='******', virtualhost='IUDX',loop=loop
# )
rmq_sub_client = await connect_robust(host='0.0.0.0',
port=29042,
login='******',
password='******',
virtualhost='IUDX',
loop=loop)
except AMQPException as e:
print('> Connection Failed!')
e.printStackTrace()
return
# redis_q_name = "redis-ingestion-queue"
redis_q_name = "vertx-rmq-redis-reader"
routing_key = "#"
async with rmq_sub_client:
channel = await rmq_sub_client.channel()
queue_redis = await channel.declare_queue(
redis_q_name, durable=True, arguments={'x-max-length': 20000})
result = await queue_redis.consume(on_message)
# Surround with a try catch block?
# remove print and use logs instead
if result is not None:
if result == 'success':
print('> Insertion was successful.')
elif result == 'failed':
print('> Insertion failed!')
else:
print('> Packet not found [or] Queue is empty!')
def get(self, request):
rj = Client(
host='localhost',
port=6379,
)
deleteRedisJSON(rj)
tasks = Task.objects.all()
for task in tasks:
rj.jsonset('task:' + str(task.id), Path.rootPath(),
TaskSerializer(task).data)
accountant = dict()
accountant.update({"total": tasks.count()})
accountant.update({"maxId": tasks.last().id})
rj.jsonset('acc', Path.rootPath(), accountant)
return Response(status=200)
def testUsageExampleShouldSucceed(self):
"Test the usage example"
# Create a new rejson-py client
rj = Client(host='localhost', port=port, decode_responses=True)
# Set the key `obj` to some object
obj = {
'answer': 42,
'arr': [None, True, 3.14],
'truth': {
'coord': 'out there'
}
}
rj.jsonset('obj', Path.rootPath(), obj)
# Get something
rv = rj.jsonget('obj', Path('.truth.coord'))
self.assertEqual(obj['truth']['coord'], rv)
# Delete something (or perhaps nothing), append something and pop it
value = "something"
rj.jsondel('obj', Path('.arr[0]'))
rj.jsonarrappend('obj', Path('.arr'), value)
rv = rj.jsonarrpop('obj', Path('.arr'))
self.assertEqual(value, rv)
# Update something else
value = 2.17
rj.jsonset('obj', Path('.answer'), value)
rv = rj.jsonget('obj', Path('.answer'))
self.assertEqual(value, rv)
# And use just like the regular redis-py client
jp = rj.pipeline()
jp.set('foo', 'bar')
jp.jsonset('baz', Path.rootPath(), 'qaz')
jp.execute()
rv1 = rj.get('foo')
self.assertEqual('bar', rv1)
rv2 = rj.jsonget('baz')
self.assertEqual('qaz', rv2)