class RedisManager(NoSqlManager):
def __init__(self,
namespace,
url=None,
data_dir=None,
lock_dir=None,
**params):
self.db = params.pop('db', None)
self.dbpass = params.pop('password', None)
self.connection_pool = params.get('redis_connection_pool', None)
self.expires = params.get('expires', params.get('expiretime', None))
NoSqlManager.__init__(self,
namespace,
url=url,
data_dir=data_dir,
lock_dir=lock_dir,
**params)
def open_connection(self, host, port, **params):
if not self.connection_pool:
self.connection_pool = ConnectionPool(host=host, port=port, db=self.db,
password=self.dbpass)
self.db_conn = StrictRedis(connection_pool=self.connection_pool, **params)
def __contains__(self, key):
return self.db_conn.exists(self._format_key(key))
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
# beaker.container.Value.set_value calls NamespaceManager.set_value
# however it (until version 1.6.4) never sets expiretime param.
#
# Checking "type(value) is tuple" is a compromise
# because Manager class can be instantiated outside container.py (See: session.py)
if (expiretime is None) and (type(value) is tuple):
expiretime = value[1]
# If the machinery above fails, then pickup the expires time from the
# init params.
if not expiretime and self.expires is not None:
expiretime = self.expires
# Set or setex, according to whether we got an expires time or not.
if expiretime:
self.db_conn.setex(key, expiretime, pickle.dumps(value, 2))
else:
self.db_conn.set(key, pickle.dumps(value, 2))
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return 'beaker:%s:%s' % (self.namespace, key.replace(' ', '\302\267'))
def _format_pool_key(self, host, port, db):
return '{0}:{1}:{2}'.format(host, port, self.db)
def do_remove(self):
self.db_conn.flush()
def keys(self):
return self.db_conn.keys('beaker:%s:*' % self.namespace)
class RedisQueue:
def __init__(self):
"""
初始化Redis
"""
self.db = StrictRedis(host=REDIS_HOST,
port=REDIS_PORT,
password=REDIS_PASSWORD)
def add(self, request):
"""
向队列添加序列化后的Request
:param request: 请求对象
:param fail_time: 失败次数
:return: 添加结果
"""
if isinstance(request, Requests):
return self.db.rpush(REDIS_KEY, dumps(request))
return False
def pop(self):
"""
取出下一个Request并反序列化
:return: Request or None
"""
if self.db.llen(REDIS_KEY):
return loads(self.db.lpop(REDIS_KEY))
else:
return False
def clear(self):
self.db.delete(REDIS_KEY)
def empty(self):
return self.db.llen(REDIS_KEY) == 0
class RedisQueue:
def __init__(self):
"""
初始化Redis
"""
self.db = StrictRedis(host=REDIS_HOST,
port=REDIS_PORT,
password=REDIS_PASSWORD)
def put(self, myrequest):
"""
向队列添加序列化后的MyRequest
:param myrequest: 请求对象
:return: 添加结果
"""
if isinstance(myrequest, MyRequest):
return self.db.rpush(REDIS_KEY, pickle.dumps(myrequest))
return False
def get(self):
"""
取出下一个MyRequest并反序列化
:return: MyRequest对象
"""
if self.db.llen(REDIS_KEY):
return pickle.loads(self.db.lpop(REDIS_KEY))
else:
return None
def clear(self):
self.db.delete(REDIS_KEY)
def empty(self):
return self.db.llen(REDIS_KEY) == 0
class RedisTestCase(SimpleTestCase):
indices = {
'my-app:ed54cda':
'<html><head><meta name="my-app/config/environment" content="%7B%22baseURL%22%3A%22/%22%7D"></head><body><h1>my-app</h1><h2>ed54cda</h2></body></html>',
'my-app:d696248':
'<html><head><meta name="my-app/config/environment" content="%7B%22baseURL%22%3A%22/%22%7D"></head><body><h1>my-app</h1><h2>d696248</h2></body></html>',
'my-app:7fabf72':
'<html><head><meta name="my-app/config/environment" content="%7B%22baseURL%22%3A%22/%22%7D"></head><body><h1>my-app</h1><h2>7fabf72</h2></body></html>',
'my-app:current':
'<html><head><meta name="my-app/config/environment" content="%7B%22baseURL%22%3A%22/%22%7D"></head><body><h1>my-app</h1><h2>7fabf72</h2></body></html>',
'other-app:fd54cda':
'<html><head><base href="/" /><meta name="other-app/config/environment" content="%7B%22baseURL%22%3A%22/other-app/%22%7D"></head><body><h1>other-app</h1><h2>fd54cda</h2></body></html>',
'other-app:e696248':
'<html><head><base href="/" /><meta name="other-app/config/environment" content="%7B%22baseURL%22%3A%22/other-app/%22%7D"></head><body><h1>other-app</h1><h2>e696248</h2></body></html>',
'other-app:8fabf72':
'<html><head><base href="/" /><meta name="other-app/config/environment" content="%7B%22baseURL%22%3A%22/other-app/%22%7D"></head><body><h1>other-app</h1><h2>8fabf72</h2></body></html>',
'other-app:current':
'<html><head><base href="/" /><meta name="other-app/config/environment" content="%7B%22baseURL%22%3A%22/other-app/%22%7D"></head><body><h1>other-app</h1><h2>8fabf72</h2></body></html>'
}
def setUp(self):
self.client = Client()
self.redis = StrictRedis()
for key, value in self.indices.items():
self.redis.set(key, value)
def tearDown(self):
self.redis.delete(*self.indices.keys())
class RedisCache(object):
def __init__(self, url=None, client=None):
if url: # pragma: no cover
self.client = StrictRedis.from_url(url)
elif client: # pragma: no cover
self.client = client
else:
self.client = StrictRedis()
def get(self, key):
return self.client.get(key)
def mget(self, keys):
return self.client.mget(keys)
def delete(self, key):
self.client.delete(key)
def mdelete(self, keys):
self.client.delete(*keys)
def set(self, key, value, ttl):
self.client.set(key, value, ttl)
def mset(self, items, ttl):
p = self.client.pipeline(transaction=False)
for key, value in items:
p.set(key, value, ttl)
p.execute()
class RedisQueue():
#初始化
def __init__(self):
self.db = StrictRedis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD)
#向队列中添加序列化后的Request
def add(self, request):
#序列化请求
if isinstance(request, WeixinRequest):
return self.db.rpush(REDIS_KEY, dumps(request))
return False
#取出下一个Request并反序列化
def pop(self):
#反序列化
if self.db.llen(REDIS_KEY):
return loads(self.db.lpop(REDIS_KEY))
else:
return False
def clear(self):
self.db.delete(REDIS_KEY)
def empty(self):
return self.db.llen(REDIS_KEY) == 0
def post_updates(self, updates, log_index):
'''
Update the cache with CRUD changes
'''
cache = StrictRedis(db=config.tokens_cache_redis_db)
self.log.info('post_updates(): posting updates to local storage')
for update in updates: # TODO: could re-add the Redis "Pipelines" feature to combine Redis requests for better performance when available
(user, token, date, action) = update
if action == 'add':
cache.hset('general', token, user) # future method - user-by-token -- really just existence of a token
cache.hset('users', user, token) # future-method - token-by-user: allow lookup of previous token on token changes
cache.set(token, user) # Current method
self.log.info('post_updates(): added token for user: '******'delete':
cache.hdel('general', token) # future method - disables the ability to authenticate
cache.hdel('users', user) # future method - removes history of token
cache.delete(token)
self.log.info('post_updates(): deleted token for user: '******'update':
prev_token = cache.hget('users', user)
cache.hdel('general', prev_token) # future method - disables the ability to authenticate with previous token
cache.hset('general', token, user) # future method - set the new token for the user
cache.hset('users', user, token) # future method - set the user as possessing the new token
cache.set(token, user)
self.log.info('post_updates(): updated token for user: '******'post_updates(): unexpected change type: ' + action)
if len(updates) > 0: # don't set if there is nothing to do and also don't set if there are errors
cache.set('log_index', log_index)
class RedisStorage(MachineBaseStorage):
def __init__(self, settings):
super().__init__(settings)
self._key_prefix = settings.get('REDIS_KEY_PREFIX', 'SM')
redis_config = gen_config_dict(settings)
self._redis = StrictRedis(**redis_config)
def _prefix(self, key):
return "{}:{}".format(self._key_prefix, key)
def has(self, key):
return self._redis.exists(self._prefix(key))
def get(self, key):
return self._redis.get(self._prefix(key))
def set(self, key, value, expires=None):
self._redis.set(self._prefix(key), value, expires)
def delete(self, key):
self._redis.delete(self._prefix(key))
def size(self):
info = self._redis.info('memory')
return info['used_memory']
class RedisSessionInterface(SessionInterface):
def __init__(self, prefix="session:"):
self.redis = StrictRedis(host=config.REDIS_HOST,
port=config.REDIS_PORT,
db=config.REDIS_DB)
self.prefix = prefix
def _generate_session_id(self):
return str(uuid4())
def open_session(self, app, request):
""" Flask requires implementaion of open_session() and save_session()
when creating custom session interface.
"""
session_id = request.cookies.get(app.session_cookie_name)
if not session_id:
session_id = self._generate_session_id()
return RedisSession(session_id=session_id, is_new=True)
serialized_data = self.redis.get(self.prefix + session_id)
if serialized_data is not None:
data = pickle.loads(serialized_data)
return RedisSession(data, session_id=session_id)
return RedisSession(session_id=session_id, is_new=True)
def _get_expiration_time_redis(self, app, session):
if session.permanent:
return app.permanent_session_lifetime
return timedelta(days=1)
def save_session(self, app, session, response):
domain = self.get_cookie_domain(app)
""" When session does not contain any meaningful information.
(i.e. dict(session) -> {})
"""
if not session:
self.redis.delete(self.prefix + session.session_id)
if session.modified:
response.delete_cookie(app.session_cookie_name, domain=domain)
return
expiration_time_redis = self._get_expiration_time_redis(app, session)
expiration_time_cookie = self.get_expiration_time(app, session)
serialized_data = pickle.dumps(dict(session))
self.redis.setex(self.prefix + session.session_id,
int(expiration_time_redis.total_seconds()),
serialized_data)
response.set_cookie(app.session_cookie_name,
session.session_id,
expires=expiration_time_cookie,
httponly=True,
domain=domain)
def test_group_chord_group_chain(self, manager):
from celery.five import bytes_if_py2
if not manager.app.conf.result_backend.startswith('redis'):
raise pytest.skip('Requires redis result backend.')
redis_connection = StrictRedis()
redis_connection.delete('redis-echo')
before = group(redis_echo.si('before {}'.format(i)) for i in range(3))
connect = redis_echo.si('connect')
after = group(redis_echo.si('after {}'.format(i)) for i in range(2))
result = (before | connect | after).delay()
result.get(timeout=TIMEOUT)
redis_messages = list(map(
bytes_if_py2,
redis_connection.lrange('redis-echo', 0, -1)
))
before_items = \
set(map(bytes_if_py2, (b'before 0', b'before 1', b'before 2')))
after_items = set(map(bytes_if_py2, (b'after 0', b'after 1')))
assert set(redis_messages[:3]) == before_items
assert redis_messages[3] == b'connect'
assert set(redis_messages[4:]) == after_items
redis_connection.delete('redis-echo')
class RedisJobStore(JobStore):
def __init__(self, db=0, key_prefix="jobs.", pickle_protocol=pickle.HIGHEST_PROTOCOL, **connect_args):
self.jobs = []
self.pickle_protocol = pickle_protocol
self.key_prefix = key_prefix
if db is None:
raise ValueError('The "db" parameter must not be empty')
if not key_prefix:
raise ValueError('The "key_prefix" parameter must not be empty')
self.redis = StrictRedis(db=db, **connect_args)
def add_job(self, job):
job.id = str(uuid4())
job_state = job.__getstate__()
job_dict = {
"job_state": pickle.dumps(job_state, self.pickle_protocol),
"runs": "0",
"next_run_time": job_state.pop("next_run_time").isoformat(),
}
self.redis.hmset(self.key_prefix + job.id, job_dict)
self.jobs.append(job)
def remove_job(self, job):
self.redis.delete(self.key_prefix + job.id)
self.jobs.remove(job)
def load_jobs(self):
jobs = []
keys = self.redis.keys(self.key_prefix + "*")
pipeline = self.redis.pipeline()
for key in keys:
pipeline.hgetall(key)
results = pipeline.execute()
for job_dict in results:
job_state = {}
try:
job = Job.__new__(Job)
job_state = pickle.loads(job_dict["job_state".encode()])
job_state["runs"] = long(job_dict["runs".encode()])
dateval = job_dict["next_run_time".encode()].decode()
job_state["next_run_time"] = datetime.strptime(dateval, "%Y-%m-%dT%H:%M:%S")
job.__setstate__(job_state)
jobs.append(job)
except Exception:
job_name = job_state.get("name", "(unknown)")
logger.exception('Unable to restore job "%s"', job_name)
self.jobs = jobs
def update_job(self, job):
attrs = {"next_run_time": job.next_run_time.isoformat(), "runs": job.runs}
self.redis.hmset(self.key_prefix + job.id, attrs)
def close(self):
self.redis.connection_pool.disconnect()
def __repr__(self):
return "<%s>" % self.__class__.__name__
def gdelete(request):
groupname = request.POST.get('name', 0)
rd = StrictRedis()
pref = settings.MY_PREFIX
prefg = pref + ":" + groupname
user = str(request.user)
print "received request for deleting", groupname, "from", user
ismember = rd.sismember(pref + ":groups", groupname)
if not ismember:
return JsonResponse({
'done': False,
'reason': 'No such group name'
})
# now check whether the requesting user is the one who created the group
d = rd.hgetall(prefg + ":hash")
if d['owner'] != user:
return JsonResponse({
'done': False,
'reason': 'Only group owner can delete the group'
})
rd.srem(pref + ":groups", groupname)
rd.delete(prefg + ":hash", pref + ":" + groupname)
rd.delete(pref + ":" + groupname)
redis_publisher = RedisPublisher(facility=pref, broadcast=True)
redis_publisher.publish_message(
RedisMessage(json.dumps({
"type": "group_delete",
"name": groupname
})))
return JsonResponse({'done': True})
def deploy_async(t_id: int, user: User, self_vars: dict):
task = Task.objects.get(id=t_id)
sls = task.project.sls.replace(settings.SALT_STATE_DIRECTORY, '')
hosts = get_all_hosts(task)
values = {
**{
value.configure.name: value.value
for value in Vars.objects.filter(inventory=task.inventory)
},
**self_vars
}
pipe = StrictRedis(host=settings.REDIS_HOST_SERVER,
port=settings.REDIS_HOST_PORT,
db=settings.REDIS_DB).pipeline()
for hostname in hosts:
pipe.delete(hostname)
pipe.hmset(hostname, values)
responses = pipe.execute()
print("sync values to redis for hostname[%s], result: %s" %
(hostname, str(responses)))
print("project %s deployed on hosts; %s" % (task.project.name, hosts))
p = Pepper().login()
result = p.local_async(tgt=hosts,
fun='state.sls',
arg=[sls],
tgt_type='list')
# {'return': [{'jid': '20190301083447106122', 'minions': ['074cda43674f']}]}
task.status = TASK_STATUS[1][0]
job_id = result['return'][0]['jid']
task.barn = result['return'][0]['minions']
task.occupy = job_id
task.operator = user
task.save()
return job_id
class redis_op(object):
"""
确保单例模式
"""
def __new__(cls, *arg, **kw):
if not hasattr(cls, "_instance"):
cls._instance = super().__new__(cls)
return cls._instance
def __init__(self, redis_key):
self.rc = StrictRedis(
connection_pool=ConnectionPool(host="127.0.0.1", port=6379, db=0))
self._lock = 0
self._lock_key = f"{redis_key}_lock_key"
def __getattr__(self, name):
return getattr(self.rc, name)
def __enter__(self):
while not self._lock:
time_out = time.time() + 10 + 1
self._lock = self.rc.setnx(self._lock_key, time_out)
if self._lock or (time.time() > float(self.rc.get(self._lock_key))
and time.time() > float(
self.rc.getset(self._lock_key, time_out))):
return self
else:
time.sleep(0.3)
def __exit__(self, exc_type, exc_val, exc_tb):
if time.time() < float(self.rc.get(self._lock_key)):
self.rc.delete(self._lock_key)
def get(self):
args = get_parser.parse_args()
# We've passed parameter validation.
redis_conn = StrictRedis(connection_pool=pool)
if not redis_conn.exists(args.username):
return 'Username not found', 404 # NOT FOUND
# Get all id's linked to username
username_ids = redis_conn.lrange(args.username, 0, -1)
# Delete username key
redis_conn.delete(args.username)
# Response from redis (messages)
db_response = redis_conn.mget(*username_ids)
# print(f'HERER {db_response}', file=sys.stderr)
if not db_response:
# No valid text messages for user args.username
return 'No text messages', 404 # NOT FOUND
# Create a JSON array
api_response = [{
'id': int(_id),
'text': text.decode('utf-8')
} for _id, text in zip(username_ids, db_response)]
# I want to try searching and removing expired entries every 5min
# print(f'HERER {api_response}', file=sys.stderr)
return api_response, 200
def init_redis_for_testing_cruncher(game_file):
buffer = Buffer(**REDIS_PARAM)
buffer.delete(TO_CRUNCHER)
buffer.delete(TO_USERCRUNCHER)
with open(game_file, "r") as f:
for game_id in f.readlines():
buffer.lpush(TO_CRUNCHER, game_id.strip())
class Cache():
def __init__(self):
self.redis = StrictRedis(settings.REDIS['host'],
settings.REDIS['port'], settings.REDIS['db'])
def set(self, key, data, time=None):
_data = pickle.dumps(data)
if time:
self.redis.setex(key, time, _data)
else:
self.redis.set(key, _data)
def get(self, key, delete=False):
value = self.redis.get(key)
if delete:
self.delete(key)
try:
return pickle.loads(value)
except:
return value
def delete(self, key):
self.redis.delete(key)
def get_token(self, data):
token = gen_token()
self.set(token, data)
return token
def get_pin(self, data):
pin = gen_pin()
self.set(pin, data)
return pin
def test_big_chunk():
rclient = StrictRedis()
rclient.delete('boo')
cmd = rw.compile('SET', bytes, bytes)
chunk = [cmd(b'boo', b'boo') for _ in range(10000)]
rw.execute(chunk, client=rclient)
def zset_score_min_max_example():
""" Example sorted set with min/max score pagination.
"""
from uuid import uuid4
from redis import StrictRedis
from zato.redis_paginator import ZSetPaginator
conn = StrictRedis()
key = 'paginator:{}'.format(uuid4().hex)
# 97-114 is 'a' to 'r' in ASCII
for x in range(1, 18):
conn.zadd(key, x, chr(96 + x))
p = ZSetPaginator(conn, key, 2, score_min=5, score_max=13)
print(p.count) # 9
print(p.num_pages) # 5
print(p.page_range) # [1, 2, 3, 4, 5]
page = p.page(3)
print(page) # <Page 3 of 5>
print(page.object_list) # ['i', 'j']
conn.delete(key)
def test_group_chord_group_chain(self, manager):
from celery.five import bytes_if_py2
if not manager.app.conf.result_backend.startswith('redis'):
raise pytest.skip('Requires redis result backend.')
redis_connection = StrictRedis()
redis_connection.delete('redis-echo')
before = group(redis_echo.si('before {}'.format(i)) for i in range(3))
connect = redis_echo.si('connect')
after = group(redis_echo.si('after {}'.format(i)) for i in range(2))
result = (before | connect | after).delay()
result.get(timeout=TIMEOUT)
redis_messages = list(map(
bytes_if_py2,
redis_connection.lrange('redis-echo', 0, -1)
))
before_items = \
set(map(bytes_if_py2, (b'before 0', b'before 1', b'before 2')))
after_items = set(map(bytes_if_py2, (b'after 0', b'after 1')))
assert set(redis_messages[:3]) == before_items
assert redis_messages[3] == b'connect'
assert set(redis_messages[4:]) == after_items
redis_connection.delete('redis-echo')
class Redis_DB(object):
def __init__(self, host='localhost', port=6666):
self.host = host
self.port = port
self.redis = StrictRedis(host=self.host,
port=self.port,
db=0,
password='')
def write_data(self, key, value):
self.redis.set(key, value)
def get_data(self, key):
value = self.redis.get(key)
return value
def get_all_data(self):
all_keys = []
if self.redis.keys():
for i in self.redis.keys():
key = i.decode('ascii')
value = self.get_data(i).decode('ascii')
all_keys.append({key: value})
else:
all_keys = None
print('find total items:\n{}'.format(all_keys))
return all_keys
def delete_data(self, key):
self.redis.delete(key)
print('delete the key: {}'.format(key))
def test_send_event(redis_client: redis.StrictRedis):
logger.remove()
logger.add(
sink,
level=logging.INFO,
filter=lambda record: "event" in record["extra"],
)
bind = {
"event": "test_event",
"kwargs": {
"arg1": 1,
"arg2": [2],
"arg3": "你好",
},
}
redis_client.delete(KEY)
logger.bind(**bind).info("logger info")
assert redis_client.llen(KEY) == 1
e = redis_client.lpop(KEY)
data = msgpack.loads(e, raw=False)
for key, value in bind.items():
assert data[key] == value
assert data["@metadata"] == {"beat": "py_logging", "version": "dev"}
assert data["msg"] == "logger info"
def zset_score_min_max_example():
""" Example sorted set with min/max score pagination.
"""
from uuid import uuid4
from redis import StrictRedis
from zato.redis_paginator import ZSetPaginator
conn = StrictRedis()
key = 'paginator:{}'.format(uuid4().hex)
# 97-114 is 'a' to 'r' in ASCII
for x in range(1, 18):
conn.zadd(key, x, chr(96 + x))
p = ZSetPaginator(conn, key, 2, score_min=5, score_max=13)
print(p.count) # 9
print(p.num_pages) # 5
print(p.page_range) # [1, 2, 3, 4, 5]
page = p.page(3)
print(page) # <Page 3 of 5>
print(page.object_list) # ['i', 'j']
conn.delete(key)
def __admin_handler(self, endpoint: bytes):
"""
Handle Admin Request
:param bytes endpoint: Endpoint (in bytes!)
:return: jsonified answer data
"""
json_data = request.get_json()
if json_data is None:
raise BadRequest()
if 'admin_token' not in json_data:
raise BadRequest()
admin_token = json_data['admin_token']
if not isinstance(admin_token, str):
raise BadRequest()
if not RULE_TOKEN.match(admin_token):
raise BadRequest()
redis = StrictRedis(connection_pool=self.api_pool)
ep_key = 'ADMIN_TOKEN:{:s}'.format(admin_token)
should_endpoint = redis.get(ep_key)
if should_endpoint is None:
raise BadRequest()
redis.delete(ep_key)
if should_endpoint != endpoint:
raise BadRequest()
if 'data' not in json_data:
raise BadRequest()
data = json_data['data']
if not isinstance(data, dict):
raise BadRequest()
return jsonify(self.queue_dispatcher({
'_': 'admin:{:s}'.format(endpoint.decode('utf-8')),
'data': data,
}))
def clearRedis(keys):
pool = ConnectionPool(host='localhost', port='6379', db=0)
redis = StrictRedis(connection_pool=pool)
for key in keys:
if redis.exists(key):
redis.delete(key)
pool.disconnect()
class RedisPipelinedClient:
def __init__(self, host='127.0.0.1', port=6379, max_async=2):
self.r = StrictRedis(host=host, port=port,
db=0).pipeline(transaction=False)
self.max_async = max_async
self.count = 0
def _pipe_check(self):
self.count += 1
if self.count % self.max_async == 0:
self.r.execute()
def get(self, key):
self.r.get(key)
self._pipe_check()
def put(self, key, value):
self.r.set(key, value, nx=True)
self._pipe_check()
def update(self, key, value):
self.r.set(key, value, xx=True)
self._pipe_check()
def remove(self, key):
self.r.delete(key)
self._pipe_check()
class RedisDB(DBSpec):
def __init__(self, port=12000, prefix=None, sync_epoch=False):
self._server = StrictRedis(port=port)
self._prefix = "" if prefix is None else prefix
self._index = 0
self._epoch = 0
self._sync_epoch = sync_epoch
@property
def num_trajectories(self) -> int:
num_trajectories = self._server.llen("trajectories") - 1
return num_trajectories
def set_sample_flag(self, sample: bool):
self._server.set("sample_flag", int(sample))
def get_sample_flag(self) -> bool:
flag = int(self._server.get("sample_flag") or -1) == int(1)
return flag
def push_trajectory(self, trajectory):
trajectory = {"trajectory": trajectory, "epoch": self._epoch}
trajectory = pack(trajectory)
self._server.rpush("trajectories", trajectory)
def get_trajectory(self, index=None):
index = index if index is not None else self._index
trajectory = self._server.lindex("trajectories", index)
if trajectory is not None:
self._index = index + 1
trajectory = unpack(trajectory)
trajectory, trajectory_epoch = \
trajectory["trajectory"], trajectory["epoch"]
if self._sync_epoch and self._epoch != trajectory_epoch:
trajectory = None
return trajectory
def clean_trajectories(self):
self._server.delete("trajectories")
self._index = 0
def dump_weights(self, weights, prefix, epoch):
self._epoch = epoch
weights = {"weights": weights, "epoch": self._epoch}
weights = pack(weights)
self._server.set(f"{self._prefix}_{prefix}_weights", weights)
def load_weights(self, prefix):
weights = self._server.get(f"{self._prefix}_{prefix}_weights")
if weights is None:
return None
weights = unpack(weights)
self._epoch = weights.get("epoch")
return weights["weights"]
def clean_weights(self, prefix):
self._server.delete(f"{self._prefix}_{prefix}_weights")
class RedisManager(NoSqlManager):
def __init__(self,
namespace,
url=None,
data_dir=None,
lock_dir=None,
**params):
self.db = params.pop('db', None)
self.connection_pools = {}
NoSqlManager.__init__(self,
namespace,
url=url,
data_dir=data_dir,
lock_dir=lock_dir,
**params)
def open_connection(self, host, port, **params):
pool_key = self._format_pool_key(host, port, self.db)
if pool_key not in self.connection_pools:
self.connection_pools[pool_key] = ConnectionPool(host=host,
port=port,
db=self.db)
self.db_conn = StrictRedis(connection_pool=self.connection_pools[pool_key],
**params)
def __contains__(self, key):
return self.db_conn.exists(self._format_key(key))
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
#
# beaker.container.Value.set_value calls NamespaceManager.set_value
# however it (until version 1.6.4) never sets expiretime param.
#
# Checking "type(value) is tuple" is a compromise
# because Manager class can be instantiated outside container.py (See: session.py)
#
if (expiretime is None) and (type(value) is tuple):
expiretime = value[1]
if expiretime:
self.db_conn.setex(key, expiretime, pickle.dumps(value, 2))
else:
self.db_conn.set(key, pickle.dumps(value, 2))
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return 'beaker:%s:%s' % (self.namespace, key.replace(' ', '\302\267'))
def _format_pool_key(self, host, port, db):
return '{0}:{1}:{2}'.format(host, port, self.db)
def do_remove(self):
self.db_conn.flush()
def keys(self):
return self.db_conn.keys('beaker:%s:*' % self.namespace)
class CacheHelper(object):
"""A thin wrapper on top of Redis."""
def __init__(self, host="localhost", port=6379, db=0):
self.host = host
self.port = port
self.db = db
def __enter__(self):
self.r = StrictRedis(host=self.host, port=self.port, db=self.db)
return self
def __exit__(self, exc_type, exc_value, traceback):
try:
if any((exc_type, exc_value, traceback,)):
raise
finally:
pass
def get(self, key):
return self.r.get(key) if key in self.r.keys() else None
def set(self, key, json):
self.r.set(key, json)
def keys(self):
return self.r.keys()
def reset(self):
for key in self.keys():
self.r.delete(key)
class JobStatsTest(TestCase):
def setUp(self):
self.redis = StrictRedis(**settings.REDISES['default'])
self.stats = JobStats(self.redis)
def tearDown(self):
self.redis.delete('job_stats.1')
def testIncr(self):
job_id = 1
field = 'success'
count = self.stats.get(job_id, field)
self.assertEqual(0, count)
self.stats.incr(job_id, field, 1)
count = self.stats.get(job_id, field)
self.assertEqual(1, count)
self.stats.incr(job_id, field, 2)
count = self.stats.get(job_id, field)
self.assertEqual(3, count)
self.stats.incr(job_id, 'failed')
self.stats.get(job_id, field)
self.assertEqual(3, count)
def testGet(self):
job_id = 1
self.stats.incr(job_id, 'failed', 1)
self.stats.incr(job_id, 'success', 2)
counts = self.stats.get(job_id)
self.assertEqual(1, counts['failed'])
self.assertEqual(2, counts['success'])
class RedisQueue(object):
def __init__(self):
self.db = StrictRedis(host=REDIS_HOST,
port=REDIS_PORT,
password=REDIS_PASSWORD)
def add(self, request):
"""
向队列添加序列化后的Request
:param request: 请求对象
:return:
"""
if isinstance(request, WeixinRequest):
return self.db.rpush(REDIS_KEY, dumps(request))
return False
def pop(self):
"""
取出下一个Request并反序列化
:return:
"""
if self.db.llen(REDIS_KEY):
return loads(self.db.lpop(REDIS_KEY))
else:
return False
def clear(self):
self.db.delete(REDIS_KEY)
def empty(self):
return self.db.llen(REDIS_KEY) == 0
class RedisManager(NoSqlManager):
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
self.connection_pool = params.pop('connection_pool', None)
NoSqlManager.__init__(self, namespace, url=url, data_dir=data_dir, lock_dir=lock_dir, **params)
def open_connection(self, host, port, **params):
self.db_conn = StrictRedis(host=host, port=int(port), connection_pool=self.connection_pool, **params)
def __contains__(self, key):
log.debug('%s contained in redis cache (as %s) : %s'%(key, self._format_key(key), self.db_conn.exists(self._format_key(key))))
return self.db_conn.exists(self._format_key(key))
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
#XXX: beaker.container.Value.set_value calls NamespaceManager.set_value
# however it(until version 1.6.3) never sets expiretime param. Why?
if expiretime:
self.db_conn.setex(key, expiretime, pickle.dumps(value))
else:
self.db_conn.set(key, pickle.dumps(value))
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return 'beaker:%s:%s' % (self.namespace, key.replace(' ', '\302\267'))
def do_remove(self):
self.db_conn.flush()
def keys(self):
return self.db_conn.keys('beaker:%s:*' % self.namespace)
def zset_example():
""" Example sorted set pagination.
"""
from uuid import uuid4
from redis import StrictRedis
from zato.redis_paginator import ZSetPaginator
conn = StrictRedis()
key = 'paginator:{}'.format(uuid4().hex)
# 97-114 is 'a' to 'r' in ASCII
for x in range(1, 18):
conn.zadd(key, x, chr(96 + x))
p = ZSetPaginator(conn, key, 6)
print(p.count) # 17
print(p.num_pages) # 3
print(p.page_range) # [1, 2, 3]
page = p.page(3)
print(page) # <Page 3 of 3>
print(page.object_list) # ['m', 'n', 'o', 'p', 'q']
conn.delete(key)
def main():
# 创建一个StrictRedis对象，连接redis
try:
strict_redis = StrictRedis(host='114.67.89.253', port=6379, db=9, password="******")
# String 类型；设置key value
# set: 如果不存在key，则设置，如果存在，则修改
result = strict_redis.set(name="name", value="tom")
print(result)
# 获取值
name = strict_redis.get("name")
print("name: %s" % name)
# 删除key以及对应的值
result = strict_redis.delete("name")
print("delete: %d" % result)
# 删除多个key value
strict_redis.set(name="name", value="tom")
strict_redis.set(name="name1", value="tom1")
result = strict_redis.delete("name", "name1")
print("delete: %d" % result)
# 获取所有的key pattern默认为*
strict_redis.set(name="name", value="tom")
strict_redis.set(name="name1", value="tom1")
result = strict_redis.keys(pattern="*")
print(result)
pass
except Exception as e:
print(e)
pass
pass
class RedisHandler:
def __init__(self):
self._config = CacheConfig()
self._redis = StrictRedis(
host=self._config.host,
port=self._config.port,
password=self._config.password,
db=self._config.db,
)
@property
def redis(self):
return self._redis
@trace_service("Redis (save)", open_tracing)
def save(self, key: str, value: str):
self._redis.set(key, value)
@trace_service("Redis (find)", open_tracing)
def find_by_key(self, key: str) -> Optional[str]:
try:
value = self._redis.get(key).decode()
except:
value = None
return value
@trace_service("Redis (delete)", open_tracing)
def delete_by_key(self, key: str):
self._redis.delete(key)
def zset_example():
""" Example sorted set pagination.
"""
from uuid import uuid4
from redis import StrictRedis
from zato.redis_paginator import ZSetPaginator
conn = StrictRedis()
key = 'paginator:{}'.format(uuid4().hex)
# 97-114 is 'a' to 'r' in ASCII
for x in range(1, 18):
conn.zadd(key, x, chr(96 + x))
p = ZSetPaginator(conn, key, 6)
print(p.count) # 17
print(p.num_pages) # 3
print(p.page_range) # [1, 2, 3]
page = p.page(3)
print(page) # <Page 3 of 3>
print(page.object_list) # ['m', 'n', 'o', 'p', 'q']
conn.delete(key)
def check_with_ehall(jasdm: str, day: int, jc: str, zylxdm: str):
redis = StrictRedis(connection_pool=__Application.redis_pool)
lock = Lock(redis, "Spider")
if lock.acquire():
try:
save_cookies(), save_time()
cookies = json.loads(redis.hget("Spider", "cookies"))
time_info = json.loads(redis.hget("Spider", "time_info"))
redis.delete("Spider")
res = requests.post(
url=
"http://ehallapp.nnu.edu.cn/jwapp/sys/jsjy/modules/jsjysq/cxyzjskjyqk.do",
cookies=cookies,
data={
'XNXQDM': time_info['XNXQDM'][0],
'ZC': time_info['ZC'],
'JASDM': jasdm
}).json()
kcb = json.loads(
res['datas']['cxyzjskjyqk']['rows'][0]['BY1'])[(day + 6) % 7]
for row in kcb:
if jc in row['JC'].split(',') and row['ZYLXDM'] in (zylxdm,
''):
return True # 数据一致，待纠错
return False # 数据不一致，待更新
finally:
lock.release()
def main():
r = StrictRedis(unix_socket_path=get_socket_path('cache'))
r.delete('cache_loaded')
website_dir = get_homedir() / 'website'
ip = get_config('generic', 'website_listen_ip')
port = get_config('generic', 'website_listen_port')
try:
p = Popen([
'gunicorn', '-w', '10', '--graceful-timeout', '2', '--timeout',
'300', '-b', f'{ip}:{port}', '--log-level', 'info', 'web:app'
],
cwd=website_dir)
set_running('website')
while True:
if shutdown_requested() or p.poll() is not None:
break
time.sleep(1)
except KeyboardInterrupt:
print('Website killed by user.')
finally:
print('Shutting down website.')
try:
# Killing everything if possible.
p.send_signal(signal.SIGWINCH)
p.send_signal(signal.SIGTERM)
except Exception:
pass
unset_running('website')
class RedisDataStore(DataStore):
"""Redis-backed datastore object."""
def __init__(self, number=0):
redis_host = os.environ.get('REDIS_PORT_6379_TCP_ADDR')
redis_port = os.environ.get('REDIS_PORT_6379_TCP_PORT')
self.redis_conn = StrictRedis(host=redis_host, port=redis_port,
db=number)
def __setitem__(self, k, v):
self.redis_conn.set(k, v)
def __getitem__(self, k):
return self.redis_conn.get(k)
def __delitem__(self, k):
self.redis_conn.delete(k)
def get(self, k):
return self.redis_conn.get(k)
def __contains__(self, k):
return self.redis_conn.exists(k)
def todict(self):
#TODO(tvoran): use paginate
#TODO(tvoran): do something besides multiple gets
data = {}
for key in self.redis_conn.keys():
data[key] = self.get(key)
return data
def clear_all(self):
self.redis_conn.flushdb()
def verify(self, delete_cache=False):
redis_cli = StrictRedis(connection_pool=redis_pool)
code = redis_cli.get(name=self.phone) or b''
if code.decode('utf8') != self.verify_code:
return False
if delete_cache:
redis_cli.delete(self.phone)
return True
def verify_code(email, code):
cli = StrictRedis(connection_pool=pool)
e_code = cli.get(email)
if e_code == code:
cli.delete(email)
return True
else:
return False
def answer(data, module):
# Extract msg
msg = receive.parse_xml(data)
msg_type = msg.MsgType
to_user = msg.FromUserName
from_user = msg.ToUserName
if isinstance(msg, receive.TextMsg):
msg_content = msg.Content.decode('utf-8')
elif isinstance(msg, receive.EventMsg):
msg_content = msg.Event.decode('utf-8')
else:
msg_content = 'default'
# Initialize environment
global dialog_module
dialog_module = module
global hkey
hkey = dialog_module.REDIS_KEY % {'open_id': to_user}
global redis_db
redis_db = StrictRedis(host=dialog_module.REDIS_HOST,
port=dialog_module.REDIS_PORT,
password=dialog_module.REDIS_PASSWORD,
db=0)
hkey = dialog_module.REDIS_KEY % {'open_id': to_user}
hist = redis_db.get(hkey)
# 新会话或者会话超时，创建新会话
if not hist:
dialog = _new_dialog(msg_type, msg_content, to_user)
logger.debug('new_dialog')
# 存在会话记录，重现上下文
else:
logger.debug('replay_dialog')
try:
dialog = _replay_dialog(hist, to_user)
except StopIteration:
logger.error('会话记录错误..重新创建会话..')
dialog = _new_dialog(msg_type, msg_content, to_user)
# 发送消息
while True:
try:
type, msg = _redis_send(hkey, dialog, msg_content)
break
except StopIteration as e:
# 会话已结束，删去redis中的记录
type, msg = e.value
redis_db.delete(hkey)
break
except UnexpectAnswer:
# 用户发送了一个不合法的回复时抛出这个异常
# BOT会认为用户希望开启一段新的会话
redis_db.delete(hkey)
dialog = _new_dialog(msg_type, msg_content, to_user)
continue
wechat_reply = getattr(reply, type)
print(wechat_reply(to_user, from_user, msg).format())
return wechat_reply(to_user, from_user, msg)
class RedisManager(NoSqlManager):
connection_pools = {}
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
self.db = params.pop("db", None)
self.dbpass = params.pop("password", None)
NoSqlManager.__init__(self, namespace, url=url, data_dir=data_dir, lock_dir=lock_dir, **params)
def open_connection(self, host, port, **params):
pool_key = self._format_pool_key(host, port, self.db)
if pool_key not in self.connection_pools:
self.connection_pools[pool_key] = ConnectionPool(host=host, port=port, db=self.db, password=self.dbpass)
self.db_conn = StrictRedis(connection_pool=self.connection_pools[pool_key], **params)
def __contains__(self, key):
return self.db_conn.exists(self._format_key(key))
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
#
# beaker.container.Value.set_value calls NamespaceManager.set_value
# however it (until version 1.6.4) never sets expiretime param.
#
# Checking "type(value) is tuple" is a compromise
# because Manager class can be instantiated outside container.py (See: session.py)
#
if (expiretime is None) and (type(value) is tuple):
expiretime = value[1]
if self.serializer == "json":
serialized_value = json.dumps(value, ensure_ascii=True)
else:
serialized_value = pickle.dumps(value, 2)
if expiretime:
self.db_conn.setex(key, expiretime, serialized_value)
else:
self.db_conn.set(key, serialized_value)
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return "beaker:%s:%s" % (self.namespace, key.replace(" ", "\302\267"))
def _format_pool_key(self, host, port, db):
return "{0}:{1}:{2}".format(host, port, self.db)
def do_remove(self):
for key in self.keys():
self.db_conn.delete(key)
def keys(self):
return self.db_conn.keys("beaker:%s:*" % self.namespace)
def do_deletes():
"""DELETE"""
db_pipe = StrictRedis().pipeline()
print 'performing a few deletes...'
del_keys = (260, 271, 285, 300, 310)
for key in del_keys:
db_pipe.delete(key)
print 'db_pipe.command_stack:\n{}'.format(db_pipe.command_stack)
db_pipe.execute()
def post(self, request):
key = request.data['course_code']
r = StrictRedis(host='localhost', port=6379, db=0)
r.delete(key)
redis_can_take_course = RedisList(key)
can_take_course = set(request.data['can_take_course'])
for item in can_take_course:
redis_can_take_course.append(item.upper())
return Response({'success': 'can_take_course'})
def get(self, request):
r = StrictRedis(host='localhost', port=6379, db=0)
r.delete(r.keys('active_class:*')[0])
try:
staff = User.objects.get(is_staff=True)
Token.objects.filter(~Q(user=staff)).delete()
except MultipleObjectsReturned:
Token.objects.all().delete()
# warn the admin of multiple staff entry possibly due to breach of password
return Response('Class ends successfully')
class RedisStorage(object):
def __init__(self, host, port):
self.connection = StrictRedis(host=host, port=port)
def get(self, item_id):
return self.connection.get(item_id)
def set(self, item_id, item_value):
self.connection.rpush(item_id, item_value)
def rm(self, item_id):
self.connection.delete(item_id)
class RedisDict(MutableMapping):
""" RedisDict - a dictionary-like interface for ``redis`` key-stores
"""
def __init__(self, namespace, collection_name='redis_dict_data',
connection=None):
"""
The actual key name on the redis server will be
``namespace``:``collection_name``
In order to deal with how redis stores data/keys,
everything, i.e. keys and data, must be pickled.
:param namespace: namespace to use
:param collection_name: name of the hash map stored in redis
(default: redis_dict_data)
:param connection: ``redis.StrictRedis`` instance.
If it's ``None`` (default), a new connection with
default options will be created
"""
if connection is not None:
self.connection = connection
else:
self.connection = Redis()
self._self_key = ':'.join([namespace, collection_name])
def __getitem__(self, key):
result = self.connection.hget(self._self_key, pickle.dumps(key))
if result is None:
raise KeyError
return pickle.loads(bytes(result))
def __setitem__(self, key, item):
self.connection.hset(self._self_key, pickle.dumps(key),
pickle.dumps(item))
def __delitem__(self, key):
if not self.connection.hdel(self._self_key, pickle.dumps(key)):
raise KeyError
def __len__(self):
return self.connection.hlen(self._self_key)
def __iter__(self):
for v in self.connection.hkeys(self._self_key):
yield pickle.loads(bytes(v))
def clear(self):
self.connection.delete(self._self_key)
def __str__(self):
return str(dict(self.items()))
class TestRedisHashDict(TestCase):
def setUp(self):
self.redis = StrictRedis()
self.redis.delete('foobar')
self.dict = RedisHashDict(self.redis, 'foobar')
def test_items(self):
self.dict[b'foo'] = b'bar'
self.dict[b'bob'] = b'alice'
truth = (b'foo', b'bar'), (b'bob', b'alice')
for item in self.dict.items():
self.assertIn(item, truth)
def test_Redis(self):
host = "localhost"
r = StrictRedis(host, socket_timeout=60)
msg = "Hello world"
r.delete("channel")
r.rpush("channel", msg)
m = r.lpop("channel")
print m
assert(m == msg)
r.rpush("channel", [1, 2, 3])
m = ast.literal_eval(r.lpop("channel"))
print m, type(m)
class TestRModels(object):
def setup_method(self, method):
self.rc = StrictRedis()
self.disabled = False
try:
self.rc.ping()
except ConnectionError:
self.disabled = True
self.prefix = "copr:test:r_models"
self.time_now = time.time()
def teardown_method(self, method):
if self.disabled:
return
keys = self.rc.keys('{}*'.format(self.prefix))
if keys:
self.rc.delete(*keys)
def test_timed_stats_events(self):
if self.disabled:
return
TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
timestamp=self.time_now, )
assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 1
TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
timestamp=self.time_now, count=2)
assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 3
TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
timestamp=self.time_now - 1000000, count=2)
TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
timestamp=self.time_now - 3000000, count=3)
assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 3
assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,
day_min=self.time_now - 2000000) == 5
assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,
day_min=self.time_now - 5000000) == 8
TimedStatEvents.trim_before(self.rc, name="foobar",
prefix=self.prefix, threshold_timestamp=self.time_now - 200000)
assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,
day_min=self.time_now - 5000000) == 3
class TaskCache(ITaskCache):
"""
`self.user_store': {'user_id': <list of task_ids>}
`self.task_store`: {'task_id`: <AsyncResult>}
`self.progress_store`: {`task_id`: `progress_value(int)`}
"""
def __init__(self, user_kwargs, task_kwargs ,progress_kwargs):
self.user_store = StrictRedis(**user_kwargs)
self.task_store = StrictRedis(**task_kwargs)
self.progress_store = LingvodocRedisClient(**progress_kwargs)
def get(self, user, remove_finished=False):
result = dict()
tasks = self.user_store.get(user.id)
if tasks is None:
return {}
tasks = pickle.loads(tasks)
remained_tasks = list()
for t in tasks:
val = self.task_store.get(t)
if val is None:
continue
async_result = pickle.loads(val)
progress = self.progress_store.get(t)
# Redis client returns byte array. We need to decode it
if progress is not None:
progress = int(progress.decode())
result[t] = {'finished': async_result.ready(),
'progress': progress}
if remove_finished:
if async_result.ready():
self.task_store.delete(t)
else:
remained_tasks.append(t)
if remove_finished:
self.user_store.set(user.id, pickle.dumps(remained_tasks))
return result
# TODO: add try/catch handlers.
# we should remove the task from caches (and queue?) if exception is raised
def set(self, user, task_key, async_task):
self.task_store.set(task_key, pickle.dumps(async_task))
cached = self.user_store.get(user.id)
if cached is None:
tmp_tasks = [task_key]
else:
tmp_tasks = pickle.loads(cached)
tmp_tasks.append(task_key)
self.user_store.set(user.id, pickle.dumps(tmp_tasks))
class Repository:
def __init__(self, host, port):
self.r = StrictRedis(host=host, port=port)
# store dictionary
# redis.hmset(hkey, dict_to_store)
# retrieve dictionary
# redis.hmget(hkey)
# lists
# redis.lappend(hkey, string)
# redis.llen
def get_item(self, key_store, index):
length = self.r.llen(key_store)
if length == 0:
return None
if index >= length:
raise Exception('Index out of range.')
item_json = self.r.lindex(key_store, length - (index + 1))
item_dict = json.loads(item_json)
return item_dict
def append_item(self, key_store, item):
item_json = json.dumps(item)
if not self.r.lpush(key_store, item_json):
raise Exception('Unable to write key_store: [%s]' % item_json)
def fetch_all(self, key_store):
q = []
length = self.r.llen(key_store)
for i in range(length):
item_json = self.r.lindex(key_store, length - (i + 1))
item_dict = json.loads(item_json)
q.append(item_dict)
return q
def set(self, key_store, list):
self.r.delete(key_store)
for item_dict in list:
item_json = json.dumps(item_dict)
if not self.r.lpush(key_store, item_json):
raise Exception('Unable to write key_store: [%s]' % item_json)
def delete(self, key_store):
self.r.delete(key_store)
def info(self):
return self.r.info()
def delete(obj, key=None):
"""
Delete a single key if specified, or all namespace if key is none
:param obj: settings object
:param key: key to delete from store location
:return: None
"""
client = StrictRedis(**obj.REDIS_FOR_DYNACONF)
holder = "DYNACONF_%s" % obj.DYNACONF_NAMESPACE
if key:
client.hdel(holder.upper(), key.upper())
obj.unset(key)
else:
keys = client.hkeys(holder.upper())
client.delete(holder.upper())
obj.unset_all(keys)
class VisualDebugger:
def __init__(self):
self.available_buckets = config["visual_debugger"]["available_buckets"]
self.bucket_generator = itertools.cycle(self.available_buckets)
self.redis_client = StrictRedis(**config["redis"])
self.clear_image_data()
def store_image_data(self, image_data, image_shape, bucket="debug"):
self.redis_client.lpush(f"{config['visual_debugger']['redis_key_prefix']}:{bucket}:SHAPE", pickle.dumps(image_shape))
self.redis_client.lpush(f"{config['visual_debugger']['redis_key_prefix']}:{bucket}", image_data.tobytes())
def retrieve_image_data(self):
bucket = next(self.bucket_generator)
bucket_key = f"{config['visual_debugger']['redis_key_prefix']}:{bucket}"
response = self.redis_client.rpop(bucket_key)
if response is not None:
bucket = bucket_key.split(":")[-1]
image_shape = self.redis_client.rpop(f"{config['visual_debugger']['redis_key_prefix']}:{bucket}:SHAPE")
image_shape = pickle.loads(image_shape)
image_data = np.fromstring(response, dtype="uint8").reshape(image_shape)
return bucket, image_data
return None
def save_image_data(self, bucket, image_data):
if bucket in self.available_buckets:
if image_data.dtype == "bool" or (image_data.dtype == "uint8" and 1 in np.unique(image_data)):
image_data = image_data.astype("uint8") * 255
skimage.io.imsave(f"{bucket}.png", image_data)
def clear_image_data(self):
visual_debugger_keys = self.redis_client.keys(f"{config['visual_debugger']['redis_key_prefix']}*")
for key in visual_debugger_keys:
self.redis_client.delete(key.decode("utf-8"))
def get_bucket_queue_length(self, bucket):
return self.redis_client.llen(f"{config['visual_debugger']['redis_key_prefix']}:{bucket}")
class RedisCache(object):
def __init__(self, params={}):
self._validate(params)
if not self.server:
raise Exception('Redis Server Not Defined')
try:
log.debug('Connecting to redis at [%s]?[%s]' % (self.server, self.database))
self.cache = StrictRedis(self.server, port=self.port, db=self.database)
except ConnectionError as ex:
raise Exception("Unable to connect to Redis", ex)
def get(self, key):
"""
Fetch a given key from the cache. If the key does not exist, return
default, which itself defaults to None.
"""
ckey = self._create_key(key)
log.debug("Getting the cache key [%s]" % ckey)
return self.cache.get(ckey)
def ping(self):
"""
This command is often used to test if the cache is still alive, or to measure latency.
"""
log.debug("Ping to the cache")
return self.cache.ping()
def store(self, key, value, expires=None):
"""
Set a value in the cache. If timeout is given, that timeout will be
used for the key; otherwise the default cache timeout will be used.
"""
ckey = self._create_key(key)
log.debug("Storing the cache key [%s]" % ckey)
return self.cache.set(ckey, value, ex=expires)
def delete(self, key):
"""
Delete a key from the cache, failing silently.
"""
ckey = self._create_key(key)
log.debug("Deleting the cache key [%s]" % ckey)
return self.cache.delete(ckey)
def _validate(self, params):
"""
Initialize all the needed parameters
"""
self.server = params.get('server', 'localhost')
self.port = params.get('port', 6379)
self.database = params.get('database', 2)
self.key_prefix = params.get('key_prefix', 'domus')
def _create_key(self, key):
return "%s.%s" % (self.key_prefix, key)
class RedisManager(NoSqlManager):
def __init__(self, namespace, url=None, data_dir=None, lock_dir=None, **params):
self.expiretime = params.pop('expiretime', None)
NoSqlManager.__init__(self, namespace, url=url, data_dir=data_dir, lock_dir=lock_dir, **params)
def open_connection(self, host, port, **params):
self.db_conn = StrictRedis(host=host, port=int(port), **params)
def __getitem__(self, key):
return pickle.loads(self.db_conn.hget(self._format_key(key), 'data'))
def __contains__(self, key):
return self.db_conn.exists(self._format_key(key))
def set_value(self, key, value, expiretime=None):
key = self._format_key(key)
#
# beaker.container.Value.set_value calls NamespaceManager.set_value
# however it (until version 1.6.4) never sets expiretime param.
#
# Checking "type(value) is tuple" is a compromise
# because Manager class can be instantiated outside container.py (See: session.py)
#
if (expiretime is None) and (type(value) is tuple):
expiretime = value[1]
self.db_conn.hset(key, 'data', pickle.dumps(value))
self.db_conn.hset(key, 'accessed', datetime.now())
self.db_conn.hsetnx(key, 'created', datetime.now())
if expiretime or self.expiretime:
self.db_conn.expire(key, expiretime or self.expiretime)
def __delitem__(self, key):
self.db_conn.delete(self._format_key(key))
def _format_key(self, key):
return 'beaker:%s:%s' % (self.namespace, key.replace(' ', '\302\267'))
def do_remove(self):
self.db_conn.flushdb()
def keys(self):
return self.db_conn.keys('beaker:%s:*' % self.namespace)
class RedisConsumer(object):
def __init__(self, timeout, group_id, hostname = 'localhost', port = 6379):
self.group_id = group_id
self.timeout = timeout
log.debug("Initializing RedisConsumer with hostname of %s and port %s" % (hostname, port))
self.r = StrictRedis(host = hostname, port = port)
def poll(self, topic):
result = None
current_index_key = get_next_index_for_topic_key(topic)
end_millis = time.time() * 1000 + self.timeout
log.debug("Polling topic %s" % topic)
while time.time() * 1000 < end_millis:
if self.r.exists(current_index_key):
current_index = long(self.r.get(current_index_key))
consumer_index_key = get_next_index_for_group_id_key(topic, self.group_id)
pl = self.r.pipeline()
pl.watch(consumer_index_key)
consumer_index = 0
if self.r.exists(consumer_index_key):
consumer_index = long(self.r.get(consumer_index_key))
if current_index > consumer_index:
try:
pl.multi()
pl.incr(consumer_index_key)
incr_result = pl.execute()
if not incr_result is None and len(incr_result) > 0:
consumer_index = long(incr_result[0])
key = get_message_key(topic, consumer_index)
if self.r.exists(key):
result = self.r.get(key)
break
except WatchError:
log.debug("Redis keys changed for topic %s and group %s, trying again" % (topic, self.group_id))
pass
return result
def unsubscribe_from_topic(self, topic):
self.r.delete(get_next_index_for_group_id_key(topic, self.group_id))
class SnapshotWriterTest(TestCase):
def setUp(self):
self.snapshot_queue = Queue()
self.db = StrictRedis(host=REDIS['host'], port=REDIS['port'], db=SNAPSHOTS_DB)
self.writer = SnapshotWriter(self.snapshot_queue)
def test_should_read_snapshots_from_queue_periodically_and_write_them_to_redis_db(self):
self.writer.start()
snapshot_one = {'id': 1, 'other_details': 'Vehicle data', 'time': 1.0}
snapshot_two = {'id': 2, 'other_details': 'Vehicle data', 'time': 1.0}
[self.snapshot_queue.put(item) for item in [snapshot_one, snapshot_two]]
sleep(0.3)
self.writer.shutdown()
snapshots_in_db = [self.db.hgetall(key) for key in ['vehicle:1:1.000000', 'vehicle:2:1.000000']]
expected_snapshots = [self.redisify_snapshot(snapshot) for snapshot in [snapshot_one, snapshot_two]]
self.assertListEqual(snapshots_in_db, expected_snapshots)
def test_should_write_snapshot_with_key_a_concatenation_of_vehicle_id_and_time_stamp_to_six_decimal_places(self):
self.writer.start()
snapshot = {'id': 1, 'other_details': 'Vehicle data', 'time': 1.123459881213}
self.snapshot_queue.put(snapshot)
sleep(0.1)
self.writer.shutdown()
self.assertTrue('vehicle:1:1.123460' in self.db.keys("*"))
def test_should_stop_reading_snapshots_if_shutdown_is_requested(self):
writer = SnapshotWriter(self.snapshot_queue)
writer.start()
writer.shutdown()
self.assertTrue(True) # Tests that we get here
def redisify_snapshot(self, snapshot):
snapshot.update({'id': str(snapshot['id']), 'time': "%.1f" % snapshot['time']})
return snapshot
def tearDown(self):
for key in self.db.keys("*"):
self.db.delete(key)