def redis():
rd = Redis()
keys = set(rd.keys())
yield rd
to_del = [k for k in rd.keys() if k not in keys]
if to_del:
rd.delete(*to_del)
class RedisClient:
conn = ""
CHANNEL = "key-update"
pubsub = ""
def __init__(self, db_path):
os.makedirs(os.path.dirname(db_path), exist_ok=True)
try:
if not self.conn:
self.conn = Redis(db_path)
except:
raise Exception("Unable to create connection with Redis server")
def get_key(self, key):
return self.conn.get(key)
def set_key(self, key, value, publish_update=False):
if publish_update:
self.publish_update(message=json.dumps({key: value}))
return self.conn.set(key, value)
def remove_key(self, key):
return self.conn.delete(key)
def publish_update(self, channel=CHANNEL, message=""):
self.conn.pubsub()
self.conn.publish(channel, message)
def get_update(self, channel=CHANNEL):
if not self.pubsub:
self.pubsub = self.conn.pubsub()
self.pubsub.subscribe(channel)
return self.pubsub.get_message(channel)
class HotQueue(object):
"""Simple FIFO message queue stored in a Redis list.
Parameters
----------
name : str
name of the queue
max_queue_length : int
Maximum length the queue can grow to (default is None allows the queue
to grow without any limits.
serializer : class, module, optional
the class or module to serialize msgs with, must have
methods or functions named ``dumps`` and ``loads``,
`pickle <http://docs.python.org/library/pickle.html>`_ is the default,
use ``None`` to store messages in plain text (suitable for strings,
integers, etc)
redis : redis.Redis, redislite.Redis, optional
redis connection object, defaults to redislite.Redis with fallback to
redis.Redis.
**kwargs
Additional kwargs to pass to :class:`redislite.Redis`, most commonly
:attr:`dbfilename`.
Examples
--------
>>> from hotqueue import HotQueue
>>> queue = HotQueue("myqueue", dbfilename="queue.rdb")
"""
def __init__(
self, name, serializer=pickle, redis=None, max_queue_length=None,
**kwargs
):
self.name = name
self.serializer = serializer
self.max_queue_length = max_queue_length
if redis:
self.__redis = redis
else:
self.__redis = Redis(**kwargs)
def __len__(self):
return self.__redis.llen(self.key)
@property
def key(self):
"""
Key in Redis to store the queue
Returns
-------
str
The name of the key containing the queue in redis.
"""
return key_for_name(self.name)
def clear(self):
"""
Clear the queue of all messages, by deleting the Redis key.
"""
self.__redis.delete(self.key)
def consume(self, **kwargs):
"""
A blocking generator that yields whenever a message is waiting in the
queue.
Parameters
----------
**kwargs
any arguments that :meth:`~hotqueue.HotQueue.get` can
accept (:attr:`block` will default to ``True`` if not given)
Yields
------
object
The deserialized object from the queue.
Examples
--------
>>> queue = HotQueue("example")
>>> for msg in queue.consume(timeout=1):
... print(msg)
my message
another message
"""
kwargs.setdefault('block', True)
try:
while True:
msg = self.get(**kwargs)
if msg is None:
break
yield msg
except KeyboardInterrupt: # pragma: no cover
print()
return
def get(self, block=False, timeout=None):
"""
Get a message from the queue.
Parameters
----------
block : bool
whether or not to wait until a msg is available in
the queue before returning; ``False`` by default
timeout : int
When using :attr:`block`, if no msg is available
for :attr:`timeout` in seconds, give up and return
Returns
-------
object
The deserialized object from the queue.
Examples
--------
>>> queue.get()
'my message'
>>> queue.get()
'another message'
"""
if block:
if timeout is None:
timeout = 0
msg = self.__redis.blpop(self.key, timeout=timeout)
if msg is not None:
msg = msg[1]
else:
msg = self.__redis.lpop(self.key)
if msg is not None and self.serializer is not None:
msg = self.serializer.loads(msg)
if isinstance(msg, bytes):
msg = msg.decode()
return msg
def put(self, *msgs):
"""Put one or more messages onto the queue. Example:
>>> queue.put("my message")
>>> queue.put("another message")
To put messages onto the queue in bulk, which can be significantly
faster if you have a large number of messages:
>>> queue.put("my message", "another message", "third message")
"""
if self.serializer is not None:
msgs = [self.serializer.dumps(m) for m in msgs]
self.__redis.rpush(self.key, *msgs)
if self.max_queue_length:
self.__redis.ltrim(self.key, 0, int(self.max_queue_length) - 1)
def worker(self, *args, **kwargs):
"""Decorator for using a function as a queue worker. Example:
>>> @queue.worker(timeout=1)
... def printer(msg):
... print(msg)
>>> printer()
my message
another message
You can also use it without passing any keyword arguments:
>>> @queue.worker
... def printer(msg):
... print(msg)
>>> printer()
my message
another message
:param kwargs: any arguments that :meth:`~hotqueue.HotQueue.get` can
accept (:attr:`block` will default to ``True`` if not given)
"""
def decorator(worker):
"""
Worker decorator
:param worker:
:return:
"""
@wraps(worker)
def wrapper(*args):
"""
Inner wrapper
:param args:
:return:
"""
for msg in self.consume(**kwargs):
worker(*args + (msg,))
return wrapper
if args:
return decorator(*args)
return decorator
class RedisStore(DataStore):
"""Implementation of Redis datastore"""
def __init__(self):
super().__init__()
if config.settings.stackl_redis_type == "fake":
logger.info("Using fake client")
self.redis = Redis()
else:
self.redis = redis.Redis(
host=config.settings.stackl_redis_host,
port=config.settings.stackl_redis_port,
password=config.settings.stackl_redis_password,
db=0)
def get(self, **keys):
"""Gets a document from a redis instance"""
document_key = keys.get("category") + '/' + keys.get(
"type") + '/' + keys.get("name")
logger.debug(f"[RedisStore] get on key '{document_key}'")
redis_value = self.redis.get(document_key)
if redis_value is None:
response = self._create_store_response(
status_code=StatusCode.NOT_FOUND, content={})
else:
content = json.loads(self.redis.get(document_key))
response = self._create_store_response(status_code=StatusCode.OK,
content=content)
logger.debug(f"[RedisStore] StoreResponse for get: {response}")
return response
def get_all(self, category, document_type, wildcard_prefix=""):
"""Gets all documents of a type from a Redis"""
document_key = f"{category}/{document_type}/{wildcard_prefix}*"
logger.debug(
f"[RedisStore] get_all in '{document_key}' for type '{document_type}'"
)
content = []
for key in self.redis.scan_iter(document_key):
content.append(json.loads(self.redis.get(key)))
response = self._create_store_response(status_code=StatusCode.OK,
content=content)
logger.debug(f"[RedisStore] StoreResponse for get: {response}")
return response
def get_history(self, category, document_type, name):
"""Gets the snapshots of document from Redis"""
document_key = category + '/' + document_type + '/' + name
logger.debug(
f"[RedisStore] get_history in '{document_key}' for type '{document_type}'"
)
content = []
for key in self.redis.scan_iter(document_key):
content.append(json.loads(self.redis.get(key)))
response = self._create_store_response(status_code=StatusCode.OK,
content=content)
logger.debug(f"[RedisStore] StoreResponse for get: {response}")
return response
def put(self, file):
"""Puts a document in Redis"""
document_key = file.get("category") + '/' + file.get(
"type") + '/' + file["name"]
logger.debug(f"[RedisStore] put on '{document_key}' with file {file}")
self.redis.set(document_key, json.dumps(file))
response = self._create_store_response(
status_code=StatusCode.CREATED,
content=json.loads(self.redis.get(document_key)))
logger.debug(f"[RedisStore] StoreResponse for put: {response}")
return response
def delete(self, **keys):
"""Deletes a document in Redis"""
document_key = keys.get("category") + '/' + keys.get(
"type") + '/' + keys.get("name")
self.redis.delete(document_key)
response = self._create_store_response(status_code=200, content={})
logger.debug(f"[RedisStore] StoreResponse for delete: {response}")
return response
