class RedisSafetyStore(object):
def __init__(self):
self.redisClient = Redis(connection_pool=BlockingConnectionPool())
def isSafe(self, url):
# if the urlstring is in the store, then it is not safe.
try:
notSafe = self.redisClient.get(url)
except:
print("Redis read error")
raise
if notSafe:
return False
else:
return True
def addUnsafeResources(self, jsonContent):
for resource in jsonContent["resources"]:
try:
self.redisClient.set(resource, 1)
except:
print("Redis write error")
raise
return True
def uget(self, name):
"""
Return the value and key ``name`` or None, and decode it if not None.
"""
value = Redis.get(self, name)
if value:
return value.decode(self.encoding)
return value
class Cache:
""" Cache class
"""
def __init__(self):
""" Store an instance of the Redis client as a private variable
named _redis
"""
self._redis = Redis()
self._redis.flushdb()
@call_history
@count_calls
def store(self, data: Union[str, bytes, int, float]) -> str:
""" Takes a data argument and returns a string
"""
key = str(uuid4())
self._redis.set(key, data)
return key
def get(self, key: str, fn: Optional[Callable] = None) ->\
Union[str, bytes, int, float]:
""" Take a key string argument and an optional Callable argument
named fn
"""
if key:
result = self._redis.get(key)
if fn:
return fn(result)
else:
return result
def get_str(self, data: bytes) -> str:
""" Will automatically parametrize Cache.get with the correct
conversion function
"""
return data.decode('utf-8')
def get_int(self, data: bytes) -> int:
""" Will automatically parametrize Cache.get with the correct
conversion function
"""
byte_order = sys.byteorder
return int.from_bytes(data, byte_order)
class Cache:
"""Redis Class"""
def __init__(self):
"""constructor"""
self._redis = Redis()
self._redis.flushdb()
@call_history
@count_calls
def store(self, data: Union[str, bytes, int, float]) -> str:
"""returns a string"""
key = str(uuid.uuid4())
self._redis.set(key, data)
return key
def get(self, key: str, fn: Optional[Callable] = None) ->\
Union[str, bytes, int, float]:
"""get method"""
if key:
result = self._redis.get(key)
return fn(result) if fn else result
class AnguisRedis(AnguisBase):
def __init__(self, host='localhost', port=6379, db=0, *args, **kwargs):
self.r = Redis(host, port, db)
super(AnguisRedis, self).__init__()
def __del__(self):
super(AnguisRedis, self).__del__()
def __getitem__(self, key):
return self.unserialize(self.r.get(key))
def __setitem__(self, key, obj):
self.r.set(key, self.serialize(obj))
def __delitem__(self, key):
return self.r.delete(key)
def __iter__(self):
return iter(self.r.keys())
def __len__(self):
return len(self.r.keys())
class Cache:
"""Cache class"""
def __init__(self):
"""constructor"""
self._redis = Redis()
self._redis.flushdb()
@call_history
@count_calls
def store(self, data: Union[str, bytes, int, float]) -> str:
"""Store data in cache"""
key = str(uuid.uuid4())
self._redis.set(key, data)
return key
def get(self, key: str, fn: Optional[Callable] = None) ->\
Union[str, bytes, int, float]:
"""get data from fuction or cache"""
if key:
result = self._redis.get(key)
if fn:
return fn(result)
else:
return result
def increment():
red = Redis()
print(red.get('key') is None)
red.set('key', 1)
red.incr('key', 2)
def getInt(r: Redis, key: str):
value = r.get(key)
return 0 if value is None else int(value)
# -*- coding: utf-8 -*-
__author__ = 'fjs'
from connection import BlockingConnectionPool
from redis.client import Redis
client = Redis(connection_pool=BlockingConnectionPool(max_connections=2))
client.set("fjs", "fjs")
print client.get("fjs")
client.lpush("nn", 1)
print client.lpop("nn")
class TSStore(object):
def __init__(self, config):
self._redis = Redis(host=config.get('redis','host'),
port=int(config.get('redis','port')),
db=int(config.get('redis','db')))
self._delta_secs = int(eval(config.get('timeseries',
'delta_secs')))
self._expiration_delay_secs = int(eval(config.get('timeseries',
'expiration_delay_secs')))
def queries_key(self):
return 'queries'
@property
def queries(self):
return self._redis.smembers(self.queries_key())
@queries.setter
def queries(self, values):
pipe = self._redis.pipeline()
pipe.delete(self.queries_key())
for v in values:
pipe.sadd(self.queries_key(),
v)
return pipe.execute()
def _interval_key(self, timestamp):
return int(timestamp) - int(timestamp) % self._delta_secs
def _ts_key(self, timestamp, query):
return 'ts:%(query)s:%(timestamp_key)s'%{'query':query,
'timestamp_key':self._interval_key(timestamp)}
def _tweet_key(self, t):
if type(t) == Tweet:
return 'tweet:%s'%t.id
return 'tweet:%s'%t
def _query_key(self, query):
return 'query:%s:last_tweet_id'%query
def _store_tweet(self, pipe, tweet):
tweet_key = self._tweet_key(tweet)
pipe.set(tweet_key, tweet.serialize())
pipe.expire(tweet_key, self._expiration_delay_secs)
def _reference_tweet(self, pipe, timestamp, query, tweet):
ts_key = self._ts_key(timestamp, query)
pipe.lpush(ts_key,tweet.id)
pipe.expire(ts_key,self._expiration_delay_secs)
def _update_last_query_tweet(self, pipe, query, tweet):
query_key = self._query_key(query)
pipe.set(query_key,tweet.id)
def append(self, query, tweet):
pipe = self._redis.pipeline()
timestamp = time.time()
self._store_tweet(pipe, tweet)
self._reference_tweet(pipe, timestamp, query, tweet)
self._update_last_query_tweet(pipe, query, tweet)
return pipe.execute()
def retrieve_ts(self, query, timestamp, n_elements=-1):
ts_key = self._ts_key(timestamp, query)
return self._redis.lrange(ts_key, 0, n_elements)
def retrieve_last_tweet_id(self, query):
query_key = self._query_key(query)
return self._redis.get(query_key)
def retrieve_tweet(self, tweet_id):
tweet_key = self._tweet_key(tweet_id)
data = self._redis.get(tweet_key)
return Tweet.deserialize(data).todict()
def retrieve(self, query, n_periods=30):
current_timestamp = now = int(time.time())
start_timestamp = now - self._delta_secs * n_periods
tweets = []
while current_timestamp > start_timestamp:
current_tweet_ids = self.retrieve_ts(query, current_timestamp)
tweets.append({'timestamp': current_timestamp,
'tweets' : [ self.retrieve_tweet(tid) for tid in current_tweet_ids ] })
current_timestamp -= self._delta_secs
return { 'now' : now,
'ts' : tweets }
# -*- coding: utf-8 -*-
__author__ = 'fjs'
from connection import BlockingConnectionPool
from redis.client import Redis
client = Redis(connection_pool=BlockingConnectionPool(max_connections=2))
client.set("fjs", "fjs")
print client.get("fjs")
client.lpush("nn", 1)
print client.lpop("nn")
class BaseStorage(object):
"""Base class for a storage engine.
Such objects provide a general abstraction over various storage backends
(in-memory dict, remote cache, ...)
"""
def __init__(self, client=None):
if not client:
self.client = Redis(host='localhost',
port=6379,
db=13,
password=None)
else:
self.client = client
def get(self, key, default=0.0):
"""Retrieve the current value for a key.
Args:
key (str): the key whose value should be retrieved
default (object): the value to use when no value exist for the key
"""
result = self.client.get(key)
if not result:
return default
else:
return float(result)
# raise NotImplementedError()
def set(self, key, value):
"""Set a new value for a given key."""
return self.client.set(key, value)
# raise NotImplementedError()
def mget(self, *keys, **kwargs):
"""Retrieve values for a set of keys.
Args:
keys (str list): the list of keys whose value should be retrieved
Keyword arguements:
default (object): the value to use for non-existent keys
coherent (bool): whether all fetched values should be "coherent",
i.e no other update was performed on any of those values while
fetching from the database.
Yields:
object: values for the keys, in the order they were passed
"""
default = kwargs.get('default')
coherent = kwargs.get('coherent', False)
for key in keys:
yield self.get(key, default=default)
def mset(self, values):
"""Set the value of several keys at once.
Args:
values (dict): maps a key to its value.
"""
for key, value in values.items():
self.set(key, value)
def incr(self, key, amount=1, default=0):
"""Increment the value of a key by a given amount.
Also works for decrementing it.
Args:
key (str): the key whose value should be incremented
amount (int): the amount by which the value should be incremented
default (int): the default value to use if the key was never set
Returns:
int, the updated value
"""
# import pdb;pdb.set_trace()
value = int(self.get(key, default)) + int(amount)
self.set(key, value)
return value
class KeyValueStorage:
"""
Key value storage with redis server
"""
def __init__(self, host: str, port: int, retries_limit: int, timeout: int):
self.host = host
self.port = port
self.timeout = timeout
self.retries_limit = retries_limit
self._kv_storage = None
self._connect()
def _connect(self) -> NoReturn:
"""
Connection to redis server with retries
:param retry_counter: counter of times of retrying
:return: Redis connection if it was successful and None otherwise
"""
try:
self._kv_storage = Redis(
host=self.host,
port=self.port,
socket_timeout=self.timeout
)
self._kv_storage.ping()
except Exception:
logging.error(
"Got error while connecting to redis with host %s and port %d.",
self.host,
self.port
)
def _reset_connection(self) -> NoReturn:
"""
Resets connection to redis server
:return:
"""
self._kv_storage.close()
self._connect()
@make_retries
def get(self, key: str) -> str:
"""
Gets value by key as from persistent data storage
:param key: key to get value for
:return: value for specified key
"""
result = self._kv_storage.get(key)
return result.decode("utf-8") if result is not None else result
def cache_get(self, key: str) -> Optional[float]:
"""
Gets value by key from cache
:param key: key to get value for
:return: float value if found some in cache and None otherwise
"""
result = None
try:
result = self.get(key)
except (ConnectionError, TimeoutError) as exception:
logging.error(
"Could not get value from cache. Encountered error: %s",
str(exception)
)
return float(result) if result is not None else result
@make_retries
def _set(self, key: str,
value: Union[float, int],
key_expire_time_sec: int) -> NoReturn:
"""
Sets the value into cache by specified key
:param key: key to set value for
:param value: value for setting
:param key_expire_time_sec: time in which key will be expired
"""
self._kv_storage.set(key, str(value), ex=key_expire_time_sec)
def cache_set(self, key: str,
value: Union[float, int],
key_expire_time_sec: int) -> NoReturn:
"""
Sets the value into cache by specified key
:param key: key to set value for
:param value: value for setting
:param key_expire_time_sec: time in which key will be expired
"""
try:
self._set(key, str(value), key_expire_time_sec=key_expire_time_sec)
except (ConnectionError, TimeoutError) as exception:
logging.error(
"Could not set value to cache. Encountered error: %s",
str(exception)
)
def clear(self) -> NoReturn:
self._kv_storage.flushall()
class CacheManager:
def __init__(self):
self.__redis_release_cache_client = Redis(
connection_pool=BlockingConnectionPool(host=server_config.redis_server_address,
port=server_config.redis_server_port,
password=server_config.redis_server_password,
db=release_cache_db_index))
def add_release_cache(self, hub_uuid: str, app_info: list, release_info: list or None = None):
key = self.__get_app_cache_key(hub_uuid, app_info)
if key is not None:
ex_h = server_config.auto_refresh_time + 4
self.__redis_release_cache_client.set(key, json.dumps(release_info),
ex=ex_h * 3600)
# 缓存完毕
logging.debug(f"release caching: {app_info}")
def get_release_cache(self, hub_uuid: str, app_info: list) -> dict or None:
key = self.__get_app_cache_key(hub_uuid, app_info)
if key is None:
logging.error(f"""WRONG FORMAT
hub_uuid: {hub_uuid}
app_info: {app_info}""")
raise NameError
if self.__redis_release_cache_client.exists(key) == 0:
raise KeyError
release_info = self.__redis_release_cache_client.get(key)
logging.debug(f"release cached: {app_info}")
return json.loads(release_info)
@property
def cached_app_queue(self) -> dict:
cache_app_info_dict = {}
for key in self.__redis_release_cache_client.scan_iter():
hub_uuid, app_info = self.__parsing_app_id(key.decode("utf-8"))
app_info_list = []
if hub_uuid in cache_app_info_dict:
app_info_list = cache_app_info_dict[hub_uuid]
app_info_list.append(app_info)
cache_app_info_dict[hub_uuid] = app_info_list
return cache_app_info_dict
@staticmethod
def __get_app_cache_key(hub_uuid: str, app_id: list) -> str or None:
if not app_id:
return None
key = hub_uuid
for k in app_id:
try:
key += (key_delimiter + k["key"] + value_dict_delimiter + k["value"])
except TypeError:
return None
return key
@staticmethod
def __parsing_app_id(key: str) -> tuple:
key_list = key.split(key_delimiter)
hub_uuid = key_list[0]
app_info = []
for k in key_list[1:]:
key, value = k.split(value_dict_delimiter, 1)
app_info.append(
{"key": key, "value": value}
)
return hub_uuid, app_info
