class RedisSafetyStore(object): def __init__(self): self.redisClient = Redis(connection_pool=BlockingConnectionPool()) def isSafe(self, url): # if the urlstring is in the store, then it is not safe. try: notSafe = self.redisClient.get(url) except: print("Redis read error") raise if notSafe: return False else: return True def addUnsafeResources(self, jsonContent): for resource in jsonContent["resources"]: try: self.redisClient.set(resource, 1) except: print("Redis write error") raise return True
def uget(self, name): """ Return the value and key ``name`` or None, and decode it if not None. """ value = Redis.get(self, name) if value: return value.decode(self.encoding) return value
class Cache: """ Cache class """ def __init__(self): """ Store an instance of the Redis client as a private variable named _redis """ self._redis = Redis() self._redis.flushdb() @call_history @count_calls def store(self, data: Union[str, bytes, int, float]) -> str: """ Takes a data argument and returns a string """ key = str(uuid4()) self._redis.set(key, data) return key def get(self, key: str, fn: Optional[Callable] = None) ->\ Union[str, bytes, int, float]: """ Take a key string argument and an optional Callable argument named fn """ if key: result = self._redis.get(key) if fn: return fn(result) else: return result def get_str(self, data: bytes) -> str: """ Will automatically parametrize Cache.get with the correct conversion function """ return data.decode('utf-8') def get_int(self, data: bytes) -> int: """ Will automatically parametrize Cache.get with the correct conversion function """ byte_order = sys.byteorder return int.from_bytes(data, byte_order)
class Cache: """Redis Class""" def __init__(self): """constructor""" self._redis = Redis() self._redis.flushdb() @call_history @count_calls def store(self, data: Union[str, bytes, int, float]) -> str: """returns a string""" key = str(uuid.uuid4()) self._redis.set(key, data) return key def get(self, key: str, fn: Optional[Callable] = None) ->\ Union[str, bytes, int, float]: """get method""" if key: result = self._redis.get(key) return fn(result) if fn else result
class AnguisRedis(AnguisBase): def __init__(self, host='localhost', port=6379, db=0, *args, **kwargs): self.r = Redis(host, port, db) super(AnguisRedis, self).__init__() def __del__(self): super(AnguisRedis, self).__del__() def __getitem__(self, key): return self.unserialize(self.r.get(key)) def __setitem__(self, key, obj): self.r.set(key, self.serialize(obj)) def __delitem__(self, key): return self.r.delete(key) def __iter__(self): return iter(self.r.keys()) def __len__(self): return len(self.r.keys())
class Cache: """Cache class""" def __init__(self): """constructor""" self._redis = Redis() self._redis.flushdb() @call_history @count_calls def store(self, data: Union[str, bytes, int, float]) -> str: """Store data in cache""" key = str(uuid.uuid4()) self._redis.set(key, data) return key def get(self, key: str, fn: Optional[Callable] = None) ->\ Union[str, bytes, int, float]: """get data from fuction or cache""" if key: result = self._redis.get(key) if fn: return fn(result) else: return result
def increment(): red = Redis() print(red.get('key') is None) red.set('key', 1) red.incr('key', 2)
def getInt(r: Redis, key: str): value = r.get(key) return 0 if value is None else int(value)
# -*- coding: utf-8 -*- __author__ = 'fjs' from connection import BlockingConnectionPool from redis.client import Redis client = Redis(connection_pool=BlockingConnectionPool(max_connections=2)) client.set("fjs", "fjs") print client.get("fjs") client.lpush("nn", 1) print client.lpop("nn")
class TSStore(object): def __init__(self, config): self._redis = Redis(host=config.get('redis','host'), port=int(config.get('redis','port')), db=int(config.get('redis','db'))) self._delta_secs = int(eval(config.get('timeseries', 'delta_secs'))) self._expiration_delay_secs = int(eval(config.get('timeseries', 'expiration_delay_secs'))) def queries_key(self): return 'queries' @property def queries(self): return self._redis.smembers(self.queries_key()) @queries.setter def queries(self, values): pipe = self._redis.pipeline() pipe.delete(self.queries_key()) for v in values: pipe.sadd(self.queries_key(), v) return pipe.execute() def _interval_key(self, timestamp): return int(timestamp) - int(timestamp) % self._delta_secs def _ts_key(self, timestamp, query): return 'ts:%(query)s:%(timestamp_key)s'%{'query':query, 'timestamp_key':self._interval_key(timestamp)} def _tweet_key(self, t): if type(t) == Tweet: return 'tweet:%s'%t.id return 'tweet:%s'%t def _query_key(self, query): return 'query:%s:last_tweet_id'%query def _store_tweet(self, pipe, tweet): tweet_key = self._tweet_key(tweet) pipe.set(tweet_key, tweet.serialize()) pipe.expire(tweet_key, self._expiration_delay_secs) def _reference_tweet(self, pipe, timestamp, query, tweet): ts_key = self._ts_key(timestamp, query) pipe.lpush(ts_key,tweet.id) pipe.expire(ts_key,self._expiration_delay_secs) def _update_last_query_tweet(self, pipe, query, tweet): query_key = self._query_key(query) pipe.set(query_key,tweet.id) def append(self, query, tweet): pipe = self._redis.pipeline() timestamp = time.time() self._store_tweet(pipe, tweet) self._reference_tweet(pipe, timestamp, query, tweet) self._update_last_query_tweet(pipe, query, tweet) return pipe.execute() def retrieve_ts(self, query, timestamp, n_elements=-1): ts_key = self._ts_key(timestamp, query) return self._redis.lrange(ts_key, 0, n_elements) def retrieve_last_tweet_id(self, query): query_key = self._query_key(query) return self._redis.get(query_key) def retrieve_tweet(self, tweet_id): tweet_key = self._tweet_key(tweet_id) data = self._redis.get(tweet_key) return Tweet.deserialize(data).todict() def retrieve(self, query, n_periods=30): current_timestamp = now = int(time.time()) start_timestamp = now - self._delta_secs * n_periods tweets = [] while current_timestamp > start_timestamp: current_tweet_ids = self.retrieve_ts(query, current_timestamp) tweets.append({'timestamp': current_timestamp, 'tweets' : [ self.retrieve_tweet(tid) for tid in current_tweet_ids ] }) current_timestamp -= self._delta_secs return { 'now' : now, 'ts' : tweets }
class BaseStorage(object): """Base class for a storage engine. Such objects provide a general abstraction over various storage backends (in-memory dict, remote cache, ...) """ def __init__(self, client=None): if not client: self.client = Redis(host='localhost', port=6379, db=13, password=None) else: self.client = client def get(self, key, default=0.0): """Retrieve the current value for a key. Args: key (str): the key whose value should be retrieved default (object): the value to use when no value exist for the key """ result = self.client.get(key) if not result: return default else: return float(result) # raise NotImplementedError() def set(self, key, value): """Set a new value for a given key.""" return self.client.set(key, value) # raise NotImplementedError() def mget(self, *keys, **kwargs): """Retrieve values for a set of keys. Args: keys (str list): the list of keys whose value should be retrieved Keyword arguements: default (object): the value to use for non-existent keys coherent (bool): whether all fetched values should be "coherent", i.e no other update was performed on any of those values while fetching from the database. Yields: object: values for the keys, in the order they were passed """ default = kwargs.get('default') coherent = kwargs.get('coherent', False) for key in keys: yield self.get(key, default=default) def mset(self, values): """Set the value of several keys at once. Args: values (dict): maps a key to its value. """ for key, value in values.items(): self.set(key, value) def incr(self, key, amount=1, default=0): """Increment the value of a key by a given amount. Also works for decrementing it. Args: key (str): the key whose value should be incremented amount (int): the amount by which the value should be incremented default (int): the default value to use if the key was never set Returns: int, the updated value """ # import pdb;pdb.set_trace() value = int(self.get(key, default)) + int(amount) self.set(key, value) return value
class KeyValueStorage: """ Key value storage with redis server """ def __init__(self, host: str, port: int, retries_limit: int, timeout: int): self.host = host self.port = port self.timeout = timeout self.retries_limit = retries_limit self._kv_storage = None self._connect() def _connect(self) -> NoReturn: """ Connection to redis server with retries :param retry_counter: counter of times of retrying :return: Redis connection if it was successful and None otherwise """ try: self._kv_storage = Redis( host=self.host, port=self.port, socket_timeout=self.timeout ) self._kv_storage.ping() except Exception: logging.error( "Got error while connecting to redis with host %s and port %d.", self.host, self.port ) def _reset_connection(self) -> NoReturn: """ Resets connection to redis server :return: """ self._kv_storage.close() self._connect() @make_retries def get(self, key: str) -> str: """ Gets value by key as from persistent data storage :param key: key to get value for :return: value for specified key """ result = self._kv_storage.get(key) return result.decode("utf-8") if result is not None else result def cache_get(self, key: str) -> Optional[float]: """ Gets value by key from cache :param key: key to get value for :return: float value if found some in cache and None otherwise """ result = None try: result = self.get(key) except (ConnectionError, TimeoutError) as exception: logging.error( "Could not get value from cache. Encountered error: %s", str(exception) ) return float(result) if result is not None else result @make_retries def _set(self, key: str, value: Union[float, int], key_expire_time_sec: int) -> NoReturn: """ Sets the value into cache by specified key :param key: key to set value for :param value: value for setting :param key_expire_time_sec: time in which key will be expired """ self._kv_storage.set(key, str(value), ex=key_expire_time_sec) def cache_set(self, key: str, value: Union[float, int], key_expire_time_sec: int) -> NoReturn: """ Sets the value into cache by specified key :param key: key to set value for :param value: value for setting :param key_expire_time_sec: time in which key will be expired """ try: self._set(key, str(value), key_expire_time_sec=key_expire_time_sec) except (ConnectionError, TimeoutError) as exception: logging.error( "Could not set value to cache. Encountered error: %s", str(exception) ) def clear(self) -> NoReturn: self._kv_storage.flushall()
class CacheManager: def __init__(self): self.__redis_release_cache_client = Redis( connection_pool=BlockingConnectionPool(host=server_config.redis_server_address, port=server_config.redis_server_port, password=server_config.redis_server_password, db=release_cache_db_index)) def add_release_cache(self, hub_uuid: str, app_info: list, release_info: list or None = None): key = self.__get_app_cache_key(hub_uuid, app_info) if key is not None: ex_h = server_config.auto_refresh_time + 4 self.__redis_release_cache_client.set(key, json.dumps(release_info), ex=ex_h * 3600) # 缓存完毕 logging.debug(f"release caching: {app_info}") def get_release_cache(self, hub_uuid: str, app_info: list) -> dict or None: key = self.__get_app_cache_key(hub_uuid, app_info) if key is None: logging.error(f"""WRONG FORMAT hub_uuid: {hub_uuid} app_info: {app_info}""") raise NameError if self.__redis_release_cache_client.exists(key) == 0: raise KeyError release_info = self.__redis_release_cache_client.get(key) logging.debug(f"release cached: {app_info}") return json.loads(release_info) @property def cached_app_queue(self) -> dict: cache_app_info_dict = {} for key in self.__redis_release_cache_client.scan_iter(): hub_uuid, app_info = self.__parsing_app_id(key.decode("utf-8")) app_info_list = [] if hub_uuid in cache_app_info_dict: app_info_list = cache_app_info_dict[hub_uuid] app_info_list.append(app_info) cache_app_info_dict[hub_uuid] = app_info_list return cache_app_info_dict @staticmethod def __get_app_cache_key(hub_uuid: str, app_id: list) -> str or None: if not app_id: return None key = hub_uuid for k in app_id: try: key += (key_delimiter + k["key"] + value_dict_delimiter + k["value"]) except TypeError: return None return key @staticmethod def __parsing_app_id(key: str) -> tuple: key_list = key.split(key_delimiter) hub_uuid = key_list[0] app_info = [] for k in key_list[1:]: key, value = k.split(value_dict_delimiter, 1) app_info.append( {"key": key, "value": value} ) return hub_uuid, app_info