class RedisCache(BaseCache): key_expire = 60 * 60 # 1 hour def __init__(self, version=None, prefix=None, **options): if not options: # inherit default options from REDIS_OPTIONS options = settings.SENTRY_REDIS_OPTIONS options.setdefault('hosts', { 0: {}, }) self.cluster = Cluster(options['hosts']) self.client = self.cluster.get_routing_client() super(RedisCache, self).__init__(version=version, prefix=prefix) def set(self, key, value, timeout, version=None): key = self.make_key(key, version=version) v = json.dumps(value) if timeout: self.client.setex(key, int(timeout), v) else: self.client.set(key, v) def delete(self, key, version=None): key = self.make_key(key, version=version) self.client.delete(key) def get(self, key, version=None): key = self.make_key(key, version=version) result = self.client.get(key) if result is not None: result = json.loads(result) return result
class RedisCache(local): key_expire = 60 * 60 # 1 hour def __init__(self, **options): if not options: # inherit default options from REDIS_OPTIONS options = settings.SENTRY_REDIS_OPTIONS options.setdefault('hosts', { 0: {}, }) self.cluster = Cluster(options['hosts']) self.client = self.cluster.get_routing_client() def set(self, key, value, timeout): v = json.dumps(value) if timeout: self.client.setex(key, int(timeout), v) else: self.client.set(key, v) def delete(self, key): self.client.delete(key) def get(self, key): result = self.client.get(key) if result is not None: result = json.loads(result) return result
def test_redis_blaster_operation_disable_instance(): from rb import Cluster cluster = Cluster(hosts={0: { 'port': DB_SETTINGS['port'] }}, host_defaults={'host': DB_SETTINGS['host']}) exercise_fanout(cluster) client = cluster.get_routing_client() exercise_redis(client)
class RedisBuffer(Buffer): key_expire = 60 * 60 # 1 hour pending_key = 'b:p' def __init__(self, **options): if not options: # inherit default options from REDIS_OPTIONS options = settings.SENTRY_REDIS_OPTIONS options.setdefault('hosts', { 0: {}, }) self.cluster = Cluster(options['hosts']) def validate(self): try: with self.cluster.all() as client: client.ping() except Exception as e: raise InvalidConfiguration(unicode(e)) def _coerce_val(self, value): if isinstance(value, models.Model): value = value.pk return smart_str(value) def _make_key(self, model, filters): """ Returns a Redis-compatible key for the model given filters. """ return 'b:k:%s:%s' % ( model._meta, md5(smart_str('&'.join('%s=%s' % (k, self._coerce_val(v)) for k, v in sorted(filters.iteritems())))).hexdigest(), ) def _make_lock_key(self, key): return 'l:%s' % (key,) def incr(self, model, columns, filters, extra=None): """ Increment the key by doing the following: - Insert/update a hashmap based on (model, columns) - Perform an incrby on counters - Perform a set (last write wins) on extra - Add hashmap key to pending flushes """ # TODO(dcramer): longer term we'd rather not have to serialize values # here (unless it's to JSON) key = self._make_key(model, filters) # We can't use conn.map() due to wanting to support multiple pending # keys (one per Redis shard) conn = self.cluster.get_local_client_for_key(key) pipe = conn.pipeline() pipe.hsetnx(key, 'm', '%s.%s' % (model.__module__, model.__name__)) pipe.hsetnx(key, 'f', pickle.dumps(filters)) for column, amount in columns.iteritems(): pipe.hincrby(key, 'i+' + column, amount) if extra: for column, value in extra.iteritems(): pipe.hset(key, 'e+' + column, pickle.dumps(value)) pipe.expire(key, self.key_expire) pipe.zadd(self.pending_key, time(), key) pipe.execute() def process_pending(self): client = self.cluster.get_routing_client() lock_key = self._make_lock_key(self.pending_key) # prevent a stampede due to celerybeat + periodic task if not client.set(lock_key, '1', nx=True, ex=60): return try: for host_id in self.cluster.hosts.iterkeys(): conn = self.cluster.get_local_client(host_id) keys = conn.zrange(self.pending_key, 0, -1) if not keys: continue for key in keys: process_incr.apply_async(kwargs={ 'key': key, }) pipe = conn.pipeline() pipe.zrem(self.pending_key, *keys) pipe.execute() finally: client.delete(lock_key) def process(self, key): client = self.cluster.get_routing_client() lock_key = self._make_lock_key(key) # prevent a stampede due to the way we use celery etas + duplicate # tasks if not client.set(lock_key, '1', nx=True, ex=10): return with self.cluster.map() as conn: values = conn.hgetall(key) conn.delete(key) if not values.value: return model = import_string(values.value['m']) filters = pickle.loads(values.value['f']) incr_values = {} extra_values = {} for k, v in values.value.iteritems(): if k.startswith('i+'): incr_values[k[2:]] = int(v) elif k.startswith('e+'): extra_values[k[2:]] = pickle.loads(v) super(RedisBuffer, self).process(model, incr_values, filters, extra_values)