def groom_redis_precaches(): """Groom the Redis data for streams precaching.""" r = get_redis_connection() user_activities = {} keys = r.keys("sh:streams:[a-z0-9_\-:]*") for key in keys: decoded_key = key.decode("utf-8") if decoded_key.endswith(':throughs'): # Skip throughs, we handle those separately below continue trim_size = get_precache_trim_size(user_activities, decoded_key) # Make the ordered set X items length at most r.zremrangebyrank(key, 0, -trim_size - 1) # Remove now obsolete throughs ID's from the throughs hash throughs_key = "%s:throughs" % decoded_key if not r.zcount(key, '-inf', '+inf'): # No need to loop, we've removed everything r.delete(throughs_key) continue delkeys = [] for content_id, _through_id in r.hgetall(throughs_key).items(): if r.zrank(key, int(content_id)) is None: delkeys.append(int(content_id)) if delkeys: r.hdel(throughs_key, *delkeys)
def mark_recently_active(self) -> None: """ Flag the user as currently active. """ r = get_redis_connection() r.set(self.activity_key, int(time.time())) r.expire(self.activity_key, settings.SOCIALHOME_USER_ACTIVITY_SECONDS)
def groom_redis_precaches(): """Groom the Redis data for streams precaching.""" r = get_redis_connection() user_activities = {} keys = r.keys("sh:streams:[a-z0-9_\-:]*") for key in keys: decoded_key = key.decode("utf-8") if decoded_key.endswith(':throughs'): # Skip throughs, we handle those separately below continue trim_size = get_precache_trim_size(user_activities, decoded_key) # Make the ordered set X items length at most r.zremrangebyrank(key, 0, -trim_size-1) # Remove now obsolete throughs ID's from the throughs hash throughs_key = "%s:throughs" % decoded_key if not r.zcount(key, '-inf', '+inf'): # No need to loop, we've removed everything r.delete(throughs_key) continue delkeys = [] for content_id, _through_id in r.hgetall(throughs_key).items(): if r.zrank(key, int(content_id)) is None: delkeys.append(int(content_id)) if delkeys: r.hdel(throughs_key, *delkeys)
def add_to_redis(content, through, keys): """Add content to a list of Redis ordered sets. :param content: Content object to add :param through: Content through object. For example on shares, this is the linked share content object :param keys: List of keys to add to """ if not keys: return r = get_redis_connection() for key in keys: rank = r.zrank(key, content.id) if rank: # If through is not content, update if content.id != through.id: throughs_key = BaseStream.get_throughs_key(key) r.hset(throughs_key, content.id, through.id) else: # Only add if not in the set already # This stops shares popping up more than once, for example r.zadd(key, {content.id: int(time.time())}) r.expire(key, settings.REDIS_DEFAULT_EXPIRY) throughs_key = BaseStream.get_throughs_key(key) r.hset(throughs_key, content.id, through.id) r.expire(throughs_key, settings.REDIS_DEFAULT_EXPIRY)
def setUp(self): super().setUp() self.r = get_redis_connection() for x in range(10): self.r.zadd("sh:streams:spamandeggs:736353:%s" % self.user.id, x, str(x**x)) for x in range(10): self.r.hset("sh:streams:spamandeggs:736353:%s:throughs" % self.user.id, str(x**x), x**x+1)
def get_recently_active_user_ids() -> List[int]: """ Returns a list of ID's for User objects that have been recently active. """ r = get_redis_connection() keys = r.keys(r"sh:users:activity:*") return [int(key.decode("utf-8").rsplit(":", 1)[1]) for key in keys]
def setUp(self): super().setUp() self.r = get_redis_connection() for x in range(10): self.r.zadd("sh:streams:spamandeggs:736353:%s" % self.user.id, {str(x**x): x}) for x in range(10): self.r.hset("sh:streams:spamandeggs:736353:%s:throughs" % self.user.id, str(x**x), x**x+1)
def setUp(self): super().setUp() self.r = get_redis_connection() for x in range(10): self.r.zadd("sh:streams:spamandeggs:1", x, str(x**x)) self.expected_z = {x: str(x**x) for x in range(7, 10)} for x in range(10): self.r.hset("sh:streams:spamandeggs:1:throughs", str(x**x), x**x+1) self.expected_h = {str(x**x): str(x**x+1) for x in range(7, 10)}
def get_cached_content_ids(self): key = self.get_key() r = get_redis_connection() index = 0 if self.last_id: last_index = r.zrevrank(key, self.last_id) if not last_index: # This item is outside our cached ids, abort return [] index = last_index + 1 return r.zrevrange(key, index, index + self.paginate_by)
def add_to_redis(content, keys): """Add content to a list of Redis ordered sets. :param content: Content object to add :param keys: List of keys to add to """ if not keys: return r = get_redis_connection() content_id = str(content.id) for key in keys: r.zadd(key, int(time.time()), content_id)
def add_to_redis(content, through, keys): """Add content to a list of Redis ordered sets. :param content: Content object to add :param through: Content through object. For example on shares, this is the linked share content object :param keys: List of keys to add to """ if not keys: return r = get_redis_connection() for key in keys: # Only add if not in the set already # This stops shares popping up more than once, for example if not r.zrank(key, content.id): r.zadd(key, int(time.time()), content.id) r.hset(BaseStream.get_throughs_key(key), content.id, through.id)
def add_to_redis(content, through, keys): """Add content to a list of Redis ordered sets. :param content: Content object to add :param through: Content through object. For example on shares, this is the linked share content object :param keys: List of keys to add to """ if not keys: return r = get_redis_connection() for key in keys: # Only add if not in the set already # This stops shares popping up more than once, for example if not r.zrank(key, content.id): r.zadd(key, int(time.time()), content.id) r.expire(key, settings.REDIS_DEFAULT_EXPIRY) throughs_key = BaseStream.get_throughs_key(key) r.hset(throughs_key, content.id, through.id) r.expire(throughs_key, settings.REDIS_DEFAULT_EXPIRY)
def delete_redis_keys(pattern: str, only_without_expiry: bool = True): """ Delete any keys matching pattern. Defaults to only those without expiry. """ r = get_redis_connection() keys = r.keys(pattern) to_delete = [] for key in keys: if only_without_expiry: delete = r.ttl(key) == -1 else: delete = True if delete: to_delete.append(key) if len(to_delete) > 1000: r.delete(*to_delete) if to_delete: r.delete(*to_delete)
def recently_active(self) -> bool: """ Return True if the user is marked as "active recently" """ r = get_redis_connection() return r.exists(self.activity_key)
def init_redis_connection(self): if not self.redis: self.redis = get_redis_connection()
def init_redis_connection(self): if not self.redis: self.redis = get_redis_connection()
def test_get_redis_connection(self, mock_redis): get_redis_connection() self.assertTrue(mock_redis.called)