def test_truncate_timeline_script(self): client = StrictRedis(db=9) timeline = 'timeline' # Preload some fake records (the contents don't matter.) records = list(itertools.islice(self.records, 10)) for record in records: client.zadd(timeline, record.timestamp, record.key) client.set(make_record_key(timeline, record.key), 'data') with self.assertChanges(lambda: client.zcard(timeline), before=10, after=5): truncate_timeline((timeline, ), (5, ), client) # Ensure the early records don't exist. for record in records[:5]: assert not client.zscore(timeline, record.key) assert not client.exists(make_record_key(timeline, record.key)) # Ensure the later records do exist. for record in records[-5:]: assert client.zscore(timeline, record.key) == float(record.timestamp) assert client.exists(make_record_key(timeline, record.key))
def test_delete(self): timeline = 'timeline' backend = RedisBackend() timeline_key = make_timeline_key(backend.namespace, timeline) digest_key = make_digest_key(timeline_key) waiting_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_WAITING) ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY) connection = backend.cluster.get_local_client_for_key(timeline_key) connection.zadd(waiting_set_key, 0, timeline) connection.zadd(ready_set_key, 0, timeline) connection.zadd(timeline_key, 0, '1') connection.set(make_record_key(timeline_key, '1'), 'data') connection.zadd(digest_key, 0, '2') connection.set(make_record_key(timeline_key, '2'), 'data') keys = ( waiting_set_key, ready_set_key, digest_key, timeline_key, make_record_key(timeline_key, '1'), make_record_key(timeline_key, '2') ) def check_keys_exist(): return map(connection.exists, keys) with self.assertChanges(check_keys_exist, before=[True] * len(keys), after=[False] * len(keys)): backend.delete(timeline)
def test_truncate_timeline_script(self): cluster = clusters.get('default') client = cluster.get_local_client(six.next(iter(cluster.hosts))) timeline = 'timeline' # Preload some fake records (the contents don't matter.) records = list(itertools.islice(self.records, 10)) for record in records: client.zadd(timeline, record.timestamp, record.key) client.set(make_record_key(timeline, record.key), 'data') with self.assertChanges(lambda: client.zcard(timeline), before=10, after=5): truncate_timeline(client, (timeline, ), (5, timeline)) # Ensure the early records don't exist. for record in records[:5]: assert not client.zscore(timeline, record.key) assert not client.exists(make_record_key(timeline, record.key)) # Ensure the later records do exist. for record in records[-5:]: assert client.zscore(timeline, record.key) == float(record.timestamp) assert client.exists(make_record_key(timeline, record.key))
def test_add_record(self): timeline = 'timeline' backend = self.get_backend() timeline_key = make_timeline_key(backend.namespace, timeline) connection = backend.cluster.get_local_client_for_key(timeline_key) record = next(self.records) ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY) record_key = make_record_key(timeline_key, record.key) get_timeline_score_in_ready_set = functools.partial( connection.zscore, ready_set_key, timeline) get_record_score_in_timeline_set = functools.partial( connection.zscore, timeline_key, record.key) def get_record_value(): value = connection.get(record_key) return backend.codec.decode(value) if value is not None else None with self.assertChanges(get_timeline_score_in_ready_set, before=None, after=record.timestamp), \ self.assertChanges(get_record_score_in_timeline_set, before=None, after=record.timestamp), \ self.assertChanges(get_record_value, before=None, after=record.value): backend.add(timeline, record)
def test_maintenance(self): timeline = 'timeline' backend = RedisBackend(ttl=3600) timeline_key = make_timeline_key(backend.namespace, timeline) digest_key = make_digest_key(timeline_key) waiting_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_WAITING) ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY) now = time.time() connection = backend.cluster.get_local_client_for_key(timeline_key) schedule_time = now - 60 connection.zadd(ready_set_key, schedule_time, timeline) connection.zadd(timeline_key, 0, '1') connection.set(make_record_key(timeline_key, '1'), 'data') connection.zadd(digest_key, 0, '2') connection.set(make_record_key(timeline_key, '2'), 'data') # Move the digest from the ready set to the waiting set. backend.maintenance(now) assert connection.zcard(ready_set_key) == 0 assert connection.zrange(waiting_set_key, 0, -1, withscores=True) == [ (timeline, schedule_time) ] connection.zrem(waiting_set_key, timeline) connection.zadd(ready_set_key, schedule_time, timeline) # Delete the digest from the ready set. with mock.patch('time.time', return_value=now + (backend.ttl + 1)): backend.maintenance(now) keys = ( ready_set_key, waiting_set_key, timeline_key, digest_key, make_record_key(timeline_key, '1'), make_record_key(timeline_key, '2'), ) for key in keys: assert connection.exists(key) is False
def test_maintenance(self): timeline = 'timeline' backend = RedisBackend(ttl=3600) timeline_key = make_timeline_key(backend.namespace, timeline) digest_key = make_digest_key(timeline_key) waiting_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_WAITING) ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY) now = time.time() connection = backend.cluster.get_local_client_for_key(timeline_key) schedule_time = now - 60 connection.zadd(ready_set_key, schedule_time, timeline) connection.zadd(timeline_key, 0, '1') connection.set(make_record_key(timeline_key, '1'), 'data') connection.zadd(digest_key, 0, '2') connection.set(make_record_key(timeline_key, '2'), 'data') # Move the digest from the ready set to the waiting set. backend.maintenance(now) assert connection.zcard(ready_set_key) == 0 assert connection.zrange(waiting_set_key, 0, -1, withscores=True) == [(timeline, schedule_time)] connection.zrem(waiting_set_key, timeline) connection.zadd(ready_set_key, schedule_time, timeline) # Delete the digest from the ready set. with mock.patch('time.time', return_value=now + (backend.ttl + 1)): backend.maintenance(now) keys = ( ready_set_key, waiting_set_key, timeline_key, digest_key, make_record_key(timeline_key, '1'), make_record_key(timeline_key, '2'), ) for key in keys: assert connection.exists(key) is False
def test_truncate_timeline_script(self): client = StrictRedis(db=9) timeline = 'timeline' # Preload some fake records (the contents don't matter.) records = list(itertools.islice(self.records, 10)) for record in records: client.zadd(timeline, record.timestamp, record.key) client.set(make_record_key(timeline, record.key), 'data') with self.assertChanges(lambda: client.zcard(timeline), before=10, after=5): truncate_timeline((timeline,), (5,), client) # Ensure the early records don't exist. for record in records[:5]: assert not client.zscore(timeline, record.key) assert not client.exists(make_record_key(timeline, record.key)) # Ensure the later records do exist. for record in records[-5:]: assert client.zscore(timeline, record.key) == float(record.timestamp) assert client.exists(make_record_key(timeline, record.key))
def test_truncate_timeline_script(self): cluster = clusters.get('default') client = cluster.get_local_client(six.next(iter(cluster.hosts))) timeline = 'timeline' # Preload some fake records (the contents don't matter.) records = list(itertools.islice(self.records, 10)) for record in records: client.zadd(timeline, record.timestamp, record.key) client.set(make_record_key(timeline, record.key), 'data') with self.assertChanges(lambda: client.zcard(timeline), before=10, after=5): truncate_timeline(client, (timeline,), (5, timeline)) # Ensure the early records don't exist. for record in records[:5]: assert not client.zscore(timeline, record.key) assert not client.exists(make_record_key(timeline, record.key)) # Ensure the later records do exist. for record in records[-5:]: assert client.zscore(timeline, record.key) == float(record.timestamp) assert client.exists(make_record_key(timeline, record.key))
def test_add_record(self): timeline = 'timeline' backend = RedisBackend() timeline_key = make_timeline_key(backend.namespace, timeline) connection = backend.cluster.get_local_client_for_key(timeline_key) record = next(self.records) ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY) record_key = make_record_key(timeline_key, record.key) get_timeline_score_in_ready_set = functools.partial(connection.zscore, ready_set_key, timeline) get_record_score_in_timeline_set = functools.partial(connection.zscore, timeline_key, record.key) def get_record_value(): value = connection.get(record_key) return backend.codec.decode(value) if value is not None else None with self.assertChanges(get_timeline_score_in_ready_set, before=None, after=record.timestamp), \ self.assertChanges(get_record_score_in_timeline_set, before=None, after=record.timestamp), \ self.assertChanges(get_record_value, before=None, after=record.value): backend.add(timeline, record)
def test_add_record(self): timeline = 'timeline' backend = self.get_backend() timeline_key = make_timeline_key(backend.namespace, timeline) connection = backend.cluster.get_local_client_for_key(timeline_key) record = next(self.records) waiting_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_WAITING) record_key = make_record_key(timeline_key, record.key) get_timeline_score_in_waiting_set = functools.partial(connection.zscore, waiting_set_key, timeline) get_timeline_iteration_counter = functools.partial(connection.get, make_iteration_key(timeline_key)) get_record_score_in_timeline_set = functools.partial(connection.zscore, timeline_key, record.key) def get_record_value(): value = connection.get(record_key) return backend.codec.decode(value) if value is not None else None with self.assertChanges(get_timeline_score_in_waiting_set, before=None, after=record.timestamp + backend.backoff(0)), \ self.assertChanges(get_timeline_iteration_counter, before=None, after='0'), \ self.assertChanges(get_record_score_in_timeline_set, before=None, after=record.timestamp), \ self.assertChanges(get_record_value, before=None, after=record.value): backend.add(timeline, record)