def test_truncate_timeline_script(self):
client = StrictRedis(db=9)
timeline = 'timeline'
# Preload some fake records (the contents don't matter.)
records = list(itertools.islice(self.records, 10))
for record in records:
client.zadd(timeline, record.timestamp, record.key)
client.set(make_record_key(timeline, record.key), 'data')
with self.assertChanges(lambda: client.zcard(timeline),
before=10,
after=5):
truncate_timeline((timeline, ), (5, ), client)
# Ensure the early records don't exist.
for record in records[:5]:
assert not client.zscore(timeline, record.key)
assert not client.exists(make_record_key(timeline, record.key))
# Ensure the later records do exist.
for record in records[-5:]:
assert client.zscore(timeline,
record.key) == float(record.timestamp)
assert client.exists(make_record_key(timeline, record.key))
def test_delete(self):
timeline = 'timeline'
backend = RedisBackend()
timeline_key = make_timeline_key(backend.namespace, timeline)
digest_key = make_digest_key(timeline_key)
waiting_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_WAITING)
ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY)
connection = backend.cluster.get_local_client_for_key(timeline_key)
connection.zadd(waiting_set_key, 0, timeline)
connection.zadd(ready_set_key, 0, timeline)
connection.zadd(timeline_key, 0, '1')
connection.set(make_record_key(timeline_key, '1'), 'data')
connection.zadd(digest_key, 0, '2')
connection.set(make_record_key(timeline_key, '2'), 'data')
keys = (
waiting_set_key,
ready_set_key,
digest_key,
timeline_key,
make_record_key(timeline_key, '1'),
make_record_key(timeline_key, '2')
)
def check_keys_exist():
return map(connection.exists, keys)
with self.assertChanges(check_keys_exist, before=[True] * len(keys), after=[False] * len(keys)):
backend.delete(timeline)
def test_truncate_timeline_script(self):
cluster = clusters.get('default')
client = cluster.get_local_client(six.next(iter(cluster.hosts)))
timeline = 'timeline'
# Preload some fake records (the contents don't matter.)
records = list(itertools.islice(self.records, 10))
for record in records:
client.zadd(timeline, record.timestamp, record.key)
client.set(make_record_key(timeline, record.key), 'data')
with self.assertChanges(lambda: client.zcard(timeline),
before=10,
after=5):
truncate_timeline(client, (timeline, ), (5, timeline))
# Ensure the early records don't exist.
for record in records[:5]:
assert not client.zscore(timeline, record.key)
assert not client.exists(make_record_key(timeline, record.key))
# Ensure the later records do exist.
for record in records[-5:]:
assert client.zscore(timeline,
record.key) == float(record.timestamp)
assert client.exists(make_record_key(timeline, record.key))
def test_add_record(self):
timeline = 'timeline'
backend = self.get_backend()
timeline_key = make_timeline_key(backend.namespace, timeline)
connection = backend.cluster.get_local_client_for_key(timeline_key)
record = next(self.records)
ready_set_key = make_schedule_key(backend.namespace,
SCHEDULE_STATE_READY)
record_key = make_record_key(timeline_key, record.key)
get_timeline_score_in_ready_set = functools.partial(
connection.zscore, ready_set_key, timeline)
get_record_score_in_timeline_set = functools.partial(
connection.zscore, timeline_key, record.key)
def get_record_value():
value = connection.get(record_key)
return backend.codec.decode(value) if value is not None else None
with self.assertChanges(get_timeline_score_in_ready_set, before=None, after=record.timestamp), \
self.assertChanges(get_record_score_in_timeline_set, before=None, after=record.timestamp), \
self.assertChanges(get_record_value, before=None, after=record.value):
backend.add(timeline, record)
def test_maintenance(self):
timeline = 'timeline'
backend = RedisBackend(ttl=3600)
timeline_key = make_timeline_key(backend.namespace, timeline)
digest_key = make_digest_key(timeline_key)
waiting_set_key = make_schedule_key(backend.namespace,
SCHEDULE_STATE_WAITING)
ready_set_key = make_schedule_key(backend.namespace,
SCHEDULE_STATE_READY)
now = time.time()
connection = backend.cluster.get_local_client_for_key(timeline_key)
schedule_time = now - 60
connection.zadd(ready_set_key, schedule_time, timeline)
connection.zadd(timeline_key, 0, '1')
connection.set(make_record_key(timeline_key, '1'), 'data')
connection.zadd(digest_key, 0, '2')
connection.set(make_record_key(timeline_key, '2'), 'data')
# Move the digest from the ready set to the waiting set.
backend.maintenance(now)
assert connection.zcard(ready_set_key) == 0
assert connection.zrange(waiting_set_key, 0, -1, withscores=True) == [
(timeline, schedule_time)
]
connection.zrem(waiting_set_key, timeline)
connection.zadd(ready_set_key, schedule_time, timeline)
# Delete the digest from the ready set.
with mock.patch('time.time', return_value=now + (backend.ttl + 1)):
backend.maintenance(now)
keys = (
ready_set_key,
waiting_set_key,
timeline_key,
digest_key,
make_record_key(timeline_key, '1'),
make_record_key(timeline_key, '2'),
)
for key in keys:
assert connection.exists(key) is False
def test_maintenance(self):
timeline = 'timeline'
backend = RedisBackend(ttl=3600)
timeline_key = make_timeline_key(backend.namespace, timeline)
digest_key = make_digest_key(timeline_key)
waiting_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_WAITING)
ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY)
now = time.time()
connection = backend.cluster.get_local_client_for_key(timeline_key)
schedule_time = now - 60
connection.zadd(ready_set_key, schedule_time, timeline)
connection.zadd(timeline_key, 0, '1')
connection.set(make_record_key(timeline_key, '1'), 'data')
connection.zadd(digest_key, 0, '2')
connection.set(make_record_key(timeline_key, '2'), 'data')
# Move the digest from the ready set to the waiting set.
backend.maintenance(now)
assert connection.zcard(ready_set_key) == 0
assert connection.zrange(waiting_set_key, 0, -1, withscores=True) == [(timeline, schedule_time)]
connection.zrem(waiting_set_key, timeline)
connection.zadd(ready_set_key, schedule_time, timeline)
# Delete the digest from the ready set.
with mock.patch('time.time', return_value=now + (backend.ttl + 1)):
backend.maintenance(now)
keys = (
ready_set_key,
waiting_set_key,
timeline_key,
digest_key,
make_record_key(timeline_key, '1'),
make_record_key(timeline_key, '2'),
)
for key in keys:
assert connection.exists(key) is False
def test_truncate_timeline_script(self):
client = StrictRedis(db=9)
timeline = 'timeline'
# Preload some fake records (the contents don't matter.)
records = list(itertools.islice(self.records, 10))
for record in records:
client.zadd(timeline, record.timestamp, record.key)
client.set(make_record_key(timeline, record.key), 'data')
with self.assertChanges(lambda: client.zcard(timeline), before=10, after=5):
truncate_timeline((timeline,), (5,), client)
# Ensure the early records don't exist.
for record in records[:5]:
assert not client.zscore(timeline, record.key)
assert not client.exists(make_record_key(timeline, record.key))
# Ensure the later records do exist.
for record in records[-5:]:
assert client.zscore(timeline, record.key) == float(record.timestamp)
assert client.exists(make_record_key(timeline, record.key))
def test_truncate_timeline_script(self):
cluster = clusters.get('default')
client = cluster.get_local_client(six.next(iter(cluster.hosts)))
timeline = 'timeline'
# Preload some fake records (the contents don't matter.)
records = list(itertools.islice(self.records, 10))
for record in records:
client.zadd(timeline, record.timestamp, record.key)
client.set(make_record_key(timeline, record.key), 'data')
with self.assertChanges(lambda: client.zcard(timeline), before=10, after=5):
truncate_timeline(client, (timeline,), (5, timeline))
# Ensure the early records don't exist.
for record in records[:5]:
assert not client.zscore(timeline, record.key)
assert not client.exists(make_record_key(timeline, record.key))
# Ensure the later records do exist.
for record in records[-5:]:
assert client.zscore(timeline, record.key) == float(record.timestamp)
assert client.exists(make_record_key(timeline, record.key))
def test_add_record(self):
timeline = 'timeline'
backend = RedisBackend()
timeline_key = make_timeline_key(backend.namespace, timeline)
connection = backend.cluster.get_local_client_for_key(timeline_key)
record = next(self.records)
ready_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_READY)
record_key = make_record_key(timeline_key, record.key)
get_timeline_score_in_ready_set = functools.partial(connection.zscore, ready_set_key, timeline)
get_record_score_in_timeline_set = functools.partial(connection.zscore, timeline_key, record.key)
def get_record_value():
value = connection.get(record_key)
return backend.codec.decode(value) if value is not None else None
with self.assertChanges(get_timeline_score_in_ready_set, before=None, after=record.timestamp), \
self.assertChanges(get_record_score_in_timeline_set, before=None, after=record.timestamp), \
self.assertChanges(get_record_value, before=None, after=record.value):
backend.add(timeline, record)
def test_add_record(self):
timeline = 'timeline'
backend = self.get_backend()
timeline_key = make_timeline_key(backend.namespace, timeline)
connection = backend.cluster.get_local_client_for_key(timeline_key)
record = next(self.records)
waiting_set_key = make_schedule_key(backend.namespace, SCHEDULE_STATE_WAITING)
record_key = make_record_key(timeline_key, record.key)
get_timeline_score_in_waiting_set = functools.partial(connection.zscore, waiting_set_key, timeline)
get_timeline_iteration_counter = functools.partial(connection.get, make_iteration_key(timeline_key))
get_record_score_in_timeline_set = functools.partial(connection.zscore, timeline_key, record.key)
def get_record_value():
value = connection.get(record_key)
return backend.codec.decode(value) if value is not None else None
with self.assertChanges(get_timeline_score_in_waiting_set, before=None, after=record.timestamp + backend.backoff(0)), \
self.assertChanges(get_timeline_iteration_counter, before=None, after='0'), \
self.assertChanges(get_record_score_in_timeline_set, before=None, after=record.timestamp), \
self.assertChanges(get_record_value, before=None, after=record.value):
backend.add(timeline, record)
