def test_system_lock(self, timeout):
semaphore1 = Semaphore(self.conn,
'test_key',
'id_1',
max_locks=10,
timeout=10)
with FreezeTime(datetime.datetime(2014, 1, 1)):
Semaphore.set_system_lock(self.conn, 'test_key', timeout)
ttl = Semaphore.get_system_lock(self.conn, 'test_key')
assert ttl == time.time() + timeout
# Should be blocked by system lock
acquired, locks = semaphore1.acquire()
assert not acquired
assert locks == -1
# System lock should still block other locks 1 second before it expires
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, timeout - 1)):
acquired, locks = semaphore1.acquire()
assert not acquired
assert locks == -1
# Wait for system lock to expire
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, timeout)):
acquired, locks = semaphore1.acquire()
assert acquired
assert locks == 1
def test_queue_system_lock(self):
"""Test queue system lock."""
with FreezeTime(datetime.datetime(2014, 1, 1)):
# Queue three tasks
for i in range(0, 3):
task = Task(self.tiger, long_task_ok, queue='a')
task.delay()
self._ensure_queues(queued={'a': 3})
# Ensure we can process one
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# Set system lock so no processing should occur for 10 seconds
self.tiger.set_queue_system_lock('a', 10)
lock_timeout = self.tiger.get_queue_system_lock('a')
assert lock_timeout == time.time() + 10
# Confirm tasks don't get processed within the system lock timeout
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 9)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 2})
# 10 seconds in the future the lock should have expired
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 10)):
worker = Worker(self.tiger)
worker.max_workers_per_queue = 2
worker.run(once=True, force_once=True)
self._ensure_queues(queued={'a': 1})
def test_group_counts_specify_recent_buckets(self, redis):
start = datetime.datetime.now()
counter = RunningCounter(redis, 10, 10, group_name='group')
with FreezeTime(start):
counter.inc(1, 'counter1')
counter.inc(3, 'counter2')
with FreezeTime(start + datetime.timedelta(seconds=counter.interval)):
counter.inc(1, 'counter1')
counter.inc(3, 'counter2')
with FreezeTime(start +
datetime.timedelta(seconds=counter.interval * 2)):
counter.inc(1, 'counter1')
counter.inc(3, 'counter2')
assert counter.group_counts(recent_buckets=1) == {
'counter1': 1.0,
'counter2': 3.0,
}
assert counter.group_counts(recent_buckets=2) == {
'counter1': 2.0,
'counter2': 6.0,
}
assert counter.group_counts() == {
'counter1': 3.0,
'counter2': 9.0,
}
assert counter.group_counts(recent_buckets=10) == {
'counter1': 3.0,
'counter2': 9.0,
}
with pytest.raises(ValueError):
counter.group_counts(recent_buckets=11)
def test_group_buckets_counts(self, redis):
start = datetime.datetime.now()
counter = RunningCounter(redis, 10, 5, group_name='group')
with FreezeTime(start):
counter.inc(1, 'counter1')
counter.inc(1, 'counter2')
with FreezeTime(start +
datetime.timedelta(seconds=counter.interval * 2)):
counter.inc(2, 'counter1')
counter.inc(3, 'counter2')
with FreezeTime(start +
datetime.timedelta(seconds=counter.interval * 4)):
current_bucket = int(time.time()) // 10
counter1_values = [0, 0, 2.0, 0, 1.0]
counter2_values = [0, 0, 3.0, 0, 1.0]
buckets = list(
range(current_bucket, current_bucket - counter.num_buckets,
-1))
counter1_bucket_values = [
BucketCount(bucket=bucket, count=count)
for bucket, count in zip(buckets, counter1_values)
]
counter2_bucket_values = [
BucketCount(bucket=bucket, count=count)
for bucket, count in zip(buckets, counter2_values)
]
assert counter.group_buckets_counts() == {
'counter1': counter1_bucket_values,
'counter2': counter2_bucket_values,
}
assert counter.group_buckets_counts(3) == {
'counter1': counter1_bucket_values[:3],
'counter2': counter2_bucket_values[:3],
}
def test_group_counter_purging(self, redis):
start = datetime.datetime.now()
counter = RunningCounter(redis, 10, 10, group_name='group')
with FreezeTime(start):
counter.inc(1.2, 'test')
assert counter.group() == ['test']
with FreezeTime(start + datetime.timedelta(seconds=counter.window)):
counter.inc(2.2, 'test2')
assert counter.group() == ['test', 'test2']
# One second past window should result in first counter being
# removed from the zset
with FreezeTime(start +
datetime.timedelta(seconds=counter.window + 1)):
counter.inc(2.2, 'test2')
assert counter.group() == ['test2']
def test_multiple_locks(self):
semaphore1 = Semaphore(self.conn,
'test_key',
'id_1',
max_locks=2,
timeout=10)
semaphore2 = Semaphore(self.conn,
'test_key',
'id_2',
max_locks=2,
timeout=10)
semaphore3 = Semaphore(self.conn,
'test_key',
'id_3',
max_locks=2,
timeout=10)
# First two locks should be acquired
with FreezeTime(datetime.datetime(2014, 1, 1)):
acquired, locks = semaphore1.acquire()
assert acquired
assert locks == 1
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 4)):
acquired, locks = semaphore2.acquire()
assert acquired
assert locks == 2
# Third lock should fail
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 6)):
acquired, locks = semaphore3.acquire()
assert not acquired
assert locks == 2
semaphore2.release()
# Releasing one of the existing locks should let a new lock succeed
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 9)):
acquired, locks = semaphore3.acquire()
assert acquired
assert locks == 2
def test_semaphores_renew(self):
semaphore1 = Semaphore(self.conn,
'test_key',
'id_1',
max_locks=1,
timeout=10)
semaphore2 = Semaphore(self.conn,
'test_key',
'id_2',
max_locks=1,
timeout=10)
with FreezeTime(datetime.datetime(2014, 1, 1)):
acquired, locks = semaphore1.acquire()
assert acquired
assert locks == 1
# Renew 5 seconds into lock timeout window
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 5)):
acquired, locks = semaphore1.renew()
assert acquired
assert locks == 1
# Fail getting a lock
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 14)):
acquired, locks = semaphore2.acquire()
assert not acquired
assert locks == 1
# Successful getting lock after renewed timeout window passes
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 15)):
acquired, locks = semaphore2.acquire()
assert acquired
assert locks == 1
# Fail renewing
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 15)):
acquired, locks = semaphore1.renew()
assert not acquired
assert locks == 1
def test_simple_semaphore(self):
"""Test semaphore."""
semaphore1 = Semaphore(self.conn,
'test_key',
'id_1',
max_locks=1,
timeout=10)
semaphore2 = Semaphore(self.conn,
'test_key',
'id_2',
max_locks=1,
timeout=10)
# Get lock and then release
with FreezeTime(datetime.datetime(2014, 1, 1)):
acquired, locks = semaphore1.acquire()
assert acquired
assert locks == 1
semaphore1.release()
# Get a new lock after releasing old
with FreezeTime(datetime.datetime(2014, 1, 1)):
acquired, locks = semaphore2.acquire()
assert acquired
assert locks == 1
# Fail getting second lock while still inside time out period
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 9)):
acquired, locks = semaphore1.acquire()
assert not acquired
assert locks == 1
# Successful getting lock after semaphore2 times out
with FreezeTime(datetime.datetime(2014, 1, 1, 0, 0, 10)):
acquired, locks = semaphore1.acquire()
assert acquired
assert locks == 1
def test_purge_errored_tasks_older_than(self):
task_timestamps = [
datetime.datetime(2015, 1, 1),
datetime.datetime(2016, 1, 1),
datetime.datetime(2017, 1, 1),
datetime.datetime(2018, 1, 1),
]
for task_timestamp in task_timestamps:
with FreezeTime(task_timestamp):
self.tiger.delay(exception_task)
Worker(self.tiger).run(once=True)
self._ensure_queues(queued={'default': 0}, error={'default': 4})
_, tasks = Task.tasks_from_queue(self.tiger, 'default', 'error')
actual_timestamps = [task.ts for task in tasks]
assert task_timestamps == actual_timestamps
assert 2 == self.tiger.purge_errored_tasks(
last_execution_before=datetime.datetime(2016, 6, 1))
self._ensure_queues(queued={'default': 0}, error={'default': 2})
def test_main(self, redis):
name = 'test'
num_buckets = 5
interval = 5
# Start counter now
now = start = datetime.datetime.utcnow().replace(second=0, minute=0)
with FreezeTime(now):
counter = RunningCounter(
redis,
interval,
num_buckets,
name,
)
# Add two values to current bucket
counter.inc(1)
counter.inc(1.2)
buckets_counts = counter.buckets_counts()
bucket = int(time.time()) // interval
assert buckets_counts == [
BucketCount(bucket, 2.2),
BucketCount(bucket - 1, 0),
BucketCount(bucket - 2, 0),
BucketCount(bucket - 3, 0),
BucketCount(bucket - 4, 0),
]
assert counter.count() == 2.2
# Move half way into window and add value to bucket
now = start + datetime.timedelta(seconds=int(num_buckets * interval /
2))
with FreezeTime(now):
counter.inc(2.3)
buckets_counts = counter.buckets_counts()
new_bucket = int(time.time()) // interval
assert buckets_counts == [
BucketCount(new_bucket, 2.3),
BucketCount(new_bucket - 1, 0),
BucketCount(new_bucket - 2, 2.2),
BucketCount(new_bucket - 3, 0),
BucketCount(new_bucket - 4, 0),
]
assert counter.count() == 4.5
# Move forward enough to drop first bucket
now = start + datetime.timedelta(seconds=num_buckets * interval + 1)
with FreezeTime(now):
buckets_counts = counter.buckets_counts()
assert buckets_counts == [
BucketCount(new_bucket + 3, 0),
BucketCount(new_bucket + 2, 0),
BucketCount(new_bucket + 1, 0),
BucketCount(new_bucket, 2.3),
BucketCount(new_bucket - 1, 0),
]
assert counter.count() == 2.3
# Move forward enough to drop all buckets
now = start + datetime.timedelta(seconds=num_buckets * interval +
int(num_buckets * interval / 2))
with FreezeTime(now):
buckets_counts = counter.buckets_counts()
current_bucket = int(time.time()) // interval
assert buckets_counts == [
BucketCount(current_bucket, 0),
BucketCount(current_bucket - 1, 0),
BucketCount(current_bucket - 2, 0),
BucketCount(current_bucket - 3, 0),
BucketCount(current_bucket - 4, 0),
]
assert counter.count() == 0