def test_put_returns_uid_string():
queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex,
autoclean_interval=1)
# TODO: mock uuid.uuid4 here
uid = queue.put('blub')
assert len(uid) == 36
assert type(uid) is str
def test_decode_responses_false():
queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex)
unicode_string = unichr(3456) + u('abcd') + unichr(3421)
queue.put(unicode_string)
return_val = queue.get()[1]
assert isinstance(return_val, bytes)
assert nativestr(unicode_string) == nativestr(return_val)
def test_decode_responses_false():
queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex)
unicode_string = unichr(3456) + u('abcd') + unichr(3421)
queue.put(unicode_string)
return_val = queue.get()[1]
assert isinstance(return_val, bytes)
assert nativestr(unicode_string) == nativestr(return_val)
def test_decode_responses_true():
queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex,
decode_responses=True)
unicode_string = unichr(3456) + u('abcd') + unichr(3421)
queue.put(unicode_string)
return_val = queue.get()[1]
assert isinstance(return_val, unicode)
assert unicode_string == return_val
def test_put_returns_uid_string():
queue = SafeRedisQueue(
name='saferedisqueue-test-%s' % uuid1().hex,
autoclean_interval=1)
# TODO: mock uuid.uuid4 here
uid = queue.put('blub')
assert len(uid) == 36
assert type(uid) is str
def test_decode_responses_true():
queue = SafeRedisQueue(
name='saferedisqueue-test-%s' % uuid1().hex,
decode_responses=True)
unicode_string = unichr(3456) + u('abcd') + unichr(3421)
queue.put(unicode_string)
return_val = queue.get()[1]
assert isinstance(return_val, unicode)
assert unicode_string == return_val
def __init__(self, s_file_id):
super().__init__(s_file_id)
self.__q = SafeRedisQueue(url=settings.REDIS_URL,
name=self.q_name_fmt.format(s_file_id),
autoclean_interval=150,
serializer=pickle)
from data_import.models import StandardizedFile
s_file = StandardizedFile.objects.get(id=s_file_id)
self.vintage = s_file.upload
def test_put_internally_converts_uuid_to_str():
queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex,
autoclean_interval=1)
# mock it
queue._redis = mock.Mock()
queue._redis.pipeline.return_value = pipeline_mock = mock.Mock()
pipeline_mock.hset.return_value = hset_mock = mock.Mock()
uid = queue.put('blub')
pipeline_mock.hset.assert_called_with(queue.ITEMS_KEY, uid, 'blub')
hset_mock.lpush.assert_called_with(queue.QUEUE_KEY, uid)
class ReviewQueue(TableNamesMixin):
def __init__(self, s_file_id):
super().__init__(s_file_id)
self.__q = SafeRedisQueue(url=settings.REDIS_URL,
name=self.q_name_fmt.format(s_file_id),
autoclean_interval=150,
serializer=pickle)
from data_import.models import StandardizedFile
s_file = StandardizedFile.objects.get(id=s_file_id)
self.vintage = s_file.upload
@property
def remaining(self):
return self.__q._redis.hlen(self.__q.ITEMS_KEY)
def add(self, item):
'''
Add an item to the queue, for the first time.
'''
return self.__q.put(item)
def checkout(self, timeout=-1):
'''
Get an item from the queue. By default, do not block.
If blocking is desired, set :timeout to time in seconds
to wait before returning None.
'''
return self.__q.get(timeout=timeout)
def remove(self, uid):
'''
Remove the given item from the queue.
'''
return self.__q.ack(uid)
def replace(self, uid):
'''
If an operation fails, put the given item back in
the queue.
'''
return self.__q.fail(uid)
def flush(self):
'''
Remove everything from the queue.
'''
self.__q._redis.flushdb()
def match_or_create(self):
raise NotImplementedError
def test_put_internally_converts_uuid_to_str():
queue = SafeRedisQueue(
name='saferedisqueue-test-%s' % uuid1().hex,
autoclean_interval=1)
# mock it
queue._redis = mock.Mock()
queue._redis.pipeline.return_value = pipeline_mock = mock.Mock()
pipeline_mock.hset.return_value = hset_mock = mock.Mock()
uid = queue.put('blub')
pipeline_mock.hset.assert_called_with(queue.ITEMS_KEY, uid, 'blub')
hset_mock.lpush.assert_called_with(queue.QUEUE_KEY, uid)
def test_serializer_calls():
serializer_mock = mock.Mock()
serializer_mock.dumps.return_value = '{"dumps": "return"}'
serializer_mock.loads.return_value = {"loads": "return"}
queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex,
autoclean_interval=1,
serializer=serializer_mock)
item = {'test': 'good', 'values': ['a', 'b', 'c']}
queue.push(item)
serializer_mock.dumps.assert_called_with(item)
assert queue.pop()[1] == {"loads": "return"}
serializer_mock.loads.assert_called_with(b('{"dumps": "return"}'))
def test_lua_rename_scripts(func_name, ok_return_val, err_return_val):
queue = SafeRedisQueue()
key1 = 'test_saferedisqueue_' + uuid1().hex
key2 = 'test_saferedisqueue_' + uuid1().hex
key3 = 'test_saferedisqueue_' + uuid1().hex
key4 = 'test_saferedisqueue_' + uuid1().hex
func = getattr(queue, '_redis_' + func_name)
assert queue._redis.exists(key1) is False
assert queue._redis.exists(key2) is False
queue._redis.set(key1, 'foobar')
assert queue._redis.exists(key1) is True
assert queue._redis.exists(key2) is False
assert func(keys=[key1, key2]) == ok_return_val
assert queue._redis.exists(key1) is False
assert queue._redis.get(key2) == b'foobar'
assert func(keys=[key1, key2]) == b'OK'
assert func(keys=[key3, key2]) == b'OK'
queue._redis.set(key4, 'foobar')
assert func(keys=[key4, key2]) == err_return_val
# cleanup
queue._redis.delete(key1)
queue._redis.delete(key2)
queue._redis.delete(key3)
queue._redis.delete(key4)
def test_serializer_calls():
serializer_mock = mock.Mock()
serializer_mock.dumps.return_value = '{"dumps": "return"}'
serializer_mock.loads.return_value = {"loads": "return"}
queue = SafeRedisQueue(
name='saferedisqueue-test-%s' % uuid1().hex,
autoclean_interval=1,
serializer=serializer_mock
)
item = {'test': 'good', 'values': ['a', 'b', 'c']}
queue.push(item)
serializer_mock.dumps.assert_called_with(item)
assert queue.pop()[1] == {"loads": "return"}
serializer_mock.loads.assert_called_with(b('{"dumps": "return"}'))
def test_serializer(serializer):
queue = SafeRedisQueue(
name='saferedisqueue-test-%s' % uuid1().hex,
autoclean_interval=1,
serializer=serializer
)
item = {'test': 'good', 'values': ['a', 'b', 'c']}
# Test when there is an element
queue.push(item)
uid_item, payload_item = queue.pop()
assert type(item) == type(payload_item)
assert item == payload_item
# Test when there is no element and it times out
uid_item, payload_item = queue.pop(timeout=1)
assert None == uid_item
assert None == payload_item
def test_serializer(serializer):
queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex,
autoclean_interval=1,
serializer=serializer)
item = {'test': 'good', 'values': ['a', 'b', 'c']}
# Test when there is an element
queue.push(item)
uid_item, payload_item = queue.pop()
assert type(item) == type(payload_item)
assert item == payload_item
# Test when there is no element and it times out
uid_item, payload_item = queue.pop(timeout=1)
assert None == uid_item
assert None == payload_item
class ReviewQueue(TableNamesMixin):
def __init__(self, s_file_id):
super().__init__(s_file_id)
self.__q = SafeRedisQueue(url=settings.REDIS_URL,
name=self.q_name_fmt.format(s_file_id),
autoclean_interval=150,
serializer=pickle)
from data_import.models import StandardizedFile
s_file = StandardizedFile.objects.get(id=s_file_id)
self.vintage = s_file.upload
@property
def remaining(self):
return self.__q._redis.hlen(self.__q.ITEMS_KEY)
def add(self, item):
'''
Add an item to the queue, for the first time.
'''
return self.__q.put(item)
def checkout(self, timeout=-1):
'''
Get an item from the queue. By default, do not block.
If blocking is desired, set :timeout to time in seconds
to wait before returning None.
'''
return self.__q.get(timeout=timeout)
def remove(self, uid):
'''
Remove the given item from the queue.
'''
return self.__q.ack(uid)
def replace(self, uid):
'''
If an operation fails, put the given item back in
the queue.
'''
return self.__q.fail(uid)
def flush(self):
empty = False
while not empty:
uid, item = self.checkout()
if not item:
empty = True
else:
item['id'] = uid
try:
self.match_or_create(item)
except:
# TO-DO: Handle better.
# It seems like the same item can be checked out 2x...
pass
def match_or_create(self):
raise NotImplementedError
def test_autocleanup():
queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex,
autoclean_interval=1)
queue.put('bad')
queue.put('good')
assert queue._redis.llen(queue.QUEUE_KEY) == 2
assert queue._redis.llen(queue.ACKBUF_KEY) == 0
assert queue._redis.llen(queue.BACKUP) == 0
uid_bad, payload_bad = queue.get()
# Pop triggered first autoclean run before popping. At that time the
# ackbuf was still empty, so nothing was moved to backup. But the
# backup lock was set, to delay the next autoclean run for
# autoclean_interval seconds.
assert queue._redis.llen(queue.QUEUE_KEY) == 1
assert queue._redis.llen(queue.ACKBUF_KEY) == 1 # bad item
assert queue._redis.llen(queue.BACKUP) == 0
uid_good, payload_good = queue.get()
# Autoclean started but instantly aborted due to backup lock.
assert queue._redis.llen(queue.ACKBUF_KEY) == 2
assert queue._redis.llen(queue.BACKUP) == 0
assert queue._redis.llen(queue.QUEUE_KEY) == 0
queue.ack(uid_good) # done with that one
assert queue._redis.llen(queue.ACKBUF_KEY) == 1 # bad item
assert queue._redis.llen(queue.BACKUP) == 0
assert queue._redis.llen(queue.QUEUE_KEY) == 0
# Pop after a autoclean_interval triggers cleanup internally
time.sleep(1.2)
assert queue.get(timeout=-1) == (None, None)
assert queue._redis.llen(queue.ACKBUF_KEY) == 0
assert queue._redis.llen(queue.BACKUP) == 1
assert queue._redis.llen(queue.QUEUE_KEY) == 0
# Next pop triggers autoclean again; requeus; pops bad item again
time.sleep(1.2)
assert queue.get(timeout=-1) == (uid_bad, payload_bad)
# After pop, queue is empty again, item waiting in ackbuf
assert queue._redis.llen(queue.ACKBUF_KEY) == 1
assert queue._redis.llen(queue.BACKUP) == 0
assert queue._redis.llen(queue.QUEUE_KEY) == 0
def test_autocleanup():
queue = SafeRedisQueue(
name='saferedisqueue-test-%s' % uuid1().hex,
autoclean_interval=1)
queue.put('bad')
queue.put('good')
assert queue._redis.llen(queue.QUEUE_KEY) == 2
assert queue._redis.llen(queue.ACKBUF_KEY) == 0
assert queue._redis.llen(queue.BACKUP) == 0
uid_bad, payload_bad = queue.get()
# Pop triggered first autoclean run before popping. At that time the
# ackbuf was still empty, so nothing was moved to backup. But the
# backup lock was set, to delay the next autoclean run for
# autoclean_interval seconds.
assert queue._redis.llen(queue.QUEUE_KEY) == 1
assert queue._redis.llen(queue.ACKBUF_KEY) == 1 # bad item
assert queue._redis.llen(queue.BACKUP) == 0
uid_good, payload_good = queue.get()
# Autoclean started but instantly aborted due to backup lock.
assert queue._redis.llen(queue.ACKBUF_KEY) == 2
assert queue._redis.llen(queue.BACKUP) == 0
assert queue._redis.llen(queue.QUEUE_KEY) == 0
queue.ack(uid_good) # done with that one
assert queue._redis.llen(queue.ACKBUF_KEY) == 1 # bad item
assert queue._redis.llen(queue.BACKUP) == 0
assert queue._redis.llen(queue.QUEUE_KEY) == 0
# Pop after a autoclean_interval triggers cleanup internally
time.sleep(1.2)
assert queue.get(timeout=-1) == (None, None)
assert queue._redis.llen(queue.ACKBUF_KEY) == 0
assert queue._redis.llen(queue.BACKUP) == 1
assert queue._redis.llen(queue.QUEUE_KEY) == 0
# Next pop triggers autoclean again; requeus; pops bad item again
time.sleep(1.2)
assert queue.get(timeout=-1) == (uid_bad, payload_bad)
# After pop, queue is empty again, item waiting in ackbuf
assert queue._redis.llen(queue.ACKBUF_KEY) == 1
assert queue._redis.llen(queue.BACKUP) == 0
assert queue._redis.llen(queue.QUEUE_KEY) == 0