def test_put_returns_uid_string(): queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex, autoclean_interval=1) # TODO: mock uuid.uuid4 here uid = queue.put('blub') assert len(uid) == 36 assert type(uid) is str
def test_decode_responses_false(): queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex) unicode_string = unichr(3456) + u('abcd') + unichr(3421) queue.put(unicode_string) return_val = queue.get()[1] assert isinstance(return_val, bytes) assert nativestr(unicode_string) == nativestr(return_val)
def test_decode_responses_true(): queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex, decode_responses=True) unicode_string = unichr(3456) + u('abcd') + unichr(3421) queue.put(unicode_string) return_val = queue.get()[1] assert isinstance(return_val, unicode) assert unicode_string == return_val
def test_put_returns_uid_string(): queue = SafeRedisQueue( name='saferedisqueue-test-%s' % uuid1().hex, autoclean_interval=1) # TODO: mock uuid.uuid4 here uid = queue.put('blub') assert len(uid) == 36 assert type(uid) is str
def test_decode_responses_true(): queue = SafeRedisQueue( name='saferedisqueue-test-%s' % uuid1().hex, decode_responses=True) unicode_string = unichr(3456) + u('abcd') + unichr(3421) queue.put(unicode_string) return_val = queue.get()[1] assert isinstance(return_val, unicode) assert unicode_string == return_val
def __init__(self, s_file_id): super().__init__(s_file_id) self.__q = SafeRedisQueue(url=settings.REDIS_URL, name=self.q_name_fmt.format(s_file_id), autoclean_interval=150, serializer=pickle) from data_import.models import StandardizedFile s_file = StandardizedFile.objects.get(id=s_file_id) self.vintage = s_file.upload
def test_put_internally_converts_uuid_to_str(): queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex, autoclean_interval=1) # mock it queue._redis = mock.Mock() queue._redis.pipeline.return_value = pipeline_mock = mock.Mock() pipeline_mock.hset.return_value = hset_mock = mock.Mock() uid = queue.put('blub') pipeline_mock.hset.assert_called_with(queue.ITEMS_KEY, uid, 'blub') hset_mock.lpush.assert_called_with(queue.QUEUE_KEY, uid)
class ReviewQueue(TableNamesMixin): def __init__(self, s_file_id): super().__init__(s_file_id) self.__q = SafeRedisQueue(url=settings.REDIS_URL, name=self.q_name_fmt.format(s_file_id), autoclean_interval=150, serializer=pickle) from data_import.models import StandardizedFile s_file = StandardizedFile.objects.get(id=s_file_id) self.vintage = s_file.upload @property def remaining(self): return self.__q._redis.hlen(self.__q.ITEMS_KEY) def add(self, item): ''' Add an item to the queue, for the first time. ''' return self.__q.put(item) def checkout(self, timeout=-1): ''' Get an item from the queue. By default, do not block. If blocking is desired, set :timeout to time in seconds to wait before returning None. ''' return self.__q.get(timeout=timeout) def remove(self, uid): ''' Remove the given item from the queue. ''' return self.__q.ack(uid) def replace(self, uid): ''' If an operation fails, put the given item back in the queue. ''' return self.__q.fail(uid) def flush(self): ''' Remove everything from the queue. ''' self.__q._redis.flushdb() def match_or_create(self): raise NotImplementedError
def test_put_internally_converts_uuid_to_str(): queue = SafeRedisQueue( name='saferedisqueue-test-%s' % uuid1().hex, autoclean_interval=1) # mock it queue._redis = mock.Mock() queue._redis.pipeline.return_value = pipeline_mock = mock.Mock() pipeline_mock.hset.return_value = hset_mock = mock.Mock() uid = queue.put('blub') pipeline_mock.hset.assert_called_with(queue.ITEMS_KEY, uid, 'blub') hset_mock.lpush.assert_called_with(queue.QUEUE_KEY, uid)
def test_serializer_calls(): serializer_mock = mock.Mock() serializer_mock.dumps.return_value = '{"dumps": "return"}' serializer_mock.loads.return_value = {"loads": "return"} queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex, autoclean_interval=1, serializer=serializer_mock) item = {'test': 'good', 'values': ['a', 'b', 'c']} queue.push(item) serializer_mock.dumps.assert_called_with(item) assert queue.pop()[1] == {"loads": "return"} serializer_mock.loads.assert_called_with(b('{"dumps": "return"}'))
def test_lua_rename_scripts(func_name, ok_return_val, err_return_val): queue = SafeRedisQueue() key1 = 'test_saferedisqueue_' + uuid1().hex key2 = 'test_saferedisqueue_' + uuid1().hex key3 = 'test_saferedisqueue_' + uuid1().hex key4 = 'test_saferedisqueue_' + uuid1().hex func = getattr(queue, '_redis_' + func_name) assert queue._redis.exists(key1) is False assert queue._redis.exists(key2) is False queue._redis.set(key1, 'foobar') assert queue._redis.exists(key1) is True assert queue._redis.exists(key2) is False assert func(keys=[key1, key2]) == ok_return_val assert queue._redis.exists(key1) is False assert queue._redis.get(key2) == b'foobar' assert func(keys=[key1, key2]) == b'OK' assert func(keys=[key3, key2]) == b'OK' queue._redis.set(key4, 'foobar') assert func(keys=[key4, key2]) == err_return_val # cleanup queue._redis.delete(key1) queue._redis.delete(key2) queue._redis.delete(key3) queue._redis.delete(key4)
def test_serializer_calls(): serializer_mock = mock.Mock() serializer_mock.dumps.return_value = '{"dumps": "return"}' serializer_mock.loads.return_value = {"loads": "return"} queue = SafeRedisQueue( name='saferedisqueue-test-%s' % uuid1().hex, autoclean_interval=1, serializer=serializer_mock ) item = {'test': 'good', 'values': ['a', 'b', 'c']} queue.push(item) serializer_mock.dumps.assert_called_with(item) assert queue.pop()[1] == {"loads": "return"} serializer_mock.loads.assert_called_with(b('{"dumps": "return"}'))
def test_serializer(serializer): queue = SafeRedisQueue( name='saferedisqueue-test-%s' % uuid1().hex, autoclean_interval=1, serializer=serializer ) item = {'test': 'good', 'values': ['a', 'b', 'c']} # Test when there is an element queue.push(item) uid_item, payload_item = queue.pop() assert type(item) == type(payload_item) assert item == payload_item # Test when there is no element and it times out uid_item, payload_item = queue.pop(timeout=1) assert None == uid_item assert None == payload_item
def test_serializer(serializer): queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex, autoclean_interval=1, serializer=serializer) item = {'test': 'good', 'values': ['a', 'b', 'c']} # Test when there is an element queue.push(item) uid_item, payload_item = queue.pop() assert type(item) == type(payload_item) assert item == payload_item # Test when there is no element and it times out uid_item, payload_item = queue.pop(timeout=1) assert None == uid_item assert None == payload_item
class ReviewQueue(TableNamesMixin): def __init__(self, s_file_id): super().__init__(s_file_id) self.__q = SafeRedisQueue(url=settings.REDIS_URL, name=self.q_name_fmt.format(s_file_id), autoclean_interval=150, serializer=pickle) from data_import.models import StandardizedFile s_file = StandardizedFile.objects.get(id=s_file_id) self.vintage = s_file.upload @property def remaining(self): return self.__q._redis.hlen(self.__q.ITEMS_KEY) def add(self, item): ''' Add an item to the queue, for the first time. ''' return self.__q.put(item) def checkout(self, timeout=-1): ''' Get an item from the queue. By default, do not block. If blocking is desired, set :timeout to time in seconds to wait before returning None. ''' return self.__q.get(timeout=timeout) def remove(self, uid): ''' Remove the given item from the queue. ''' return self.__q.ack(uid) def replace(self, uid): ''' If an operation fails, put the given item back in the queue. ''' return self.__q.fail(uid) def flush(self): empty = False while not empty: uid, item = self.checkout() if not item: empty = True else: item['id'] = uid try: self.match_or_create(item) except: # TO-DO: Handle better. # It seems like the same item can be checked out 2x... pass def match_or_create(self): raise NotImplementedError
def test_autocleanup(): queue = SafeRedisQueue(name='saferedisqueue-test-%s' % uuid1().hex, autoclean_interval=1) queue.put('bad') queue.put('good') assert queue._redis.llen(queue.QUEUE_KEY) == 2 assert queue._redis.llen(queue.ACKBUF_KEY) == 0 assert queue._redis.llen(queue.BACKUP) == 0 uid_bad, payload_bad = queue.get() # Pop triggered first autoclean run before popping. At that time the # ackbuf was still empty, so nothing was moved to backup. But the # backup lock was set, to delay the next autoclean run for # autoclean_interval seconds. assert queue._redis.llen(queue.QUEUE_KEY) == 1 assert queue._redis.llen(queue.ACKBUF_KEY) == 1 # bad item assert queue._redis.llen(queue.BACKUP) == 0 uid_good, payload_good = queue.get() # Autoclean started but instantly aborted due to backup lock. assert queue._redis.llen(queue.ACKBUF_KEY) == 2 assert queue._redis.llen(queue.BACKUP) == 0 assert queue._redis.llen(queue.QUEUE_KEY) == 0 queue.ack(uid_good) # done with that one assert queue._redis.llen(queue.ACKBUF_KEY) == 1 # bad item assert queue._redis.llen(queue.BACKUP) == 0 assert queue._redis.llen(queue.QUEUE_KEY) == 0 # Pop after a autoclean_interval triggers cleanup internally time.sleep(1.2) assert queue.get(timeout=-1) == (None, None) assert queue._redis.llen(queue.ACKBUF_KEY) == 0 assert queue._redis.llen(queue.BACKUP) == 1 assert queue._redis.llen(queue.QUEUE_KEY) == 0 # Next pop triggers autoclean again; requeus; pops bad item again time.sleep(1.2) assert queue.get(timeout=-1) == (uid_bad, payload_bad) # After pop, queue is empty again, item waiting in ackbuf assert queue._redis.llen(queue.ACKBUF_KEY) == 1 assert queue._redis.llen(queue.BACKUP) == 0 assert queue._redis.llen(queue.QUEUE_KEY) == 0
def test_autocleanup(): queue = SafeRedisQueue( name='saferedisqueue-test-%s' % uuid1().hex, autoclean_interval=1) queue.put('bad') queue.put('good') assert queue._redis.llen(queue.QUEUE_KEY) == 2 assert queue._redis.llen(queue.ACKBUF_KEY) == 0 assert queue._redis.llen(queue.BACKUP) == 0 uid_bad, payload_bad = queue.get() # Pop triggered first autoclean run before popping. At that time the # ackbuf was still empty, so nothing was moved to backup. But the # backup lock was set, to delay the next autoclean run for # autoclean_interval seconds. assert queue._redis.llen(queue.QUEUE_KEY) == 1 assert queue._redis.llen(queue.ACKBUF_KEY) == 1 # bad item assert queue._redis.llen(queue.BACKUP) == 0 uid_good, payload_good = queue.get() # Autoclean started but instantly aborted due to backup lock. assert queue._redis.llen(queue.ACKBUF_KEY) == 2 assert queue._redis.llen(queue.BACKUP) == 0 assert queue._redis.llen(queue.QUEUE_KEY) == 0 queue.ack(uid_good) # done with that one assert queue._redis.llen(queue.ACKBUF_KEY) == 1 # bad item assert queue._redis.llen(queue.BACKUP) == 0 assert queue._redis.llen(queue.QUEUE_KEY) == 0 # Pop after a autoclean_interval triggers cleanup internally time.sleep(1.2) assert queue.get(timeout=-1) == (None, None) assert queue._redis.llen(queue.ACKBUF_KEY) == 0 assert queue._redis.llen(queue.BACKUP) == 1 assert queue._redis.llen(queue.QUEUE_KEY) == 0 # Next pop triggers autoclean again; requeus; pops bad item again time.sleep(1.2) assert queue.get(timeout=-1) == (uid_bad, payload_bad) # After pop, queue is empty again, item waiting in ackbuf assert queue._redis.llen(queue.ACKBUF_KEY) == 1 assert queue._redis.llen(queue.BACKUP) == 0 assert queue._redis.llen(queue.QUEUE_KEY) == 0