def test_15_celery_flush_buffer_trip_lock(self):
config.BUFFERING = True
config.BUFFER_GRACE_PERIOD = 2
config.BUFFER_BLOCK_SIZE = 2
models.Record.bulk = mock_bulk_blocked
models.Record.pull = mock_pull
global ARCHIVE
# load some records into the buffer
records = [
{"identifier" : [{"canonical" : "doi:123"}]},
{"identifier" : [{"canonical" : "doi:456"}]},
{"identifier" : [{"canonical" : "doi:789"}]}
]
for record in records:
models.Record.store(record)
# run the thing the first time
result = models.flush_buffer()
assert result
# now straight away run it a second time, which should trip the lock
result2 = models.flush_buffer()
assert not result2
# now wait until the lock should be released and try again
time.sleep(2.1)
result3 = models.flush_buffer()
assert result3
def test_13_celery_flush_buffer_prelocked(self):
config.BUFFERING = True
# manually set a lock on the process
client = redis.StrictRedis(host=config.REDIS_BUFFER_HOST, port=config.REDIS_BUFFER_PORT, db=config.REDIS_BUFFER_DB)
client.set("flush_buffer_lock", "lock")
result = models.flush_buffer()
assert not result
def test_14_celery_flush_buffer_success(self):
config.BUFFERING = True
config.BUFFER_GRACE_PERIOD = 30
config.BUFFER_BLOCK_SIZE = 2
models.Record.bulk = mock_bulk_blocked
models.Record.pull = mock_pull
global ARCHIVE
# load some records into the buffer
records = [
{"identifier" : [{"canonical" : "doi:123"}]},
{"identifier" : [{"canonical" : "doi:456"}]},
{"identifier" : [{"canonical" : "doi:789"}]}
]
for record in records:
models.Record.store(record)
result = models.flush_buffer()
assert result
# check that everything got archived
assert len(ARCHIVE) == 2, ARCHIVE # the archive in this case is a list of lists, and there should be 2 lists
assert len(ARCHIVE[0]) == 2 # the first block should have been 2 long
assert len(ARCHIVE[1]) == 1 # the second block should have been 1 long
# check that the things are still in the buffer
obj1 = models.Record.check_archive("doi:123")
obj2 = models.Record.check_archive("doi:456")
obj3 = models.Record.check_archive("doi:789")
assert obj1['identifier'][0]['canonical'] in ["doi:123", "doi:456", "doi:789"]
assert obj2['identifier'][0]['canonical'] in ["doi:123", "doi:456", "doi:789"]
assert obj3['identifier'][0]['canonical'] in ["doi:123", "doi:456", "doi:789"]
# check that the lock is still set
client = redis.StrictRedis(host=config.REDIS_BUFFER_HOST, port=config.REDIS_BUFFER_PORT, db=config.REDIS_BUFFER_DB)
lock = client.get("flush_buffer_lock")
assert lock is not None
def test_12_celery_flush_buffer_no_buffering(self):
config.BUFFERING = False
result = models.flush_buffer()
assert not result
