def test_15_celery_flush_buffer_trip_lock(self): config.BUFFERING = True config.BUFFER_GRACE_PERIOD = 2 config.BUFFER_BLOCK_SIZE = 2 models.Record.bulk = mock_bulk_blocked models.Record.pull = mock_pull global ARCHIVE # load some records into the buffer records = [ {"identifier" : [{"canonical" : "doi:123"}]}, {"identifier" : [{"canonical" : "doi:456"}]}, {"identifier" : [{"canonical" : "doi:789"}]} ] for record in records: models.Record.store(record) # run the thing the first time result = models.flush_buffer() assert result # now straight away run it a second time, which should trip the lock result2 = models.flush_buffer() assert not result2 # now wait until the lock should be released and try again time.sleep(2.1) result3 = models.flush_buffer() assert result3
def test_13_celery_flush_buffer_prelocked(self): config.BUFFERING = True # manually set a lock on the process client = redis.StrictRedis(host=config.REDIS_BUFFER_HOST, port=config.REDIS_BUFFER_PORT, db=config.REDIS_BUFFER_DB) client.set("flush_buffer_lock", "lock") result = models.flush_buffer() assert not result
def test_14_celery_flush_buffer_success(self): config.BUFFERING = True config.BUFFER_GRACE_PERIOD = 30 config.BUFFER_BLOCK_SIZE = 2 models.Record.bulk = mock_bulk_blocked models.Record.pull = mock_pull global ARCHIVE # load some records into the buffer records = [ {"identifier" : [{"canonical" : "doi:123"}]}, {"identifier" : [{"canonical" : "doi:456"}]}, {"identifier" : [{"canonical" : "doi:789"}]} ] for record in records: models.Record.store(record) result = models.flush_buffer() assert result # check that everything got archived assert len(ARCHIVE) == 2, ARCHIVE # the archive in this case is a list of lists, and there should be 2 lists assert len(ARCHIVE[0]) == 2 # the first block should have been 2 long assert len(ARCHIVE[1]) == 1 # the second block should have been 1 long # check that the things are still in the buffer obj1 = models.Record.check_archive("doi:123") obj2 = models.Record.check_archive("doi:456") obj3 = models.Record.check_archive("doi:789") assert obj1['identifier'][0]['canonical'] in ["doi:123", "doi:456", "doi:789"] assert obj2['identifier'][0]['canonical'] in ["doi:123", "doi:456", "doi:789"] assert obj3['identifier'][0]['canonical'] in ["doi:123", "doi:456", "doi:789"] # check that the lock is still set client = redis.StrictRedis(host=config.REDIS_BUFFER_HOST, port=config.REDIS_BUFFER_PORT, db=config.REDIS_BUFFER_DB) lock = client.get("flush_buffer_lock") assert lock is not None
def test_12_celery_flush_buffer_no_buffering(self): config.BUFFERING = False result = models.flush_buffer() assert not result