def test_filequeue():
# Test we can create a queue and read from it.
file_queue = dl.FileQueue()
file_queue.load_epochs(files)
sleep(0.5)
for i in range(1000):
file_queue.get(timeout=MAX_TIMEOUT)
def test_imgqueue_repr():
file_queue = dl.FileQueue()
file_queue.load_epochs(files)
sleep(0.5)
img_queue = dl.ImgQueue()
img_queue.start_loaders(file_queue, num_threads=3, img_dir=IMG_DIR)
img_queue.__repr__()
def test_lastbatch():
# Test the status of the last_batch flag is working as expected
file_queue = dl.FileQueue()
file_queue.load_epochs(files, max_epochs=10)
sleep(0.5)
img_queue = dl.ImgQueue()
img_queue.start_loaders(file_queue, num_threads=3, img_dir=IMG_DIR)
sleep(1)
# Get an entire epoch first
data, labels = img_queue.get_batch(len(files))
assert img_queue.last_batch
assert not img_queue.last_batch
# Get batches of 10 images and test the flag works
num_batches = np.ceil(len(files) / 10).astype('int')
for b in range(num_batches - 1):
data, labels = img_queue.get_batch(10)
assert not img_queue.last_batch
data, labels = img_queue.get_batch(10)
assert img_queue.last_batch
# Get batches of 13 images and test the flag works
num_batches = np.ceil(len(files) / 13).astype('int')
for b in range(num_batches - 1):
data, labels = img_queue.get_batch(13)
assert not img_queue.last_batch
data, labels = img_queue.get_batch(13)
assert img_queue.last_batch
def test_imgqueue_nostart():
# Test we get the right exception when we try to read from the image queue
# without starting the loaders
file_queue = dl.FileQueue()
file_queue.load_epochs(files, max_epochs=1)
sleep(0.5)
img_queue = dl.ImgQueue()
with pytest.raises(dl.ImgQueueNotStarted):
img_queue.get_batch(1)
def test_imgqueue_epochreached():
# Test we get the right exception when we've hit the sample limit
file_queue = dl.FileQueue()
file_queue.load_epochs(files, max_epochs=1)
sleep(0.5)
img_queue = dl.ImgQueue()
img_queue.start_loaders(file_queue, num_threads=3, img_dir=IMG_DIR)
sleep(0.5)
img_queue.get_batch(batch_size=len(files))
with pytest.raises(dl.FileQueueDepleted):
img_queue.get_batch(1)
def test_imgqueue():
# Test the imgqueue is working
file_queue = dl.FileQueue()
file_queue.load_epochs(files)
sleep(0.5)
img_queue = dl.ImgQueue()
img_queue.start_loaders(file_queue, num_threads=3, img_dir=IMG_DIR)
# Allow time to load in data
sleep(1)
img_queue.get_batch(100)
def test_filequeue_unique():
# Test that each file in an epoch is unique
file_queue = dl.FileQueue()
file_queue.load_epochs(files, max_epochs=1)
sleep(0.5)
new_files = [
file_queue.get(timeout=MAX_TIMEOUT)
for _ in range(file_queue.epoch_size)
]
import collections
assert collections.Counter(new_files) == collections.Counter(files)
def test_imgqueue_lots():
# Test we get the right behaviour when we try to read lots from the image
# queue without letting it fill up
file_queue = dl.FileQueue()
file_queue.load_epochs(files)
sleep(0.5)
img_queue = dl.ImgQueue()
img_queue.start_loaders(file_queue, num_threads=3, img_dir=IMG_DIR)
img_queue.get_batch(1000)
img_queue.get_batch(1000)
def test_filequeue_max_epochs():
# Test the max_epoch argument works as expected
file_queue = dl.FileQueue()
epoch_size = len(files)
file_queue.load_epochs(files, max_epochs=2)
sleep(0.5)
# Eat up everything
for i in range(2 * epoch_size):
file_queue.get(timeout=MAX_TIMEOUT)
# Make sure the queue is now empty
with pytest.raises(queue.Empty):
file_queue.get(timeout=MAX_TIMEOUT)
def test_imgqueue_loadersalive():
file_queue = dl.FileQueue(maxsize=10)
file_queue.load_epochs(files, max_epochs=1)
sleep(0.5)
assert file_queue.filling
img_queue = dl.ImgQueue(maxsize=10)
img_queue.start_loaders(file_queue, num_threads=3, img_dir=IMG_DIR)
sleep(1)
assert not img_queue.loaders_finished
assert img_queue.filling
img_queue.join_loaders()
assert not file_queue.filling
assert not img_queue.filling
assert img_queue.loaders_finished
def test_filequeue_shuffle():
# Test the shuffling is working
file_queue = dl.FileQueue()
file_queue.load_epochs(files, max_epochs=1)
sleep(0.5)
new_files = [
file_queue.get(timeout=MAX_TIMEOUT)
for _ in range(file_queue.epoch_size)
]
same = True
assert len(files) == len(new_files)
for f1, f2 in zip(files, new_files):
if f1 != f2:
same = False
break
assert not same
def test_transform():
# Test we get the right behaviour when we try to read lots from the image
# queue without letting it fill up
file_queue = dl.FileQueue()
file_queue.load_epochs(files)
sleep(0.5)
img_queue = dl.ImgQueue()
def transform(x):
return x - np.mean(x)
img_queue.start_loaders(file_queue,
num_threads=3,
img_dir=IMG_DIR,
transform=transform)
data, labels = img_queue.get_batch(10)
for im in data:
assert abs(np.mean(im)) <= 0.0001
def test_filequeue_nostart():
# Test handling an empty file queue
file_queue = dl.FileQueue()
with pytest.raises(dl.FileQueueNotStarted):
file_queue.get()
def test_filequeue_emptyfiles():
# Test handling an empty file queue
files = []
file_queue = dl.FileQueue()
with pytest.raises(ValueError):
file_queue.load_epochs(files)
def test_filequeue_repr():
file_queue = dl.FileQueue()
file_queue.load_epochs(files)
sleep(0.5)
file_queue.__repr__()
