def test_loader_broken_image():
manifest = random_manifest(2, broken_image_index=1)
config = generic_config(manifest.name, batch_size)
with pytest.raises(Exception) as ex:
dl = DataLoader(config)
assert 'Decoding image failed due to invalid data in the image file' in str(ex)
def test_loader_invalid_manifest():
filename = tempfile.mkstemp()[1]
config = generic_config(invalid_image(filename), batch_size)
with pytest.raises(Exception) as ex:
dl = DataLoader(config)
assert 'must be string, but is null' in str(ex)
def test_loader_repeat_iter():
# NOTE: manifest needs to stay in scope until DataLoader has read it.
manifest = random_manifest(10)
config = generic_config(manifest.name, batch_size)
dl = DataLoader(config)
assert len(list(iter(dl))) == math.ceil(10. / batch_size)
def test_loader_exception_iter():
# NOTE: manifest needs to stay in scope until DataLoader has read it.
manifest = random_manifest(10, 2)
config = generic_config(manifest.name)
dl = DataLoader(config, gen_backend('cpu'))
assert len(list(iter(dl))) == 4
def test_loader_invalid_config_type():
manifest = random_manifest(10)
config = generic_config(manifest.name)
config['type'] = 'invalid type name'
with pytest.raises(Exception) as ex:
dl = DataLoader(config, gen_backend('cpu'))
def test_loader():
# NOTE: manifest needs to stay in scope until DataLoader has read it.
for i in range(1, 10):
manifest = random_manifest(i)
config = generic_config(manifest.name, batch_size)
dl = DataLoader(config)
assert len(list(iter(dl))) == math.ceil(float(i) / batch_size)
def test_loader_missing_config_field():
manifest = random_manifest(10)
config = generic_config(manifest.name, batch_size)
del config['etl'][0]["height"]
with pytest.raises(RuntimeError) as ex:
dl = DataLoader(config)
assert 'height' in str(ex)
def test_loader_invalid_config_type():
manifest = random_manifest(10)
config = generic_config(manifest.name, batch_size)
config["etl"][0]["type"] = 'invalid type name'
with pytest.raises(RuntimeError) as ex:
dl = DataLoader(config)
assert 'unsupported' in str(ex)
def test_loader_exception_next():
# NOTE: manifest needs to stay in scope until DataLoader has read it.
manifest = random_manifest(10, 2)
config = generic_config(manifest.name)
dl = DataLoader(config, gen_backend('cpu'))
dl.next()
with pytest.raises(LoaderRuntimeError):
dl.next()
def test_loader_missing_config_field():
manifest = random_manifest(10)
config = generic_config(manifest.name)
del config['image']
with pytest.raises(Exception) as ex:
dl = DataLoader(config, gen_backend('cpu'))
assert 'image' in str(ex)
def test_loader_broken_image_next():
manifest = random_manifest(9, broken_image_index=8)
config = generic_config(manifest.name, batch_size)
dl = DataLoader(config)
with pytest.raises(Exception) as ex:
for i in range(5):
dl.next()
assert 'Decoding image failed due to invalid data in the image file' in str(ex)
dl2 = DataLoader(config)
with pytest.raises(Exception) as ex:
for data in dl2:
pass
assert 'Decoding image failed due to invalid data in the image file' in str(ex)
def test_loader_exception_iter():
# NOTE: manifest needs to stay in scope until DataLoader has read it.
cwd = os.getcwd()
dir_path = os.path.dirname(os.path.realpath(__file__))
os.chdir(dir_path+'/test_data')
manifest = open("manifest.tsv")
config = generic_config(manifest.name, batch_size)
dl = DataLoader(config)
num_of_manifest_entries = 120.
assert len(list(iter(dl))) == math.ceil(num_of_manifest_entries/batch_size)
manifest.close()
os.chdir(cwd)
def test_loader_exception_next():
# NOTE: manifest needs to stay in scope until DataLoader has read it.
cwd = os.getcwd()
dir_path = os.path.dirname(os.path.realpath(__file__))
os.chdir(dir_path + '/test_data')
manifest = open("manifest.tsv")
config = generic_config(manifest.name, batch_size)
dl = DataLoader(config)
num_of_batches_in_manifest = 60
for x in range(0, num_of_batches_in_manifest):
next(dl)
with pytest.raises(StopIteration) as ex:
next(dl)
manifest.close()
os.chdir(cwd)
def test_loader_non_existant_manifest():
config = generic_config('/this_manifest_file_does_not_exist', batch_size)
with pytest.raises(RuntimeError) as ex:
dl = DataLoader(config)
assert "doesn't exist" in str(ex)
def test_loader_invalid_manifest():
filename = tempfile.mkstemp()[1]
config = generic_config(invalid_image(filename))
with pytest.raises(Exception):
dl = DataLoader(config, gen_backend('cpu'))
def test_loader_non_existant_manifest():
config = generic_config('/this_manifest_file_does_not_exist')
with pytest.raises(Exception):
dl = DataLoader(config, gen_backend('cpu'))
