def test_data_disk_loader_with_ops_meta(tmpdir):
cache_dir = _prepare_dir(tmpdir)
src = SourceTest({'samples-dir': str(tmpdir)})
data = Data(input=src,
cache_dir=cache_dir,
ops=[AppendStringOperation()],
cache_input='disk',
cache_output=False)
_test_data_meta(data)
def test_data_mem_out_loader_ops_num_samples(tmpdir):
cache_dir = _prepare_dir(tmpdir)
src = SourceTest({'samples-dir': str(tmpdir)})
data = Data(input=src,
cache_dir=cache_dir,
ops=[AppendStringOperation()],
cache_input=False,
cache_output='mem')
_test_data_num_samples(data)
def test_data_live_loader_meta(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
data:
input:
type: test
cache: none
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_meta(data)
def test_data_disk_loader_read_samples(tmpdir):
cache_dir = _prepare_dir(tmpdir)
src = SourceTest({
'samples-dir': str(tmpdir),
'test-split': 2,
'val-split': 2
})
data = Data(input=src,
cache_dir=cache_dir,
cache_input='disk',
cache_output=False)
_test_data_read_samples(data)
def test_data_disk_loader_num_samples(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
data:
input:
type: test
cache: disk-in
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_num_samples(data)
def test_data_disk_out_loader_with_ops(tmpdir):
cache_dir = _prepare_dir(tmpdir)
src = SourceTest({
'samples-dir': str(tmpdir),
'test-split': 2,
'val-split': 2
})
data = Data(input=src,
cache_dir=cache_dir,
ops=[AppendStringOperation()],
cache_input=False,
cache_output='disk')
_test_data_read_samples_transformed(data)
def test_data_mem_loader_read_samples(tmpdir):
# TEST THIS!
cache_dir = _prepare_dir(tmpdir)
st = SourceTest({
'samples-dir': str(tmpdir),
'test-split': 2,
'val-split': 2
})
data = Data(input=st,
cache_dir=cache_dir,
cache_input='mem',
cache_output=False)
_test_data_read_samples(data)
def test_data_mem_out_loader_with_multiplier_ops(tmpdir):
cache_dir = _prepare_dir(tmpdir)
src = SourceTest({
'samples-dir': str(tmpdir),
'test-split': 2,
'val-split': 2
})
data = Data(input=src,
cache_dir=cache_dir,
ops=[AugmentOperation(variants=2)],
cache_input=False,
cache_output='mem')
_test_data_read_samples_transformed_x2(data)
def test_data_disk_loader_with_multiplier_ops_between(tmpdir):
cache_dir = _prepare_dir(tmpdir)
src = SourceTest({
'samples-dir': str(tmpdir),
'test-split': 2,
'val-split': 2
})
data = Data(input=src,
cache_dir=cache_dir,
ops=[AugmentOperation(variants=2)],
cache_input='disk',
cache_output=False)
_test_data_read_samples_x2_between(data)
def test_data_mem_out_loader_ops_num_samples(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
data:
input:
type: test
preprocess:
- op: append
cache: mem
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_num_samples(data)
def test_data_mem_loader_read_samples(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
test-split: 2
val-split: 2
data:
input:
type: test
cache: mem-in
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_read_samples(data)
def test_data_disk_loader_with_ops_meta(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
data:
input:
type: test
preprocess:
- op: append
cache: disk-in
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_meta(data)
def test_data_disk_out_loader_with_ops(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
test-split: 2
val-split: 2
data:
input:
type: test
preprocess:
- op: append
cache: disk
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_read_samples_transformed(data)
def test_data_disk_loader_with_multiplier_ops_between(tmpdir):
_prepare_dir(tmpdir)
project_file = tmpdir.join("vergeml.yaml")
project_file.write("""\
test-split: 2
val-split: 2
data:
input:
type: test
preprocess:
- op: augment
variants: 2
cache: disk-in
""")
env = Environment(project_dir=str(tmpdir), project_file=str(project_file), plugins=PLUGINS)
data = Data(env, plugins=PLUGINS)
_test_data_read_samples_x2_between(data)
def test_data_live_loader_meta(tmpdir):
cache_dir = _prepare_dir(tmpdir)
st = SourceTest({'samples-dir': str(tmpdir)})
data = Data(input=st, cache_dir=cache_dir, cache_input=False)
_test_data_meta(data)
def test_data_disk_loader_meta(tmpdir):
cache_dir = _prepare_dir(tmpdir)
src = SourceTest({'samples-dir': str(tmpdir)})
data = Data(input=src, cache_dir=cache_dir, cache_input='disk')
_test_data_meta(data)
def test_data_live_loader_num_samples(tmpdir):
cache_dir = _prepare_dir(tmpdir)
src = SourceTest({'samples-dir': str(tmpdir)})
data = Data(input=src, cache_dir=cache_dir, cache_input=False)
_test_data_num_samples(data)
def data(self):
"""Return the data loader.
"""
if not self._data:
self._data = Data(self, plugins=self.plugins)
return self._data
def test_data_mem_loader_num_samples(tmpdir):
cache_dir = _prepare_dir(tmpdir)
st = SourceTest({'samples-dir': str(tmpdir)})
data = Data(input=st, cache_dir=cache_dir, cache_input='mem')
_test_data_num_samples(data)