def _get_loader(self, cache_input, cache_output):
if cache_input == 'disk':
input_loader = FileCachedLoader(self.cache_dir, self.input, progress_callback=self._progress_callback)
elif cache_input == 'mem':
input_loader = MemoryCachedLoader(self.cache_dir, self.input, progress_callback=self._progress_callback)
else:
input_loader = self.input
if cache_output == 'disk':
return FileCachedLoader(self.cache_dir, input_loader, self.ops, self.output, progress_callback=self._progress_callback)
elif cache_output == 'mem':
return MemoryCachedLoader(self.cache_dir, input_loader, self.ops, self.output, progress_callback=self._progress_callback)
else:
return LiveLoader(self.cache_dir, input_loader, self.ops, self.output)
def test_mem_out_loader_with_ops_meta(tmpdir):
cache_dir = _prepare_dir(tmpdir)
st = SourceTest({'samples-dir': str(tmpdir)})
loader = MemoryCachedLoader(cache_dir,
st,
ops=[AppendStringOperation()],
output=st)
_test_loader_meta(loader)
def test_mem_loader_with_rng_no_reset(tmpdir):
cache_dir = _prepare_dir(tmpdir)
st = SourceTest({
'samples-dir': str(tmpdir),
'test-split': 2,
'val-split': 2
})
loader = MemoryCachedLoader(cache_dir, st)
_test_loader_with_rng_no_reset(loader)
def test_mem_loader_ops_num_samples(tmpdir):
cache_dir = _prepare_dir(tmpdir)
st = SourceTest({'samples-dir': str(tmpdir)})
loader = MemoryCachedLoader(cache_dir, st)
loader2 = LiveLoader(cache_dir,
loader,
ops=[AppendStringOperation()],
output=st)
_test_loader_num_samples(loader2)
def test_mem_out_loader_with_ops(tmpdir):
cache_dir = _prepare_dir(tmpdir)
st = SourceTest({
'samples-dir': str(tmpdir),
'test-split': 2,
'val-split': 2
})
loader = MemoryCachedLoader(cache_dir,
st,
ops=[AppendStringOperation()],
output=st)
_test_loader_read_samples_transformed(loader)
def test_mem_out_loader_with_multiplier_ops_between(tmpdir):
cache_dir = _prepare_dir(tmpdir)
st = SourceTest({
'samples-dir': str(tmpdir),
'test-split': 2,
'val-split': 2
})
loader = MemoryCachedLoader(
cache_dir,
st,
ops=[AugmentOperation(variants=2),
AppendStringOperation()],
output=st)
_test_loader_read_samples_x2_between(loader)
def test_mem_loader_num_samples(tmpdir):
cache_dir = _prepare_dir(tmpdir)
st = SourceTest({'samples-dir': str(tmpdir)})
loader = MemoryCachedLoader(cache_dir, st)
_test_loader_num_samples(loader)