Beispiel #1
0
def _preview(data, output_dir, split, num_samples):
    if not data.output.supports_preview():
        return False

    data.output.begin_preview(output_dir)

    loader = LiveLoader(data.cache_dir, data.input, data.ops, output=data.output)
    loader.transform = False
    loader.begin_read_samples()
    total = 0

    for split_ in SPLITS:
        if split not in ('all', split_):
            continue

        mul = int(loader.multipliers[split_])

        for index in range(loader.num_samples(split_)):
            if num_samples is not None and total >= num_samples:
                break

            samples = loader.read_samples(split_, index, mul)

            for sample in samples:
                data.output.write_preview(output_dir, split_, sample)

            total += 1


    loader.end_read_samples()
    data.output.end_preview(output_dir)

    return True
Beispiel #2
0
    def _get_loader(self, cache_input, cache_output):

        if cache_input in ('disk', 'mem'):

            # When doing input caching, wrap the input object in
            # a cached loader.

            loader_class = FileCachedLoader if cache_input == 'disk' else MemoryCachedLoader
            input_loader = loader_class(self.cache_dir, self.input)
            input_loader.progress_callback = self._progress_callback
        else:

            # otherwise, use the input object directly
            input_loader = self.input

        if cache_output in ('disk', 'mem'):

            # set up output caching

            loader_class = FileCachedLoader if cache_output == 'disk' else MemoryCachedLoader
            loader = loader_class(self.cache_dir, input_loader, self.ops,
                                  self.output)
            loader.progress_callback = self._progress_callback

            return loader

        return LiveLoader(self.cache_dir, input_loader, self.ops, self.output)
Beispiel #3
0
def test_live_loader_with_ops_meta(tmpdir):
    cache_dir = _prepare_dir(tmpdir)
    st = SourceTest({'samples-dir': str(tmpdir)})
    loader = LiveLoader(cache_dir,
                        st,
                        ops=[AppendStringOperation()],
                        output=st)
    _test_loader_meta(loader)
Beispiel #4
0
def test_disk_loader_ops_num_samples(tmpdir):
    cache_dir = _prepare_dir(tmpdir)
    st = SourceTest({'samples-dir': str(tmpdir)})
    loader = FileCachedLoader(cache_dir, st)
    loader2 = LiveLoader(cache_dir,
                         loader,
                         ops=[AppendStringOperation()],
                         output=st)
    _test_loader_num_samples(loader2)
Beispiel #5
0
def test_live_loader_with_rng_no_reset(tmpdir):
    cache_dir = _prepare_dir(tmpdir)
    st = SourceTest({
        'samples-dir': str(tmpdir),
        'test-split': 2,
        'val-split': 2
    })
    loader = LiveLoader(cache_dir, st)
    _test_loader_with_rng_no_reset(loader)
Beispiel #6
0
def test_listview_random2():
    loader = LiveLoader('.cache', SourceTest())
    listview = ListView(loader, 'train', randomize=True, fetch_size=1)
    listview2 = ListView(loader,
                         'train',
                         randomize=True,
                         random_seed=2601,
                         fetch_size=1)
    assert list(map(lambda tp: tp[0], listview2[:10])) \
        != list(map(lambda tp: tp[0], listview[:10]))
Beispiel #7
0
def test_iterview_random2():
    loader = LiveLoader('.cache', SourceTest())
    iterview = IteratorView(loader, 'train', randomize=True, fetch_size=1)
    iterview2 = IteratorView(loader,
                             'train',
                             randomize=True,
                             random_seed=2601,
                             fetch_size=1)
    assert list(map(lambda tp: tp[0], itertools.islice(iterview2, 10))) \
        != list(map(lambda tp: tp[0], itertools.islice(iterview, 10)))
Beispiel #8
0
def test_live_loader_with_ops(tmpdir):
    cache_dir = _prepare_dir(tmpdir)
    st = SourceTest({
        'samples-dir': str(tmpdir),
        'test-split': 2,
        'val-split': 2
    })
    loader = LiveLoader(cache_dir,
                        st,
                        ops=[AppendStringOperation()],
                        output=st)
    _test_loader_read_samples_transformed(loader)
Beispiel #9
0
    def _get_loader(self, cache_input, cache_output):
        if cache_input == 'disk':
            input_loader = FileCachedLoader(self.cache_dir, self.input, progress_callback=self._progress_callback)
        elif cache_input == 'mem':
            input_loader = MemoryCachedLoader(self.cache_dir, self.input, progress_callback=self._progress_callback)
        else:
            input_loader = self.input

        if cache_output == 'disk':
            return FileCachedLoader(self.cache_dir, input_loader, self.ops, self.output, progress_callback=self._progress_callback)
        elif cache_output == 'mem':
            return MemoryCachedLoader(self.cache_dir, input_loader, self.ops, self.output, progress_callback=self._progress_callback)
        else:
            return LiveLoader(self.cache_dir, input_loader, self.ops, self.output)
Beispiel #10
0
def test_live_loader_with_multiplier_ops_between(tmpdir):
    cache_dir = _prepare_dir(tmpdir)
    st = SourceTest({
        'samples-dir': str(tmpdir),
        'test-split': 2,
        'val-split': 2
    })
    loader = LiveLoader(
        cache_dir,
        st,
        ops=[AugmentOperation(variants=2),
             AppendStringOperation()],
        output=st)
    _test_loader_read_samples_x2_between(loader)
Beispiel #11
0
def test_mem_loader_with_multiplier_ops(tmpdir):
    cache_dir = _prepare_dir(tmpdir)
    st = SourceTest({
        'samples-dir': str(tmpdir),
        'test-split': 2,
        'val-split': 2
    })
    loader = MemoryCachedLoader(cache_dir, st)
    loader2 = LiveLoader(
        cache_dir,
        loader,
        ops=[AugmentOperation(variants=2),
             AppendStringOperation()],
        output=st)
    _test_loader_read_samples_x2(loader2)
Beispiel #12
0
def test_listview_neg():
    loader = LiveLoader('.cache', SourceTest())
    listview = ListView(loader, 'train')
    assert listview[-1][0] == 99
Beispiel #13
0
def test_listview_slice4():
    loader = LiveLoader('.cache', SourceTest())
    listview = ListView(loader, 'val')
    assert list(map(lambda tp: tp[0], listview[1:3])) == [1, 2]
Beispiel #14
0
def test_listview_max_samples():
    loader = LiveLoader('.cache', SourceTest())
    listview = ListView(loader, 'train', max_samples=10)
    assert list(map(lambda tp: tp[0], listview)) == list(range(10))
Beispiel #15
0
def test_listview_random():
    loader = LiveLoader('.cache', SourceTest())
    listview = ListView(loader, 'train', randomize=True, fetch_size=1)
    assert list(map(lambda tp: tp[0],
                    listview[:10])) == [92, 1, 43, 61, 35, 73, 48, 18, 98, 36]
Beispiel #16
0
def test_listview_transform_y():
    loader = LiveLoader('.cache', SourceTest())
    listview = ListView(loader, 'train', transform_y=lambda _: 'transformed_y')
    assert listview[0][1] == 'transformed_y'
Beispiel #17
0
def test_live_loader_num_samples(tmpdir):
    cache_dir = _prepare_dir(tmpdir)
    st = SourceTest({'samples-dir': str(tmpdir)})
    loader = LiveLoader(cache_dir, st)
    _test_loader_num_samples(loader)
Beispiel #18
0
def test_iterview_meta():
    loader = LiveLoader('.cache', SourceTest())
    iterview = IteratorView(loader, 'train', with_meta=True)
    assert next(iterview) == (0, 5, dict(meta=0))
    assert next(iterview) == (1, 6, dict(meta=1))
    assert next(iterview) == (2, 7, dict(meta=2))
Beispiel #19
0
def test_iterview_transform():
    loader = LiveLoader('.cache', SourceTest())
    iterview = IteratorView(loader, 'train', transform_x=lambda x: x + 10)
    assert list(map(lambda tp: tp[0], iterview)) == list(range(10, 110))
Beispiel #20
0
def test_listview_random_fetch_size():
    loader = LiveLoader('.cache', SourceTest())
    listview = ListView(loader, 'train', randomize=True, fetch_size=10)
    assert list(map(lambda tp: tp[0], listview[:10])) == list(range(70, 80))
Beispiel #21
0
def test_iterview_random():
    loader = LiveLoader('.cache', SourceTest())
    iterview = IteratorView(loader, 'train', randomize=True, fetch_size=1)
    assert list(map(lambda tp: tp[0], itertools.islice(iterview, 10))) \
        == [92, 1, 43, 61, 35, 73, 48, 18, 98, 36]
Beispiel #22
0
def test_iterview_infinite():
    loader = LiveLoader('.cache', SourceTest())
    iterview = IteratorView(loader, 'train', infinite=True)
    assert list(map(lambda tp: tp[0], itertools.islice(iterview, 150))) \
        == list(range(100)) + list(range(50))
Beispiel #23
0
def test_listview_val():
    loader = LiveLoader('.cache', SourceTest())
    listview = ListView(loader, 'val')
    assert list(map(lambda tp: tp[0], listview)) == list(range(10))
Beispiel #24
0
def test_iterview_transform_y():
    loader = LiveLoader('.cache', SourceTest())
    iterview = IteratorView(loader,
                            'train',
                            transform_y=lambda _: 'transformed_y')
    assert next(iterview)[1] == 'transformed_y'
Beispiel #25
0
def test_iterview_max_samples():
    loader = LiveLoader('.cache', SourceTest())
    iterview = IteratorView(loader, 'train', max_samples=10)
    assert list(map(lambda tp: tp[0], iterview)) == list(range(10))
Beispiel #26
0
def test_iterview_test():
    loader = LiveLoader('.cache', SourceTest())
    iterview = IteratorView(loader, 'test')
    assert list(map(lambda tp: tp[0], iterview)) == list(range(20))
Beispiel #27
0
def test_iterview_random_fetch_size():
    loader = LiveLoader('.cache', SourceTest())
    iterview = IteratorView(loader, 'train', randomize=True, fetch_size=10)
    assert list(map(lambda tp: tp[0], itertools.islice(iterview, 10))) \
        == list(range(70, 80))
Beispiel #28
0
def test_listview_meta():
    loader = LiveLoader('.cache', SourceTest())
    listview = ListView(loader, 'train', with_meta=True)
    assert listview[0] == (0, 5, dict(meta=0))
    assert listview[1] == (1, 6, dict(meta=1))
    assert listview[2] == (2, 7, dict(meta=2))