def test_to_binary(tmpdir, eng): a = [arange(8, dtype='int16').reshape((4, 2)), arange(8, 16, dtype='int16').reshape((4, 2))] fromlist(a, engine=eng).tobinary(os.path.join(str(tmpdir), 'binary'), prefix='image') files = [os.path.basename(f) for f in glob.glob(str(tmpdir) + '/binary/image*')] assert sorted(files) == ['image-00000.bin', 'image-00001.bin'] with open(str(tmpdir) + '/binary/conf.json', 'r') as f: conf = json.load(f) assert conf['shape'] == [4, 2] assert conf['dtype'] == 'int16'
def test_nanmax(eng): original = arange(24).reshape((2, 3, 4)).astype(float64) data = fromlist(list(original), engine=eng) assert allclose(data.nanmax().shape, (1, 3, 4)) assert allclose(data.nanmax().toarray(), original.max(axis=0)) original[0, 2, 3] = nan original[1, 0, 2] = nan original[1, 2, 2] = nan data = fromlist(list(original), engine=eng) assert allclose(data.nanmax().shape, (1, 3, 4)) assert allclose(data.nanmax().toarray(), nanmax(original, axis=0))
def test_map(eng): a = arange(8).reshape((4, 2)) data = fromlist([a, a], engine=eng) map1 = data.toblocks((4, 2)).map(lambda x: 1.0 * x, dtype=float64).toimages() map2 = data.toblocks((4, 2)).map(lambda x: 1.0 * x).toimages() assert map1.dtype == float64 assert map2.dtype == float64
def test_median_filter_3d_empty(eng): data = fromlist([arange(24).reshape((2, 3, 4))], engine=eng) test1 = data.median_filter([2, 2, 0])[:, :, :, 0].squeeze().toarray() test2 = data[:, :, :, 0].squeeze().median_filter([2, 2]).toarray() assert test1.shape == (2, 3) assert test2.shape == (2, 3) assert allclose(test1, test2)
def test_count(eng): a = arange(8).reshape((2, 4)) data = fromlist([a], engine=eng) assert data.toblocks((1, 1)).count() == 8 assert data.toblocks((1, 2)).count() == 4 assert data.toblocks((2, 2)).count() == 2 assert data.toblocks((2, 4)).count() == 1
def test_squeeze(eng): data = fromlist([array([[1, 5], [1, 5]]), array([[1, 10], [1, 10]])], engine=eng) assert data.shape == (2, 2, 2) assert data[:, :, 0].shape == (2, 2, 1) assert data[:, 0, 0].shape == (2, 1, 1) assert data[:, :, 0].squeeze().shape == (2, 2) assert data[:, 0, 0].squeeze().shape == (2, 1)
def test_series_roundtrip_simple(eng): if eng is None: return a = arange(8).reshape((4, 2)) data = fromlist([a, a], engine=eng) vals = data.toseries().toimages() assert allclose(vals.toarray(), data.toarray())
def test_conversion_series_3d(eng): if eng is None: return a = arange(24).reshape((2, 3, 4)) data = fromlist([a], engine=eng) vals = data.toblocks((2, 3, 4)).toseries().toarray() assert allclose(vals, a)
def test_local_mode(eng): a = arange(64).reshape((8, 8)) data = fromlist([a, a]) if data.mode == 'local': blocks = data.toblocks((4, 4)) assert allclose(blocks.values, data.values) assert blocks.count() == 1 assert blocks.blockshape == (2, 8, 8)
def test_shape(eng): if eng is None: return data = fromlist([ones((30, 30)) for _ in range(0, 3)], engine=eng) blocks = data.toblocks((10, 10)) values = [v for k, v in blocks.tordd().collect()] assert blocks.blockshape == (3, 10, 10) assert all([v.shape == (3, 10, 10) for v in values])
def test_conversion(eng): if eng is None: return a = arange(8).reshape((4, 2)) data = fromlist([a, a], engine=eng) vals = data.toblocks((2, 2)).tordd().sortByKey().values().collect() truth = [array([a[0:2, 0:2], a[0:2, 0:2]]), array([a[2:4, 0:2], a[2:4, 0:2]])] assert allclose(vals, truth)
def test_map_generic(eng): a = arange(3*4).reshape((3, 4)) data = fromlist([a, a], engine=eng) b = asarray(data.toblocks((2, 2)).map_generic(lambda x: [0, 1])) assert b.shape == (2, 2) assert b.dtype == object truth = [v == [0, 1] for v in b.flatten()] assert all(truth)
def test_labels(eng): x = arange(10).reshape(10, 1, 1) data = fromlist(x, labels=range(10), engine=eng) assert allclose(data.filter(lambda x: x[0, 0]%2==0).labels, array([0, 2, 4, 6, 8])) assert allclose(data[4:6].labels, array([4, 5])) assert allclose(data[5].labels, array([5])) assert allclose(data[[0, 3, 8]].labels, array([0, 3, 8]))
def test_full(eng): if eng is None: return a = arange(8).reshape((4, 2)) data = fromlist([a, a], engine=eng) vals = data.toblocks((4, 2)).tordd().values().collect() truth = [a, a] assert allclose(vals, truth)
def test_blocksize(eng): a = arange(100*100, dtype='int16').reshape((100, 100)) data = fromlist(10*[a], engine=eng) blocks = data.toblocks((5, 5)) assert blocks.blockshape == (10, 5, 5) blocks = data.toblocks('1') assert blocks.blockshape == (10, 5, 100)
def test_padding(eng): a = arange(30).reshape((5, 6)) data = fromlist([a, a], engine=eng) blocks = data.toblocks((2, 3), padding=(1, 1)) vals = blocks.collect_blocks() shapes = list(map(lambda x: x.shape, vals)) truth = [(2, 3, 4), (2, 3, 4), (2, 4, 4), (2, 4, 4), (2, 2, 4), (2, 2, 4)] assert allclose(array(shapes), array(truth)) truth = data.toarray() assert allclose(data.toblocks((2, 3), padding=1).toarray(), truth) assert allclose(data.toblocks((2, 3), padding=(0, 1)).toarray(), truth) assert allclose(data.toblocks((2, 3), padding=(1, 1)).toarray(), truth)
def test_map_as_series(eng): original = arange(4*4).reshape(4, 4) data = fromlist(5*[original], engine=eng) # function does not change size of series def f(x): return x - mean(x) result = apply_along_axis(f, 0, data.toarray()) size = (2, 2) assert allclose(data.map_as_series(f, block_size=size).toarray(), result) assert allclose(data.map_as_series(f, block_size=size, value_size=5).toarray(), result) # function does change size of series def f(x): return x[:-1] result = apply_along_axis(f, 0, data.toarray()) assert allclose(data.map_as_series(f, block_size=size).toarray(), result) assert allclose(data.map_as_series(f, block_size=size, value_size=4).toarray(), result)
def test_subtract(eng): original = arange(24).reshape((4, 6)) data = fromlist([original], engine=eng) assert allclose(data.subtract(1).toarray(), original - 1) sub = arange(24).reshape((4, 6)) assert allclose(data.subtract(sub).toarray(), original - sub)
def test_std(eng): original = arange(24).reshape((2, 3, 4)) data = fromlist(list(original), engine=eng) assert allclose(data.std().shape, (1, 3, 4)) assert allclose(data.std().toarray(), original.std(axis=0))
def test_filter(eng): data = fromlist([arange(6).reshape((2, 3)), arange(6).reshape((2, 3)) * 2], engine=eng) assert allclose(data.filter(lambda x: x.sum() > 21).toarray(), [[0, 2, 4], [6, 8, 10]])
def test_subsample(eng): data = fromlist([arange(24).reshape((4, 6))], engine=eng) vals = data.subsample(2).toarray() truth = [[0, 2, 4], [12, 14, 16]] assert allclose(vals, truth)
def test_map_singleton(eng): data = fromlist([arange(6).reshape((2, 3)), arange(6).reshape((2, 3))], engine=eng) mapped = data.map(lambda x: x.mean()) assert mapped.shape == (2, 1)
def test_uniform_filter_3d(eng): data = fromlist([arange(24).reshape((2, 3, 4))], engine=eng) assert data.uniform_filter(2).toarray().shape == (2, 3, 4) assert data.uniform_filter([2, 2, 2]).toarray().shape == (2, 3, 4) assert data.uniform_filter([2, 2, 0]).toarray().shape == (2, 3, 4) assert allclose(data.uniform_filter(2).toarray(), data.uniform_filter([2, 2, 2]).toarray())
def test_toseries(eng): data = fromlist([arange(6).reshape((2, 3))], engine=eng) truth = [[0, 1, 2], [3, 4, 5]] assert isinstance(data.toseries(), Series) assert allclose(data.toseries().toarray(), truth)
def test_map_singleton(eng): data = fromlist([arange(6).reshape( (2, 3)), arange(6).reshape((2, 3))], engine=eng) mapped = data.map(lambda x: x.mean()) assert mapped.shape == (2, 1)
def test_sample(eng): data = fromlist([array([[1, 5], [1, 5]]), array([[1, 10], [1, 10]])], engine=eng) assert allclose(data.sample(2).shape, (2, 2, 2)) assert allclose(data.sample(1).shape, (1, 2, 2)) assert allclose(data.filter(lambda x: x.max() > 5).sample(1).toarray(), [[1, 10], [1, 10]])
def test_first(eng): data = fromlist([array([[1, 5], [1, 5]]), array([[1, 10], [1, 10]])], engine=eng) assert allclose(data.first(), [[1, 5], [1, 5]])
def test_labels_setting(eng): x = arange(10).reshape(10, 1, 1) data = fromlist(x, engine=eng) with pytest.raises(ValueError): data.labels = range(8)
def test_toseries_pack_3d(eng): original = arange(24).reshape((2, 3, 4)) data = fromlist([original], engine=eng) assert allclose(data.toseries().toarray(), original)
def test_gaussian_filter_2d(eng): data = fromlist([arange(24).reshape((4, 6))], engine=eng) assert data.gaussian_filter(2).toarray().shape == (4, 6) assert data.gaussian_filter([2, 2]).toarray().shape == (4, 6) assert allclose(data.gaussian_filter(2).toarray(), data.gaussian_filter([2, 2]).toarray())
def test_toseries_roundtrip(eng): data = fromlist([arange(6).reshape((2, 3)), arange(6).reshape((2, 3))], engine=eng) assert isinstance(data.toseries(), Series) assert isinstance(data.toseries().toimages(), Images) assert allclose(data.toseries().toimages().toarray(), data.toarray())
def test_var(eng): original = arange(24).reshape((2, 3, 4)) data = fromlist(list(original), engine=eng) assert allclose(data.var().shape, (1, 3, 4)) assert allclose(data.var().toarray(), original.var(axis=0))
def test_map(eng): data = fromlist([arange(6).reshape((2, 3))], engine=eng) assert allclose( data.map(lambda x: x + 1).toarray(), [[1, 2, 3], [4, 5, 6]])