def test_to_binary(tmpdir, eng):
a = [arange(8, dtype='int16').reshape((4, 2)), arange(8, 16, dtype='int16').reshape((4, 2))]
fromlist(a, engine=eng).tobinary(os.path.join(str(tmpdir), 'binary'), prefix='image')
files = [os.path.basename(f) for f in glob.glob(str(tmpdir) + '/binary/image*')]
assert sorted(files) == ['image-00000.bin', 'image-00001.bin']
with open(str(tmpdir) + '/binary/conf.json', 'r') as f:
conf = json.load(f)
assert conf['shape'] == [4, 2]
assert conf['dtype'] == 'int16'
def test_nanmax(eng):
original = arange(24).reshape((2, 3, 4)).astype(float64)
data = fromlist(list(original), engine=eng)
assert allclose(data.nanmax().shape, (1, 3, 4))
assert allclose(data.nanmax().toarray(), original.max(axis=0))
original[0, 2, 3] = nan
original[1, 0, 2] = nan
original[1, 2, 2] = nan
data = fromlist(list(original), engine=eng)
assert allclose(data.nanmax().shape, (1, 3, 4))
assert allclose(data.nanmax().toarray(), nanmax(original, axis=0))
def test_map(eng):
a = arange(8).reshape((4, 2))
data = fromlist([a, a], engine=eng)
map1 = data.toblocks((4, 2)).map(lambda x: 1.0 * x, dtype=float64).toimages()
map2 = data.toblocks((4, 2)).map(lambda x: 1.0 * x).toimages()
assert map1.dtype == float64
assert map2.dtype == float64
def test_median_filter_3d_empty(eng):
data = fromlist([arange(24).reshape((2, 3, 4))], engine=eng)
test1 = data.median_filter([2, 2, 0])[:, :, :, 0].squeeze().toarray()
test2 = data[:, :, :, 0].squeeze().median_filter([2, 2]).toarray()
assert test1.shape == (2, 3)
assert test2.shape == (2, 3)
assert allclose(test1, test2)
def test_count(eng):
a = arange(8).reshape((2, 4))
data = fromlist([a], engine=eng)
assert data.toblocks((1, 1)).count() == 8
assert data.toblocks((1, 2)).count() == 4
assert data.toblocks((2, 2)).count() == 2
assert data.toblocks((2, 4)).count() == 1
def test_squeeze(eng):
data = fromlist([array([[1, 5], [1, 5]]), array([[1, 10], [1, 10]])], engine=eng)
assert data.shape == (2, 2, 2)
assert data[:, :, 0].shape == (2, 2, 1)
assert data[:, 0, 0].shape == (2, 1, 1)
assert data[:, :, 0].squeeze().shape == (2, 2)
assert data[:, 0, 0].squeeze().shape == (2, 1)
def test_series_roundtrip_simple(eng):
if eng is None:
return
a = arange(8).reshape((4, 2))
data = fromlist([a, a], engine=eng)
vals = data.toseries().toimages()
assert allclose(vals.toarray(), data.toarray())
def test_conversion_series_3d(eng):
if eng is None:
return
a = arange(24).reshape((2, 3, 4))
data = fromlist([a], engine=eng)
vals = data.toblocks((2, 3, 4)).toseries().toarray()
assert allclose(vals, a)
def test_local_mode(eng):
a = arange(64).reshape((8, 8))
data = fromlist([a, a])
if data.mode == 'local':
blocks = data.toblocks((4, 4))
assert allclose(blocks.values, data.values)
assert blocks.count() == 1
assert blocks.blockshape == (2, 8, 8)
def test_shape(eng):
if eng is None:
return
data = fromlist([ones((30, 30)) for _ in range(0, 3)], engine=eng)
blocks = data.toblocks((10, 10))
values = [v for k, v in blocks.tordd().collect()]
assert blocks.blockshape == (3, 10, 10)
assert all([v.shape == (3, 10, 10) for v in values])
def test_conversion(eng):
if eng is None:
return
a = arange(8).reshape((4, 2))
data = fromlist([a, a], engine=eng)
vals = data.toblocks((2, 2)).tordd().sortByKey().values().collect()
truth = [array([a[0:2, 0:2], a[0:2, 0:2]]), array([a[2:4, 0:2], a[2:4, 0:2]])]
assert allclose(vals, truth)
def test_map_generic(eng):
a = arange(3*4).reshape((3, 4))
data = fromlist([a, a], engine=eng)
b = asarray(data.toblocks((2, 2)).map_generic(lambda x: [0, 1]))
assert b.shape == (2, 2)
assert b.dtype == object
truth = [v == [0, 1] for v in b.flatten()]
assert all(truth)
def test_labels(eng):
x = arange(10).reshape(10, 1, 1)
data = fromlist(x, labels=range(10), engine=eng)
assert allclose(data.filter(lambda x: x[0, 0]%2==0).labels, array([0, 2, 4, 6, 8]))
assert allclose(data[4:6].labels, array([4, 5]))
assert allclose(data[5].labels, array([5]))
assert allclose(data[[0, 3, 8]].labels, array([0, 3, 8]))
def test_full(eng):
if eng is None:
return
a = arange(8).reshape((4, 2))
data = fromlist([a, a], engine=eng)
vals = data.toblocks((4, 2)).tordd().values().collect()
truth = [a, a]
assert allclose(vals, truth)
def test_blocksize(eng):
a = arange(100*100, dtype='int16').reshape((100, 100))
data = fromlist(10*[a], engine=eng)
blocks = data.toblocks((5, 5))
assert blocks.blockshape == (10, 5, 5)
blocks = data.toblocks('1')
assert blocks.blockshape == (10, 5, 100)
def test_padding(eng):
a = arange(30).reshape((5, 6))
data = fromlist([a, a], engine=eng)
blocks = data.toblocks((2, 3), padding=(1, 1))
vals = blocks.collect_blocks()
shapes = list(map(lambda x: x.shape, vals))
truth = [(2, 3, 4), (2, 3, 4), (2, 4, 4), (2, 4, 4), (2, 2, 4), (2, 2, 4)]
assert allclose(array(shapes), array(truth))
truth = data.toarray()
assert allclose(data.toblocks((2, 3), padding=1).toarray(), truth)
assert allclose(data.toblocks((2, 3), padding=(0, 1)).toarray(), truth)
assert allclose(data.toblocks((2, 3), padding=(1, 1)).toarray(), truth)
def test_map_as_series(eng):
original = arange(4*4).reshape(4, 4)
data = fromlist(5*[original], engine=eng)
# function does not change size of series
def f(x):
return x - mean(x)
result = apply_along_axis(f, 0, data.toarray())
size = (2, 2)
assert allclose(data.map_as_series(f, block_size=size).toarray(), result)
assert allclose(data.map_as_series(f, block_size=size, value_size=5).toarray(), result)
# function does change size of series
def f(x):
return x[:-1]
result = apply_along_axis(f, 0, data.toarray())
assert allclose(data.map_as_series(f, block_size=size).toarray(), result)
assert allclose(data.map_as_series(f, block_size=size, value_size=4).toarray(), result)
def test_subtract(eng):
original = arange(24).reshape((4, 6))
data = fromlist([original], engine=eng)
assert allclose(data.subtract(1).toarray(), original - 1)
sub = arange(24).reshape((4, 6))
assert allclose(data.subtract(sub).toarray(), original - sub)
def test_std(eng):
original = arange(24).reshape((2, 3, 4))
data = fromlist(list(original), engine=eng)
assert allclose(data.std().shape, (1, 3, 4))
assert allclose(data.std().toarray(), original.std(axis=0))
def test_filter(eng):
data = fromlist([arange(6).reshape((2, 3)), arange(6).reshape((2, 3)) * 2], engine=eng)
assert allclose(data.filter(lambda x: x.sum() > 21).toarray(), [[0, 2, 4], [6, 8, 10]])
def test_subsample(eng):
data = fromlist([arange(24).reshape((4, 6))], engine=eng)
vals = data.subsample(2).toarray()
truth = [[0, 2, 4], [12, 14, 16]]
assert allclose(vals, truth)
def test_map_singleton(eng):
data = fromlist([arange(6).reshape((2, 3)), arange(6).reshape((2, 3))], engine=eng)
mapped = data.map(lambda x: x.mean())
assert mapped.shape == (2, 1)
def test_uniform_filter_3d(eng):
data = fromlist([arange(24).reshape((2, 3, 4))], engine=eng)
assert data.uniform_filter(2).toarray().shape == (2, 3, 4)
assert data.uniform_filter([2, 2, 2]).toarray().shape == (2, 3, 4)
assert data.uniform_filter([2, 2, 0]).toarray().shape == (2, 3, 4)
assert allclose(data.uniform_filter(2).toarray(), data.uniform_filter([2, 2, 2]).toarray())
def test_subtract(eng):
original = arange(24).reshape((4, 6))
data = fromlist([original], engine=eng)
assert allclose(data.subtract(1).toarray(), original - 1)
sub = arange(24).reshape((4, 6))
assert allclose(data.subtract(sub).toarray(), original - sub)
def test_toseries(eng):
data = fromlist([arange(6).reshape((2, 3))], engine=eng)
truth = [[0, 1, 2], [3, 4, 5]]
assert isinstance(data.toseries(), Series)
assert allclose(data.toseries().toarray(), truth)
def test_map_singleton(eng):
data = fromlist([arange(6).reshape(
(2, 3)), arange(6).reshape((2, 3))],
engine=eng)
mapped = data.map(lambda x: x.mean())
assert mapped.shape == (2, 1)
def test_sample(eng):
data = fromlist([array([[1, 5], [1, 5]]), array([[1, 10], [1, 10]])], engine=eng)
assert allclose(data.sample(2).shape, (2, 2, 2))
assert allclose(data.sample(1).shape, (1, 2, 2))
assert allclose(data.filter(lambda x: x.max() > 5).sample(1).toarray(), [[1, 10], [1, 10]])
def test_first(eng):
data = fromlist([array([[1, 5], [1, 5]]), array([[1, 10], [1, 10]])], engine=eng)
assert allclose(data.first(), [[1, 5], [1, 5]])
def test_labels_setting(eng):
x = arange(10).reshape(10, 1, 1)
data = fromlist(x, engine=eng)
with pytest.raises(ValueError):
data.labels = range(8)
def test_toseries_pack_3d(eng):
original = arange(24).reshape((2, 3, 4))
data = fromlist([original], engine=eng)
assert allclose(data.toseries().toarray(), original)
def test_toseries_pack_3d(eng):
original = arange(24).reshape((2, 3, 4))
data = fromlist([original], engine=eng)
assert allclose(data.toseries().toarray(), original)
def test_gaussian_filter_2d(eng):
data = fromlist([arange(24).reshape((4, 6))], engine=eng)
assert data.gaussian_filter(2).toarray().shape == (4, 6)
assert data.gaussian_filter([2, 2]).toarray().shape == (4, 6)
assert allclose(data.gaussian_filter(2).toarray(), data.gaussian_filter([2, 2]).toarray())
def test_toseries_roundtrip(eng):
data = fromlist([arange(6).reshape((2, 3)), arange(6).reshape((2, 3))], engine=eng)
assert isinstance(data.toseries(), Series)
assert isinstance(data.toseries().toimages(), Images)
assert allclose(data.toseries().toimages().toarray(), data.toarray())
def test_var(eng):
original = arange(24).reshape((2, 3, 4))
data = fromlist(list(original), engine=eng)
assert allclose(data.var().shape, (1, 3, 4))
assert allclose(data.var().toarray(), original.var(axis=0))
def test_subsample(eng):
data = fromlist([arange(24).reshape((4, 6))], engine=eng)
vals = data.subsample(2).toarray()
truth = [[0, 2, 4], [12, 14, 16]]
assert allclose(vals, truth)
def test_map(eng):
data = fromlist([arange(6).reshape((2, 3))], engine=eng)
assert allclose(
data.map(lambda x: x + 1).toarray(), [[1, 2, 3], [4, 5, 6]])