def test_to_binary(tmpdir, eng): a = arange(8, dtype='int16').reshape((4, 2)) p = str(tmpdir) + '/data' fromarray(a, npartitions=1, engine=eng).tobinary(p) files = [os.path.basename(f) for f in glob.glob(str(tmpdir) + '/data/*')] assert sorted(files) == ['SUCCESS', 'conf.json', 'series-00000.bin'] with open(str(tmpdir) + '/data/conf.json', 'r') as f: conf = json.load(f) assert conf['shape'] == [4, 2] assert conf['dtype'] == 'int16'
def test_nanmin(eng): arr = array([arange(8), arange(8)]).astype(float64) data = fromarray(arr, engine=eng) val = data.nanmin().toarray() expected = nanmin(data.toarray(), axis=0) assert allclose(val, expected) assert str(val.dtype) == 'float64' arr[0, 4] = nan arr[1, 3] = nan arr[1, 4] = nan data = fromarray(arr, engine=eng) val = data.nanmin().toarray() expected = nanmin(data.toarray(), axis=0) assert allclose(val, expected, equal_nan=True) assert str(val.dtype) == 'float64'
def test_from_array_vector(eng): a = arange(8, dtype='int16').reshape((4, 2)) data = fromarray(a, engine=eng) assert data.shape == (4, 2) assert data.dtype == 'int16' assert allclose(data.index, [0, 1]) assert allclose(data.toarray(), a)
def test_to_binary_roundtrip_3d(tmpdir, eng): a = arange(16, dtype='int16').reshape((4, 2, 2)) p = str(tmpdir) + '/data' data = fromarray(a, npartitions=1, engine=eng) data.tobinary(p) loaded = frombinary(p, engine=eng) assert allclose(data.toarray(), loaded.toarray())
def tolocal(self): """ Convert to local mode. """ from thunder.series.readers import fromarray if self.mode == 'local': logging.getLogger('thunder').warn('images already in local mode') pass return fromarray(self.toarray(), index=self.index, labels=self.labels)
def test_from_array_bolt(eng): a = arange(8, dtype='int16').reshape((4, 2)) if eng is not None: b = barray(a, context=eng) else: b = barray(a) data = fromarray(b, engine=eng) assert data.shape == (4, 2) assert data.dtype == 'int16' assert allclose(data.index, [0, 1]) assert allclose(data.toarray(), a)
def tospark(self, engine=None): """ Convert to spark mode. """ from thunder.series.readers import fromarray if self.mode == 'spark': logging.getLogger('thunder').warn('images already in local mode') pass if engine is None: raise ValueError('Must provide SparkContext') return fromarray(self.toarray(), index=self.index, labels=self.labels, engine=engine)
def test_reshape(eng): original = fromarray(arange(72).reshape(6, 6, 2), engine=eng) arr = original.toarray() assert allclose(arr.reshape(12, 3, 2), original.reshape(12, 3, 2).toarray()) assert allclose(arr.reshape(36, 2), original.reshape(36, 2).toarray()) assert allclose(arr.reshape(4, 3, 3, 2), original.reshape(4, 3, 3, 2).toarray()) # must conserve number of elements with pytest.raises(ValueError): original.reshape(6, 3, 2) # cannot change length of series with pytest.raises(ValueError): original.reshape(6, 3, 4)
def test_from_array_index(eng): a = arange(8, dtype='int16').reshape((4, 2)) data = fromarray(a, index=[2, 3], engine=eng) assert allclose(data.index, [2, 3])
def test_flatten(eng): arr = arange(2 * 2 * 5).reshape(2, 2, 5) data = fromarray(arr, engine=eng) assert data.flatten().shape == (4, 5) assert allclose(data.flatten().toarray(), arr.reshape(2 * 2, 5))
def test_flatten(eng): arr = arange(2*2*5).reshape(2, 2, 5) data = fromarray(arr, engine=eng) assert data.flatten().shape == (4, 5) assert allclose(data.flatten().toarray(), arr.reshape(2*2, 5))