def testBroadcastTo(self): arr = ones((10, 5), chunk_size=2) arr2 = broadcast_to(arr, (20, 10, 5)) arr2.tiles() self.assertEqual(arr2.shape, (20, 10, 5)) self.assertEqual(calc_shape(arr2), arr2.shape) self.assertEqual(len(arr2.chunks), len(arr.chunks)) self.assertEqual(arr2.chunks[0].shape, (20, 2, 2)) self.assertEqual(calc_shape(arr2.chunks[-1]), arr2.chunks[-1].shape) arr = ones((10, 5, 1), chunk_size=2) arr3 = broadcast_to(arr, (5, 10, 5, 6)) arr3.tiles() self.assertEqual(arr3.shape, (5, 10, 5, 6)) self.assertEqual(calc_shape(arr3), arr3.shape) self.assertEqual(len(arr3.chunks), len(arr.chunks)) self.assertEqual(arr3.nsplits, ((5, ), (2, 2, 2, 2, 2), (2, 2, 1), (6, ))) self.assertEqual(arr3.chunks[0].shape, (5, 2, 2, 6)) self.assertEqual(calc_shape(arr3.chunks[-1]), arr3.chunks[-1].shape) arr = ones((10, 1), chunk_size=2) arr4 = broadcast_to(arr, (20, 10, 5)) arr4.tiles() self.assertEqual(arr4.shape, (20, 10, 5)) self.assertEqual(calc_shape(arr4), arr4.shape) self.assertEqual(len(arr4.chunks), len(arr.chunks)) self.assertEqual(arr4.chunks[0].shape, (20, 2, 5)) self.assertEqual(calc_shape(arr4.chunks[-1]), arr4.chunks[-1].shape) with self.assertRaises(ValueError): broadcast_to(arr, (10, )) with self.assertRaises(ValueError): broadcast_to(arr, (5, 1))
def testBoolIndexingTiles(self): t = ones((100, 200, 300), chunks=30) indexed = t[t < 2] indexed.tiles() self.assertEqual(len(indexed.chunks), 280) self.assertEqual(indexed.chunks[0].index, (0, )) self.assertEqual(indexed.chunks[20].index, (20, )) self.assertIs(indexed.chunks[20].inputs[0], t.cix[(0, 2, 0)].data) self.assertIs(indexed.chunks[20].inputs[1], indexed.op.indexes[0].cix[0, 2, 0].data) t2 = ones((100, 200), chunks=30) indexed2 = t[t2 < 2] indexed2.tiles() self.assertEqual(len(indexed2.chunks), 280) self.assertEqual(len(indexed2.chunks[0].shape), 2) self.assertTrue(np.isnan(indexed2.chunks[0].shape[0])) self.assertEqual(indexed2.chunks[0].shape[1], 30) self.assertEqual(indexed2.chunks[20].inputs[0], t.cix[(0, 2, 0)].data) self.assertEqual(indexed2.chunks[20].inputs[1], indexed2.op.indexes[0].cix[0, 2].data)
def testFancyIndexing(self): t = ones((100, 200, 300)) t2 = t[[0, 1], [2, 3]] self.assertEqual(t2.shape, (2, 300)) t3 = t[[[0, 1], [2, 3]], [4, 5]] self.assertEqual(t3.shape, (2, 2, 300)) with self.assertRaises(IndexError) as cm: _ = t[[1, 2], [3, 4, 5]] e = str(cm.exception) self.assertEqual( e, 'shape mismatch: indexing arrays could not be broadcast ' 'together with shapes (2,) (3,)') t = ones((100, 200, 300), chunks=10) t4 = t[:10, -10:, [13, 244, 151, 242, 34]].tiles() self.assertEqual(t4.shape, (10, 10, 5)) self.assertEqual(t4.chunk_shape, (1, 1, 1)) t5 = t[:10, -10:, [1, 10, 20, 33, 34, 200]].tiles() self.assertEqual(t5.shape, (10, 10, 6)) self.assertEqual(t5.chunk_shape, (1, 1, 5))
def testCumReduction(self): cumsum = lambda x, *args, **kwargs: x.cumsum(*args, **kwargs).tiles() cumprod = lambda x, *args, **kwargs: x.cumprod(*args, **kwargs).tiles() res1 = cumsum(ones((10, 8), chunk_size=3), axis=0) res2 = cumprod(ones((10, 8), chunk_size=3), axis=0) self.assertEqual(res1.shape, (10, 8)) self.assertIsNotNone(res1.dtype) self.assertEqual(calc_shape(res1), res1.shape) self.assertEqual(calc_shape(res1.chunks[0]), res1.chunks[0].shape) self.assertEqual(res2.shape, (10, 8)) self.assertIsNotNone(res2.dtype) self.assertEqual(calc_shape(res2), res2.shape) self.assertEqual(calc_shape(res2.chunks[0]), res2.chunks[0].shape) res1 = cumsum(ones((10, 8, 8), chunk_size=3), axis=1) res2 = cumprod(ones((10, 8, 8), chunk_size=3), axis=1) self.assertEqual(res1.shape, (10, 8, 8)) self.assertEqual(calc_shape(res1), res1.shape) self.assertEqual(calc_shape(res1.chunks[0]), res1.chunks[0].shape) self.assertEqual(res2.shape, (10, 8, 8)) self.assertEqual(calc_shape(res2), res2.shape) self.assertEqual(calc_shape(res2.chunks[0]), res2.chunks[0].shape)
def testAstype(self): arr = ones((10, 20, 30), chunk_size=3) arr2 = arr.astype(np.int32) arr2.tiles() self.assertEqual(arr2.shape, (10, 20, 30)) self.assertEqual(calc_shape(arr2), arr2.shape) self.assertTrue(np.issubdtype(arr2.dtype, np.int32)) self.assertEqual(arr2.op.casting, 'unsafe') self.assertEqual(calc_shape(arr2.chunks[0]), arr2.chunks[0].shape) with self.assertRaises(TypeError): arr.astype(np.int32, casting='safe')
def testBoolIndexingTiles(self): t = ones((100, 200, 300), chunk_size=30) indexed = t[t < 2] indexed.tiles() self.assertEqual(len(indexed.chunks), 280) self.assertEqual(indexed.chunks[0].index, (0, )) self.assertEqual(indexed.chunks[20].index, (20, )) self.assertIs(indexed.chunks[20].inputs[0], t.cix[(0, 2, 0)].data) self.assertIs(indexed.chunks[20].inputs[1], indexed.op.indexes[0].cix[0, 2, 0].data) self.assertEqual(calc_shape(indexed.chunks[0]), indexed.chunks[0].shape) self.assertEqual(calc_shape(indexed.chunks[20]), indexed.chunks[20].shape) self.assertEqual(indexed.chunks[0].rough_nbytes, t.chunks[0].nbytes) self.assertEqual(indexed.chunks[20].rough_nbytes, t.chunks[20].nbytes) t2 = ones((100, 200), chunk_size=30) indexed2 = t[t2 < 2] indexed2.tiles() self.assertEqual(len(indexed2.chunks), 280) self.assertEqual(len(indexed2.chunks[0].shape), 2) self.assertTrue(np.isnan(indexed2.chunks[0].shape[0])) self.assertEqual(indexed2.chunks[0].shape[1], 30) self.assertEqual(indexed2.chunks[20].inputs[0], t.cix[(0, 2, 0)].data) self.assertEqual(indexed2.chunks[20].inputs[1], indexed2.op.indexes[0].cix[0, 2].data) self.assertEqual(indexed2.chunks[0].rough_nbytes, 30 * 30 * 30 * t2.dtype.itemsize) self.assertEqual(calc_shape(indexed2.chunks[0]), indexed2.chunks[0].shape) self.assertEqual(calc_shape(indexed2.chunks[20]), indexed2.chunks[20].shape) self.assertEqual(indexed2.chunks[0].rough_nbytes, t.chunks[0].nbytes) self.assertEqual(indexed2.chunks[20].rough_nbytes, t.chunks[20].nbytes)
def testMixedIndexingTiles(self): t = ones((100, 200, 300, 400), chunk_size=24) cmp = ones(400, chunk_size=24) < 2 t2 = t[10:90:3, 5, ..., None, cmp] t2.tiles() self.assertEqual(t2.shape[:-1], (27, 300, 1)) self.assertTrue(np.isnan(t2.shape[-1])) self.assertEqual(t2.rough_nbytes, 27 * 300 * 400 * t.dtype.itemsize) self.assertEqual(t2.chunk_shape, (4, 13, 1, 17)) self.assertEqual( t2.chunks[0].op.indexes, [slice(10, 24, 3), 5, slice(None), None, cmp.cix[0, ].data]) self.assertEqual(calc_shape(t2.chunks[0]), t2.chunks[0].shape) self.assertEqual(t2.chunks[0].rough_nbytes, 5 * 24 * 24 * t.dtype.itemsize) # multiple tensor type as indexes t3 = ones((100, 200, 300, 400), chunk_size=24) cmp2 = ones((100, 200), chunk_size=24) < 2 cmp3 = ones(400, chunk_size=24) < 2 t4 = t3[cmp2, 5, None, cmp3] t4.tiles() self.assertEqual(t4.shape[1], 1) self.assertTrue(np.isnan(t4.shape[0])) self.assertTrue(np.isnan(t4.shape[-1])) self.assertEqual(calc_shape(t4), t4.shape) self.assertEqual(t4.rough_nbytes, 100 * 200 * 400 * t.dtype.itemsize) self.assertEqual(t4.chunk_shape, (45, 1, 17)) self.assertEqual(t4.chunks[0].op.indexes, [cmp2.cix[0, 0].data, 5, None, cmp3.cix[0, ].data]) self.assertEqual(calc_shape(t4.chunks[0]), t4.chunks[0].shape) self.assertEqual(t4.chunks[0].rough_nbytes, 24 * 24 * 24 * t3.dtype.itemsize)
def testTranspose(self): arr = ones((10, 20, 30), chunk_size=[4, 3, 5]) arr2 = transpose(arr) arr2.tiles() self.assertEqual(arr2.shape, (30, 20, 10)) self.assertEqual(calc_shape(arr2), arr2.shape) self.assertEqual(len(arr2.chunks), 126) self.assertEqual(arr2.chunks[0].shape, (5, 3, 4)) self.assertEqual(arr2.chunks[-1].shape, (5, 2, 2)) self.assertEqual(calc_shape(arr2.chunks[0]), arr2.chunks[0].shape) self.assertEqual(calc_shape(arr2.chunks[-1]), arr2.chunks[-1].shape) with self.assertRaises(ValueError): transpose(arr, axes=(1, 0)) arr3 = transpose(arr, (-2, 2, 0)) arr3.tiles() self.assertEqual(arr3.shape, (20, 30, 10)) self.assertEqual(calc_shape(arr3), arr3.shape) self.assertEqual(len(arr3.chunks), 126) self.assertEqual(arr3.chunks[0].shape, (3, 5, 4)) self.assertEqual(arr3.chunks[-1].shape, (2, 5, 2)) self.assertEqual(calc_shape(arr3.chunks[0]), arr3.chunks[0].shape) self.assertEqual(calc_shape(arr3.chunks[-1]), arr3.chunks[-1].shape) arr4 = arr.transpose(-2, 2, 0) arr4.tiles() self.assertEqual(arr4.shape, (20, 30, 10)) self.assertEqual(calc_shape(arr4), arr4.shape) self.assertEqual(len(arr4.chunks), 126) self.assertEqual(arr4.chunks[0].shape, (3, 5, 4)) self.assertEqual(arr4.chunks[-1].shape, (2, 5, 2)) self.assertEqual(calc_shape(arr4.chunks[0]), arr4.chunks[0].shape) self.assertEqual(calc_shape(arr4.chunks[-1]), arr4.chunks[-1].shape) arr5 = arr.T arr5.tiles() self.assertEqual(arr5.shape, (30, 20, 10)) self.assertEqual(calc_shape(arr5), arr5.shape) self.assertEqual(len(arr5.chunks), 126) self.assertEqual(arr5.chunks[0].shape, (5, 3, 4)) self.assertEqual(arr5.chunks[-1].shape, (5, 2, 2)) self.assertEqual(calc_shape(arr5.chunks[0]), arr5.chunks[0].shape) self.assertEqual(calc_shape(arr5.chunks[-1]), arr5.chunks[-1].shape)
def testFrexp(self): t1 = ones((3, 4, 5), chunks=2) op_type = type(t1.op) o1, o2 = frexp(t1) self.assertIs(o1.op, o2.op) self.assertNotEqual(o1.dtype, o2.dtype) o1, o2 = frexp(t1, t1) self.assertIs(o1, t1) self.assertIsNot(o1.inputs[0], t1) self.assertIsInstance(o1.inputs[0].op, op_type) self.assertIsNot(o2.inputs[0], t1)
def testAddWithOut(self): t1 = ones((3, 4), chunk_size=2) t2 = ones(4, chunk_size=2) t3 = add(t1, t2, out=t1) self.assertIsInstance(t1.op, Add) self.assertEqual(t1.op.out.key, t1.op.lhs.key) self.assertIs(t3, t1) self.assertEqual(t3.shape, (3, 4)) self.assertEqual(t3.op.lhs.params.raw_chunk_size, 2) self.assertIs(t3.op.rhs, t2.data) self.assertNotEqual(t3.key, t3.op.lhs.key) self.assertEqual(calc_shape(t3), t3.shape) t3.tiles() self.assertIsInstance(t1.chunks[0].op, Add) self.assertEqual(t1.chunks[0].op.out.key, t1.chunks[0].op.lhs.key) self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape) with self.assertRaises(TypeError): add(t1, t2, out=1) with self.assertRaises(ValueError): add(t1, t2, out=t2) with self.assertRaises(TypeError): truediv(t1, t2, out=t1.astype('i8')) t1 = ones((3, 4), chunk_size=2, dtype=float) t2 = ones(4, chunk_size=2, dtype=int) t3 = add(t2, 1, out=t1) self.assertEqual(t3.shape, (3, 4)) self.assertEqual(t3.dtype, np.float64)
def testIndicesIndexingTiles(self): t = ones((10, 20, 30), chunks=(2, 20, 30)) t2 = t[3] t2.tiles() self.assertEqual(len(t2.chunks), 1) self.assertIs(t2.chunks[0].inputs[0], t.cix[1, 0, 0].data) self.assertEqual(t2.chunks[0].op.indexes[0], 1) t3 = t[4] t3.tiles() self.assertEqual(len(t3.chunks), 1) self.assertIs(t3.chunks[0].inputs[0], t.cix[2, 0, 0].data) self.assertEqual(t3.chunks[0].op.indexes[0], 0)
def testBaseExecution(self): arr = ones((10, 8), chunk_size=2) arr2 = arr + 1 res = self.executor.execute_tensor(arr2) self.assertTrue((res[0] == np.ones((2, 2)) + 1).all()) data = np.random.random((10, 8, 3)) arr = tensor(data, chunk_size=2) arr2 = arr + 1 res = self.executor.execute_tensor(arr2) self.assertTrue((res[0] == data[:2, :2, :2] + 1).all())
def testDtypeExecution(self): a = ones((10, 20), dtype='f4', chunks=5) c = truediv(a, 2, dtype='f8') res = self.executor.execute_tensor(c, concat=True)[0] self.assertEqual(res.dtype, np.float64) c = truediv(a, 0, dtype='f8') res = self.executor.execute_tensor(c, concat=True)[0] self.assertTrue(np.isinf(res[0, 0])) with self.assertRaises(FloatingPointError): with np.errstate(divide='raise'): c = truediv(a, 0, dtype='f8') _ = self.executor.execute_tensor(c, concat=True)[0]
def testStandardFFT(self): t = ones((10, 20, 30), chunk_size=(3, 20, 30)) t1 = fft.fft(t) self.assertEqual(t1.shape, (10, 20, 30)) self.assertEqual(calc_shape(t1), t1.shape) t1.tiles() self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits)) self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape) t = ones((10, 20, 30), chunk_size=(3, 20, 30)) t1 = fft.ifft(t) self.assertEqual(t1.shape, (10, 20, 30)) self.assertEqual(calc_shape(t1), t1.shape) t1.tiles() self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits)) self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape) t = ones((10, 20, 30), chunk_size=(3, 20, 30)) t1 = fft.fft2(t, s=(23, 21)) self.assertEqual(calc_shape(t1), t1.shape) self.assertEqual(t1.shape, (10, 23, 21)) t1.tiles() self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits)) self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape) t = ones((10, 20, 30), chunk_size=(3, 20, 30)) t1 = fft.ifft2(t, s=(11, 9), axes=(1, 2)) self.assertEqual(calc_shape(t1), t1.shape) self.assertEqual(t1.shape, (10, 11, 9)) t1.tiles() self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits)) self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape) t = ones((10, 20, 30), chunk_size=(3, 20, 30)) t1 = fft.fftn(t, s=(11, 9), axes=(1, 2)) self.assertEqual(calc_shape(t1), t1.shape) self.assertEqual(t1.shape, (10, 11, 9)) t1.tiles() self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits)) self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape) t = ones((10, 20, 30), chunk_size=(3, 20, 30)) t1 = fft.ifftn(t, s=(11, 9), axes=(1, 2)) self.assertEqual(calc_shape(t1), t1.shape) self.assertEqual(t1.shape, (10, 11, 9)) t1.tiles() self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits)) self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape)
def testRechunk(self): tensor = ones((12, 9), chunk_size=4) new_tensor = tensor.rechunk(3) new_tensor.tiles() self.assertEqual(len(new_tensor.chunks), 12) self.assertEqual(new_tensor.chunks[0].inputs[0], tensor.chunks[0].data) self.assertEqual(len(new_tensor.chunks[1].inputs), 2) self.assertEqual(new_tensor.chunks[1].inputs[0].op.slices, (slice(None, 3, None), slice(3, None, None))) self.assertEqual(new_tensor.chunks[1].inputs[1].op.slices, (slice(None, 3, None), slice(None, 2, None))) self.assertEqual(len(new_tensor.chunks[-1].inputs), 2) self.assertEqual(new_tensor.chunks[-1].inputs[0].op.slices, (slice(1, None, None), slice(2, None, None))) self.assertEqual(new_tensor.chunks[-1].inputs[1].op.slices, (slice(1, None, None), slice(None, None, None)))
def testLogWithOutWhere(self): t1 = ones((3, 4), chunk_size=2) t2 = log(t1, out=t1) self.assertIsInstance(t2.op, TensorLog) self.assertEqual(t1.op.out.key, t1.op.input.key) self.assertIs(t2, t1) self.assertEqual(t2.op.input.extra_params.raw_chunk_size, 2) self.assertNotEqual(t2.key, t2.op.input.key) t3 = empty((3, 4), chunk_size=2) t4 = log(t1, out=t3, where=t1 > 0) self.assertIsInstance(t4.op, TensorLog) self.assertIs(t4, t3) self.assertEqual(t2.op.input.extra_params.raw_chunk_size, 2) self.assertNotEqual(t2.key, t2.op.input.key)
def testArgReduction(self): argmax = lambda x, *args, **kwargs: x.argmax(*args, **kwargs).tiles() argmin = lambda x, *args, **kwargs: x.argmin(*args, **kwargs).tiles() res1 = argmax(ones((10, 8, 10), chunk_size=3)) res2 = argmin(ones((10, 8, 10), chunk_size=3)) self.assertEqual(res1.shape, ()) self.assertIsNotNone(res1.dtype) self.assertEqual(res2.shape, ()) self.assertIsInstance(res1.chunks[0].op, TensorArgmax) self.assertIsInstance(res2.chunks[0].op, TensorArgmin) self.assertIsInstance(res1.chunks[0].inputs[0].op, TensorConcatenate) self.assertIsInstance(res2.chunks[0].inputs[0].op, TensorConcatenate) self.assertIsInstance(res1.chunks[0].inputs[0].inputs[0].op, TensorArgmaxCombine) self.assertIsInstance(res2.chunks[0].inputs[0].inputs[0].op, TensorArgminCombine) res1 = argmax(ones((10, 8), chunk_size=3), axis=1, keepdims=True) res2 = argmin(ones((10, 8), chunk_size=3), axis=1, keepdims=True) self.assertEqual(res1.shape, (10, 1)) self.assertEqual(res2.shape, (10, 1)) self.assertIsInstance(res1.chunks[0].op, TensorArgmax) self.assertIsInstance(res2.chunks[0].op, TensorArgmin) self.assertIsInstance(res1.chunks[0].inputs[0].op, TensorConcatenate) self.assertIsInstance(res2.chunks[0].inputs[0].op, TensorConcatenate) self.assertIsInstance(res1.chunks[0].inputs[0].inputs[0].op, TensorArgmaxChunk) self.assertIsInstance(res2.chunks[0].inputs[0].inputs[0].op, TensorArgminChunk) self.assertRaises( TypeError, lambda: argmax(ones((10, 8, 10), chunk_size=3), axis=(0, 1))) self.assertRaises( TypeError, lambda: argmin(ones((10, 8, 10), chunk_size=3), axis=(0, 1))) self.assertRaises( np.AxisError, lambda: argmin(ones((10, 8, 10), chunk_size=3), axis=3)) self.assertRaises( np.AxisError, lambda: argmin(ones((10, 8, 10), chunk_size=3), axis=-4))
def testSliceTiles(self): t = ones((100, 200, 300), chunks=30) t2 = t[10:40, 199:, -30:303] t2.tiles() self.assertEqual(t2.chunk_shape, (2, 1, 1)) self.assertEqual(t2.chunks[0].inputs[0], t.cix[0, -1, -1].data) self.assertEqual( t2.chunks[0].op.indexes, [slice(10, 30, 1), slice(19, 20, 1), slice(None)]) self.assertEqual(t2.chunks[0].index, (0, 0, 0)) self.assertEqual(t2.chunks[1].inputs[0], t.cix[1, -1, -1].data) self.assertEqual( t2.chunks[1].op.indexes, [slice(0, 10, 1), slice(19, 20, 1), slice(None)]) self.assertEqual(t2.chunks[1].index, (1, 0, 0))
def testRealFFT(self): t = ones((10, 20, 30), chunks=(3, 20, 30)) t1 = fft.rfft(t) self.assertEqual(t1.shape, np.fft.rfft(np.ones(t.shape)).shape) t1.tiles() self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits)) t = ones((10, 20, 30), chunks=(3, 20, 30)) t1 = fft.irfft(t) self.assertEqual(t1.shape, np.fft.irfft(np.ones(t.shape)).shape) t1.tiles() self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits)) t = ones((10, 20, 30), chunks=(3, 20, 30)) t1 = fft.rfft2(t, s=(23, 21)) self.assertEqual(t1.shape, np.fft.rfft2(np.ones(t.shape), s=(23, 21)).shape) t1.tiles() self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits)) t = ones((10, 20, 30), chunks=(3, 20, 30)) t1 = fft.irfft2(t, s=(11, 9), axes=(1, 2)) self.assertEqual( t1.shape, np.fft.irfft2(np.ones(t.shape), s=(11, 9), axes=(1, 2)).shape) t1.tiles() self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits)) t = ones((10, 20, 30), chunks=(3, 20, 30)) t1 = fft.rfftn(t, s=(11, 30), axes=(1, 2)) self.assertEqual( t1.shape, np.fft.rfftn(np.ones(t.shape), s=(11, 30), axes=(1, 2)).shape) t1.tiles() self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits)) t = ones((10, 20, 30), chunks=(3, 20, 30)) t1 = fft.irfftn(t, s=(11, 9), axes=(1, 2)) self.assertEqual( t1.shape, np.fft.irfftn(np.ones(t.shape), s=(11, 9), axes=(1, 2)).shape) t1.tiles() self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
def testComputeRechunk(self): tensor = ones((12, 8), chunk_size=((4, 4, 3, 1), (3, 3, 2))) tensor.tiles() new_tensor = compute_rechunk(tensor, ((9, 2, 1), (2, 1, 4, 1))) self.assertEqual(len(new_tensor.chunks), 12) self.assertEqual(len(new_tensor.chunks[0].inputs), 3) self.assertIsInstance(new_tensor.chunks[0].inputs[0].op, TensorSlice) self.assertIs(new_tensor.chunks[0].inputs[0].inputs[0], tensor.chunks[0].data) self.assertEqual(new_tensor.chunks[0].inputs[0].op.slices, (slice(None, None, None), slice(None, 2, None))) self.assertIs(new_tensor.chunks[0].inputs[1].inputs[0], tensor.chunks[3].data) self.assertEqual(new_tensor.chunks[0].inputs[1].op.slices, (slice(None, None, None), slice(None, 2, None))) self.assertIs(new_tensor.chunks[0].inputs[2].inputs[0], tensor.chunks[6].data) self.assertEqual(new_tensor.chunks[0].inputs[2].op.slices, (slice(None, 1, None), slice(None, 2, None))) self.assertIs(new_tensor.chunks[-1].inputs[0], tensor.chunks[-1].data) self.assertEqual(new_tensor.chunks[-1].op.slices, (slice(None, None, None), slice(1, None, None)))
def testCreateView(self): arr = ones((10, 20, 30), chunk_size=[4, 3, 5]) arr2 = transpose(arr) self.assertTrue(arr2.op.create_view) arr3 = transpose(arr) self.assertTrue(arr3.op.create_view) arr4 = arr.swapaxes(0, 1) self.assertTrue(arr4.op.create_view) arr5 = moveaxis(arr, 1, 0) self.assertTrue(arr5.op.create_view) arr6 = atleast_1d(1) self.assertTrue(arr6.op.create_view) arr7 = atleast_2d([1, 1]) self.assertTrue(arr7.op.create_view) arr8 = atleast_3d([1, 1]) self.assertTrue(arr8.op.create_view) arr9 = arr[:3, [1, 2, 3]] # no view cuz of fancy indexing self.assertFalse(arr9.op.create_view) arr10 = arr[:3, None, :5] self.assertTrue(arr10.op.create_view) data = np.array([[[0], [1], [2]]]) x = tensor(data) t = squeeze(x) self.assertTrue(t.op.create_view) y = x.reshape(3) self.assertTrue(y.op.create_view)
def testFrexp(self): t1 = ones((3, 4, 5), chunk_size=2) t2 = empty((3, 4, 5), dtype=np.float_, chunk_size=2) op_type = type(t1.op) o1, o2 = frexp(t1) self.assertIs(o1.op, o2.op) self.assertNotEqual(o1.dtype, o2.dtype) o1, o2 = frexp(t1, t1) self.assertIs(o1, t1) self.assertIsNot(o1.inputs[0], t1) self.assertIsInstance(o1.inputs[0].op, op_type) self.assertIsNot(o2.inputs[0], t1) o1, o2 = frexp(t1, t2, where=t1 > 0) op_type = type(t2.op) self.assertIs(o1, t2) self.assertIsNot(o1.inputs[0], t1) self.assertIsInstance(o1.inputs[0].op, op_type) self.assertIsNot(o2.inputs[0], t1)
def testMeanReduction(self): mean = lambda x, *args, **kwargs: x.mean(*args, **kwargs).tiles() res = mean(ones((10, 8), chunk_size=3)) self.assertEqual(res.shape, ()) self.assertIsNotNone(res.dtype) self.assertIsInstance(res.chunks[0].op, Mean) self.assertIsInstance(res.chunks[0].inputs[0].op, Concatenate) self.assertIsInstance(res.chunks[0].inputs[0].inputs[0].op, MeanCombine) res = mean(ones((8, 8), chunk_size=8)) self.assertEqual(res.shape, ()) self.assertEqual(calc_shape(res), res.shape) res = mean(ones((10, 8), chunk_size=3), axis=0) self.assertEqual(res.shape, (8, )) self.assertEqual(calc_shape(res), res.shape) res = mean(ones((10, 8), chunk_size=3), axis=1) self.assertEqual(res.shape, (10, )) self.assertEqual(calc_shape(res), res.shape) res = mean(ones((10, 8), chunk_size=3), keepdims=True) self.assertEqual(res.shape, (1, 1)) self.assertEqual(calc_shape(res), res.shape) res = mean(ones((10, 8), chunk_size=3), axis=0, keepdims=True) self.assertEqual(res.shape, (1, 8)) self.assertEqual(calc_shape(res), res.shape) res = mean(ones((10, 8), chunk_size=3), axis=1, keepdims=True) self.assertEqual(res.shape, (10, 1)) self.assertEqual(calc_shape(res), res.shape) self.assertIsInstance(res.chunks[0].op, Mean) self.assertIsInstance(res.chunks[0].inputs[0].op, Concatenate) self.assertIsInstance(res.chunks[0].inputs[0].inputs[0].op, MeanChunk) self.assertEqual(calc_shape(res.chunks[0]), res.chunks[0].shape)
def testCopyto(self): a = ones((10, 20), chunk_size=3) b = ones(10, chunk_size=4) with self.assertRaises(ValueError): copyto(a, b) tp = type(a.op) b = ones(20, chunk_size=4) copyto(a, b) self.assertIsInstance(a.op, CopyTo) self.assertIs(a.inputs[0], b.data) self.assertIsInstance(a.inputs[1].op, tp) self.assertEqual(calc_shape(a), a.shape) a.tiles() self.assertIsInstance(a.chunks[0].op, CopyTo) self.assertEqual(len(a.chunks[0].inputs), 2) self.assertEqual(calc_shape(a.chunks[0]), a.chunks[0].shape) a = ones((10, 20), chunk_size=3, dtype='i4') b = ones(20, chunk_size=4, dtype='f8') with self.assertRaises(TypeError): copyto(a, b) b = ones(20, chunk_size=4, dtype='i4') copyto(a, b, where=b > 0) self.assertIsNotNone(a.op.where) self.assertEqual(calc_shape(a), a.shape) a.tiles() self.assertIsInstance(a.chunks[0].op, CopyTo) self.assertEqual(len(a.chunks[0].inputs), 3) self.assertEqual(calc_shape(a.chunks[0]), a.chunks[0].shape) with self.assertRaises(ValueError): copyto(a, a, where=np.ones(30, dtype='?'))
def testBaseReduction(self): sum = lambda x, *args, **kwargs: x.sum(*args, **kwargs).tiles() prod = lambda x, *args, **kwargs: x.prod(*args, **kwargs).tiles() max = lambda x, *args, **kwargs: x.max(*args, **kwargs).tiles() min = lambda x, *args, **kwargs: x.min(*args, **kwargs).tiles() all = lambda x, *args, **kwargs: x.all(*args, **kwargs).tiles() any = lambda x, *args, **kwargs: x.any(*args, **kwargs).tiles() for f in [sum, prod, max, min, all, any]: res = f(ones((8, 8), chunk_size=8)) self.assertEqual(res.shape, ()) res = f(ones((10, 8), chunk_size=3)) self.assertIsNotNone(res.dtype) self.assertEqual(res.shape, ()) res = f(ones((10, 8), chunk_size=3), axis=0) self.assertEqual(res.shape, (8, )) res = f(ones((10, 8), chunk_size=3), axis=1) self.assertEqual(res.shape, (10, )) with self.assertRaises(np.AxisError): f(ones((10, 8), chunk_size=3), axis=2) res = f(ones((10, 8), chunk_size=3), axis=-1) self.assertEqual(res.shape, (10, )) with self.assertRaises(np.AxisError): f(ones((10, 8), chunk_size=3), axis=-3) res = f(ones((10, 8), chunk_size=3), keepdims=True) self.assertEqual(res.shape, (1, 1)) res = f(ones((10, 8), chunk_size=3), axis=0, keepdims=True) self.assertEqual(res.shape, (1, 8)) res = f(ones((10, 8), chunk_size=3), axis=1, keepdims=True) self.assertEqual(res.shape, (10, 1)) res = f(ones((10, 8, 10), chunk_size=3), axis=1) self.assertEqual(res.shape, (10, 10)) res = f(ones((10, 8, 10), chunk_size=3), axis=1, keepdims=True) self.assertEqual(res.shape, (10, 1, 10)) res = f(ones((10, 8, 10), chunk_size=3), axis=(0, 2)) self.assertEqual(res.shape, (8, )) res = f(ones((10, 8, 10), chunk_size=3), axis=(0, 2), keepdims=True) self.assertEqual(res.shape, (1, 8, 1))
def testPrefetch(self): t1 = ones((10, 8), chunk_size=10) t2 = ones((1, 8), chunk_size=10) t3 = t1 + t2 self.executor.execute_tensor(t3)
def testAdd(self): t1 = ones((3, 4), chunk_size=2) t2 = ones(4, chunk_size=2) t3 = t1 + t2 k1 = t3.key t3.tiles() self.assertNotEqual(t3.key, k1) self.assertEqual(t3.shape, (3, 4)) self.assertEqual(len(t3.chunks), 4) self.assertEqual(calc_shape(t3), t3.shape) self.assertEqual(t3.chunks[0].inputs, [t1.chunks[0].data, t2.chunks[0].data]) self.assertEqual(t3.chunks[1].inputs, [t1.chunks[1].data, t2.chunks[1].data]) self.assertEqual(t3.chunks[2].inputs, [t1.chunks[2].data, t2.chunks[0].data]) self.assertEqual(t3.chunks[3].inputs, [t1.chunks[3].data, t2.chunks[1].data]) self.assertEqual(t3.op.dtype, np.dtype('f8')) self.assertEqual(t3.chunks[0].op.dtype, np.dtype('f8')) self.assertEqual(calc_shape(t3.chunks[0]), t3.chunks[0].shape) t4 = t1 + 1 t4.tiles() self.assertEqual(t4.shape, (3, 4)) self.assertEqual(len(t3.chunks), 4) self.assertEqual(calc_shape(t4), t4.shape) self.assertEqual(t4.chunks[0].inputs, [t1.chunks[0].data]) self.assertEqual(t4.chunks[0].op.constant[0], 1) self.assertEqual(t4.chunks[1].inputs, [t1.chunks[1].data]) self.assertEqual(t4.chunks[1].op.constant[0], 1) self.assertEqual(t4.chunks[2].inputs, [t1.chunks[2].data]) self.assertEqual(t4.chunks[2].op.constant[0], 1) self.assertEqual(t4.chunks[3].inputs, [t1.chunks[3].data]) self.assertEqual(t4.chunks[3].op.constant[0], 1) self.assertEqual(calc_shape(t4.chunks[0]), t4.chunks[0].shape) # sparse tests t5 = add([1, 2, 3, 4], 1) t5.tiles() self.assertEqual(t4.chunks[0].inputs, [t1.chunks[0].data]) t1 = tensor([[0, 1, 0], [1, 0, 0]], chunk_size=2).tosparse() t = t1 + 1 self.assertFalse(t.issparse()) self.assertIs(type(t), Tensor) t.tiles() self.assertFalse(t.chunks[0].op.sparse) t = t1 + 0 self.assertTrue(t.issparse()) self.assertIs(type(t), SparseTensor) t2 = tensor([[1, 0, 0]], chunk_size=2).tosparse() t = t1 + t2 self.assertTrue(t.issparse()) self.assertIs(type(t), SparseTensor) t.tiles() self.assertTrue(t.chunks[0].op.sparse) t3 = tensor([1, 1, 1], chunk_size=2) t = t1 + t3 self.assertFalse(t.issparse()) self.assertIs(type(t), Tensor) t.tiles() self.assertFalse(t.chunks[0].op.sparse)
def testTensordotExecution(self): size_executor = Executor( sync_provider_type=Executor.SyncProviderType.MOCK) a_data = np.arange(60).reshape(3, 4, 5) a = tensor(a_data, chunk_size=2) b_data = np.arange(24).reshape(4, 3, 2) b = tensor(b_data, chunk_size=2) axes = ([1, 0], [0, 1]) c = tensordot(a, b, axes=axes) size_res = size_executor.execute_tensor(c, mock=True) res = self.executor.execute_tensor(c) expected = np.tensordot(a_data, b_data, axes=axes) self.assertEqual(sum(s[0] for s in size_res), c.nbytes) self.assertTrue(np.array_equal(res[0], expected[:2, :])) self.assertTrue(np.array_equal(res[1], expected[2:4, :])) self.assertTrue(np.array_equal(res[2], expected[4:, :])) a = ones((1000, 2000), chunk_size=500) b = ones((2000, 100), chunk_size=500) c = dot(a, b) res = self.executor.execute_tensor(c) expected = np.dot(np.ones((1000, 2000)), np.ones((2000, 100))) self.assertEqual(len(res), 2) self.assertTrue(np.array_equal(res[0], expected[:500, :])) self.assertTrue(np.array_equal(res[1], expected[500:, :])) a = ones((10, 8), chunk_size=2) b = ones((8, 10), chunk_size=2) c = a.dot(b) res = self.executor.execute_tensor(c) self.assertEqual(len(res), 25) for r in res: self.assertTrue(np.array_equal(r, np.tile([8], [2, 2]))) a = ones((500, 500), chunk_size=500) b = ones((500, 100), chunk_size=500) c = a.dot(b) res = self.executor.execute_tensor(c) self.assertTrue(np.array_equal(res[0], np.tile([500], [500, 100]))) raw_a = np.random.random((100, 200, 50)) raw_b = np.random.random((200, 10, 100)) a = tensor(raw_a, chunk_size=50) b = tensor(raw_b, chunk_size=33) c = tensordot(a, b, axes=((0, 1), (2, 0))) res = self.executor.execute_tensor(c, concat=True) expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes)) self.assertTrue(np.allclose(res[0], expected)) a = ones((1000, 2000), chunk_size=500) b = ones((100, 2000), chunk_size=500) c = inner(a, b) res = self.executor.execute_tensor(c) expected = np.inner(np.ones((1000, 2000)), np.ones((100, 2000))) self.assertEqual(len(res), 2) self.assertTrue(np.array_equal(res[0], expected[:500, :])) self.assertTrue(np.array_equal(res[1], expected[500:, :])) a = ones((100, 100), chunk_size=30) b = ones((100, 100), chunk_size=30) c = a.dot(b) res = self.executor.execute_tensor(c, concat=True)[0] np.testing.assert_array_equal(res, np.ones((100, 100)) * 100)
def testBuildMode(self): t1 = ones((2, 3), chunk_size=2) self.assertTrue(t1 == 2) with build_mode(): self.assertFalse(t1 == 2)
def testMatmul(self): a_data = [[1, 0], [0, 1]] b_data = [[4, 1], [2, 2]] a = tensor(a_data, chunk_size=1) b = tensor(b_data, chunk_size=1) t = matmul(a, b) self.assertEqual(t.shape, (2, 2)) self.assertEqual(calc_shape(t), t.shape) t.tiles() self.assertEqual(t.shape, tuple(sum(s) for s in t.nsplits)) self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape) b_data = [1, 2] b = tensor(b_data, chunk_size=1) t = matmul(a, b) self.assertEqual(t.shape, (2, )) self.assertEqual(calc_shape(t), t.shape) t.tiles() self.assertEqual(t.shape, tuple(sum(s) for s in t.nsplits)) self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape) t = matmul(b, a) self.assertEqual(t.shape, (2, )) self.assertEqual(calc_shape(t), t.shape) t.tiles() self.assertEqual(t.shape, tuple(sum(s) for s in t.nsplits)) self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape) a_data = np.arange(2 * 2 * 4).reshape((2, 2, 4)) b_data = np.arange(2 * 2 * 4).reshape((2, 4, 2)) a = tensor(a_data, chunk_size=1) b = tensor(b_data, chunk_size=1) t = matmul(a, b) self.assertEqual(t.shape, (2, 2, 2)) self.assertEqual(calc_shape(t), t.shape) t.tiles() self.assertEqual(t.shape, tuple(sum(s) for s in t.nsplits)) self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape) t = matmul(tensor([2j, 3j], chunk_size=1), tensor([2j, 3j], chunk_size=1)) self.assertEqual(t.shape, ()) self.assertEqual(calc_shape(t), t.shape) t.tiles() self.assertEqual(t.shape, tuple(sum(s) for s in t.nsplits)) self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape) with self.assertRaises(ValueError): matmul([1, 2], 3) with self.assertRaises(ValueError): matmul(np.random.randn(2, 3, 4), np.random.randn(3, 4, 3)) t = matmul(tensor(np.random.randn(2, 3, 4), chunk_size=2), tensor(np.random.randn(3, 1, 4, 3), chunk_size=3)) self.assertEqual(t.shape, (3, 2, 3, 3)) v = ones((100, 100), chunk_size=10) tv = matmul(v, v) self.assertEqual(tv.shape, (100, 100)) self.assertEqual(calc_shape(tv), tv.shape) tv.tiles() self.assertEqual(tv.shape, tuple(sum(s) for s in tv.nsplits)) self.assertEqual(calc_shape(tv.chunks[0]), tv.chunks[0].shape)