def testBroadcastTo(self):
arr = ones((10, 5), chunk_size=2)
arr2 = broadcast_to(arr, (20, 10, 5))
arr2.tiles()
self.assertEqual(arr2.shape, (20, 10, 5))
self.assertEqual(calc_shape(arr2), arr2.shape)
self.assertEqual(len(arr2.chunks), len(arr.chunks))
self.assertEqual(arr2.chunks[0].shape, (20, 2, 2))
self.assertEqual(calc_shape(arr2.chunks[-1]), arr2.chunks[-1].shape)
arr = ones((10, 5, 1), chunk_size=2)
arr3 = broadcast_to(arr, (5, 10, 5, 6))
arr3.tiles()
self.assertEqual(arr3.shape, (5, 10, 5, 6))
self.assertEqual(calc_shape(arr3), arr3.shape)
self.assertEqual(len(arr3.chunks), len(arr.chunks))
self.assertEqual(arr3.nsplits,
((5, ), (2, 2, 2, 2, 2), (2, 2, 1), (6, )))
self.assertEqual(arr3.chunks[0].shape, (5, 2, 2, 6))
self.assertEqual(calc_shape(arr3.chunks[-1]), arr3.chunks[-1].shape)
arr = ones((10, 1), chunk_size=2)
arr4 = broadcast_to(arr, (20, 10, 5))
arr4.tiles()
self.assertEqual(arr4.shape, (20, 10, 5))
self.assertEqual(calc_shape(arr4), arr4.shape)
self.assertEqual(len(arr4.chunks), len(arr.chunks))
self.assertEqual(arr4.chunks[0].shape, (20, 2, 5))
self.assertEqual(calc_shape(arr4.chunks[-1]), arr4.chunks[-1].shape)
with self.assertRaises(ValueError):
broadcast_to(arr, (10, ))
with self.assertRaises(ValueError):
broadcast_to(arr, (5, 1))
def testBoolIndexingTiles(self):
t = ones((100, 200, 300), chunks=30)
indexed = t[t < 2]
indexed.tiles()
self.assertEqual(len(indexed.chunks), 280)
self.assertEqual(indexed.chunks[0].index, (0, ))
self.assertEqual(indexed.chunks[20].index, (20, ))
self.assertIs(indexed.chunks[20].inputs[0], t.cix[(0, 2, 0)].data)
self.assertIs(indexed.chunks[20].inputs[1],
indexed.op.indexes[0].cix[0, 2, 0].data)
t2 = ones((100, 200), chunks=30)
indexed2 = t[t2 < 2]
indexed2.tiles()
self.assertEqual(len(indexed2.chunks), 280)
self.assertEqual(len(indexed2.chunks[0].shape), 2)
self.assertTrue(np.isnan(indexed2.chunks[0].shape[0]))
self.assertEqual(indexed2.chunks[0].shape[1], 30)
self.assertEqual(indexed2.chunks[20].inputs[0], t.cix[(0, 2, 0)].data)
self.assertEqual(indexed2.chunks[20].inputs[1],
indexed2.op.indexes[0].cix[0, 2].data)
def testFancyIndexing(self):
t = ones((100, 200, 300))
t2 = t[[0, 1], [2, 3]]
self.assertEqual(t2.shape, (2, 300))
t3 = t[[[0, 1], [2, 3]], [4, 5]]
self.assertEqual(t3.shape, (2, 2, 300))
with self.assertRaises(IndexError) as cm:
_ = t[[1, 2], [3, 4, 5]]
e = str(cm.exception)
self.assertEqual(
e, 'shape mismatch: indexing arrays could not be broadcast '
'together with shapes (2,) (3,)')
t = ones((100, 200, 300), chunks=10)
t4 = t[:10, -10:, [13, 244, 151, 242, 34]].tiles()
self.assertEqual(t4.shape, (10, 10, 5))
self.assertEqual(t4.chunk_shape, (1, 1, 1))
t5 = t[:10, -10:, [1, 10, 20, 33, 34, 200]].tiles()
self.assertEqual(t5.shape, (10, 10, 6))
self.assertEqual(t5.chunk_shape, (1, 1, 5))
def testCumReduction(self):
cumsum = lambda x, *args, **kwargs: x.cumsum(*args, **kwargs).tiles()
cumprod = lambda x, *args, **kwargs: x.cumprod(*args, **kwargs).tiles()
res1 = cumsum(ones((10, 8), chunk_size=3), axis=0)
res2 = cumprod(ones((10, 8), chunk_size=3), axis=0)
self.assertEqual(res1.shape, (10, 8))
self.assertIsNotNone(res1.dtype)
self.assertEqual(calc_shape(res1), res1.shape)
self.assertEqual(calc_shape(res1.chunks[0]), res1.chunks[0].shape)
self.assertEqual(res2.shape, (10, 8))
self.assertIsNotNone(res2.dtype)
self.assertEqual(calc_shape(res2), res2.shape)
self.assertEqual(calc_shape(res2.chunks[0]), res2.chunks[0].shape)
res1 = cumsum(ones((10, 8, 8), chunk_size=3), axis=1)
res2 = cumprod(ones((10, 8, 8), chunk_size=3), axis=1)
self.assertEqual(res1.shape, (10, 8, 8))
self.assertEqual(calc_shape(res1), res1.shape)
self.assertEqual(calc_shape(res1.chunks[0]), res1.chunks[0].shape)
self.assertEqual(res2.shape, (10, 8, 8))
self.assertEqual(calc_shape(res2), res2.shape)
self.assertEqual(calc_shape(res2.chunks[0]), res2.chunks[0].shape)
def testAstype(self):
arr = ones((10, 20, 30), chunk_size=3)
arr2 = arr.astype(np.int32)
arr2.tiles()
self.assertEqual(arr2.shape, (10, 20, 30))
self.assertEqual(calc_shape(arr2), arr2.shape)
self.assertTrue(np.issubdtype(arr2.dtype, np.int32))
self.assertEqual(arr2.op.casting, 'unsafe')
self.assertEqual(calc_shape(arr2.chunks[0]), arr2.chunks[0].shape)
with self.assertRaises(TypeError):
arr.astype(np.int32, casting='safe')
def testBoolIndexingTiles(self):
t = ones((100, 200, 300), chunk_size=30)
indexed = t[t < 2]
indexed.tiles()
self.assertEqual(len(indexed.chunks), 280)
self.assertEqual(indexed.chunks[0].index, (0, ))
self.assertEqual(indexed.chunks[20].index, (20, ))
self.assertIs(indexed.chunks[20].inputs[0], t.cix[(0, 2, 0)].data)
self.assertIs(indexed.chunks[20].inputs[1],
indexed.op.indexes[0].cix[0, 2, 0].data)
self.assertEqual(calc_shape(indexed.chunks[0]),
indexed.chunks[0].shape)
self.assertEqual(calc_shape(indexed.chunks[20]),
indexed.chunks[20].shape)
self.assertEqual(indexed.chunks[0].rough_nbytes, t.chunks[0].nbytes)
self.assertEqual(indexed.chunks[20].rough_nbytes, t.chunks[20].nbytes)
t2 = ones((100, 200), chunk_size=30)
indexed2 = t[t2 < 2]
indexed2.tiles()
self.assertEqual(len(indexed2.chunks), 280)
self.assertEqual(len(indexed2.chunks[0].shape), 2)
self.assertTrue(np.isnan(indexed2.chunks[0].shape[0]))
self.assertEqual(indexed2.chunks[0].shape[1], 30)
self.assertEqual(indexed2.chunks[20].inputs[0], t.cix[(0, 2, 0)].data)
self.assertEqual(indexed2.chunks[20].inputs[1],
indexed2.op.indexes[0].cix[0, 2].data)
self.assertEqual(indexed2.chunks[0].rough_nbytes,
30 * 30 * 30 * t2.dtype.itemsize)
self.assertEqual(calc_shape(indexed2.chunks[0]),
indexed2.chunks[0].shape)
self.assertEqual(calc_shape(indexed2.chunks[20]),
indexed2.chunks[20].shape)
self.assertEqual(indexed2.chunks[0].rough_nbytes, t.chunks[0].nbytes)
self.assertEqual(indexed2.chunks[20].rough_nbytes, t.chunks[20].nbytes)
def testMixedIndexingTiles(self):
t = ones((100, 200, 300, 400), chunk_size=24)
cmp = ones(400, chunk_size=24) < 2
t2 = t[10:90:3, 5, ..., None, cmp]
t2.tiles()
self.assertEqual(t2.shape[:-1], (27, 300, 1))
self.assertTrue(np.isnan(t2.shape[-1]))
self.assertEqual(t2.rough_nbytes, 27 * 300 * 400 * t.dtype.itemsize)
self.assertEqual(t2.chunk_shape, (4, 13, 1, 17))
self.assertEqual(
t2.chunks[0].op.indexes,
[slice(10, 24, 3), 5,
slice(None), None, cmp.cix[0, ].data])
self.assertEqual(calc_shape(t2.chunks[0]), t2.chunks[0].shape)
self.assertEqual(t2.chunks[0].rough_nbytes,
5 * 24 * 24 * t.dtype.itemsize)
# multiple tensor type as indexes
t3 = ones((100, 200, 300, 400), chunk_size=24)
cmp2 = ones((100, 200), chunk_size=24) < 2
cmp3 = ones(400, chunk_size=24) < 2
t4 = t3[cmp2, 5, None, cmp3]
t4.tiles()
self.assertEqual(t4.shape[1], 1)
self.assertTrue(np.isnan(t4.shape[0]))
self.assertTrue(np.isnan(t4.shape[-1]))
self.assertEqual(calc_shape(t4), t4.shape)
self.assertEqual(t4.rough_nbytes, 100 * 200 * 400 * t.dtype.itemsize)
self.assertEqual(t4.chunk_shape, (45, 1, 17))
self.assertEqual(t4.chunks[0].op.indexes,
[cmp2.cix[0, 0].data, 5, None, cmp3.cix[0, ].data])
self.assertEqual(calc_shape(t4.chunks[0]), t4.chunks[0].shape)
self.assertEqual(t4.chunks[0].rough_nbytes,
24 * 24 * 24 * t3.dtype.itemsize)
def testTranspose(self):
arr = ones((10, 20, 30), chunk_size=[4, 3, 5])
arr2 = transpose(arr)
arr2.tiles()
self.assertEqual(arr2.shape, (30, 20, 10))
self.assertEqual(calc_shape(arr2), arr2.shape)
self.assertEqual(len(arr2.chunks), 126)
self.assertEqual(arr2.chunks[0].shape, (5, 3, 4))
self.assertEqual(arr2.chunks[-1].shape, (5, 2, 2))
self.assertEqual(calc_shape(arr2.chunks[0]), arr2.chunks[0].shape)
self.assertEqual(calc_shape(arr2.chunks[-1]), arr2.chunks[-1].shape)
with self.assertRaises(ValueError):
transpose(arr, axes=(1, 0))
arr3 = transpose(arr, (-2, 2, 0))
arr3.tiles()
self.assertEqual(arr3.shape, (20, 30, 10))
self.assertEqual(calc_shape(arr3), arr3.shape)
self.assertEqual(len(arr3.chunks), 126)
self.assertEqual(arr3.chunks[0].shape, (3, 5, 4))
self.assertEqual(arr3.chunks[-1].shape, (2, 5, 2))
self.assertEqual(calc_shape(arr3.chunks[0]), arr3.chunks[0].shape)
self.assertEqual(calc_shape(arr3.chunks[-1]), arr3.chunks[-1].shape)
arr4 = arr.transpose(-2, 2, 0)
arr4.tiles()
self.assertEqual(arr4.shape, (20, 30, 10))
self.assertEqual(calc_shape(arr4), arr4.shape)
self.assertEqual(len(arr4.chunks), 126)
self.assertEqual(arr4.chunks[0].shape, (3, 5, 4))
self.assertEqual(arr4.chunks[-1].shape, (2, 5, 2))
self.assertEqual(calc_shape(arr4.chunks[0]), arr4.chunks[0].shape)
self.assertEqual(calc_shape(arr4.chunks[-1]), arr4.chunks[-1].shape)
arr5 = arr.T
arr5.tiles()
self.assertEqual(arr5.shape, (30, 20, 10))
self.assertEqual(calc_shape(arr5), arr5.shape)
self.assertEqual(len(arr5.chunks), 126)
self.assertEqual(arr5.chunks[0].shape, (5, 3, 4))
self.assertEqual(arr5.chunks[-1].shape, (5, 2, 2))
self.assertEqual(calc_shape(arr5.chunks[0]), arr5.chunks[0].shape)
self.assertEqual(calc_shape(arr5.chunks[-1]), arr5.chunks[-1].shape)
def testFrexp(self):
t1 = ones((3, 4, 5), chunks=2)
op_type = type(t1.op)
o1, o2 = frexp(t1)
self.assertIs(o1.op, o2.op)
self.assertNotEqual(o1.dtype, o2.dtype)
o1, o2 = frexp(t1, t1)
self.assertIs(o1, t1)
self.assertIsNot(o1.inputs[0], t1)
self.assertIsInstance(o1.inputs[0].op, op_type)
self.assertIsNot(o2.inputs[0], t1)
def testAddWithOut(self):
t1 = ones((3, 4), chunk_size=2)
t2 = ones(4, chunk_size=2)
t3 = add(t1, t2, out=t1)
self.assertIsInstance(t1.op, Add)
self.assertEqual(t1.op.out.key, t1.op.lhs.key)
self.assertIs(t3, t1)
self.assertEqual(t3.shape, (3, 4))
self.assertEqual(t3.op.lhs.params.raw_chunk_size, 2)
self.assertIs(t3.op.rhs, t2.data)
self.assertNotEqual(t3.key, t3.op.lhs.key)
self.assertEqual(calc_shape(t3), t3.shape)
t3.tiles()
self.assertIsInstance(t1.chunks[0].op, Add)
self.assertEqual(t1.chunks[0].op.out.key, t1.chunks[0].op.lhs.key)
self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape)
with self.assertRaises(TypeError):
add(t1, t2, out=1)
with self.assertRaises(ValueError):
add(t1, t2, out=t2)
with self.assertRaises(TypeError):
truediv(t1, t2, out=t1.astype('i8'))
t1 = ones((3, 4), chunk_size=2, dtype=float)
t2 = ones(4, chunk_size=2, dtype=int)
t3 = add(t2, 1, out=t1)
self.assertEqual(t3.shape, (3, 4))
self.assertEqual(t3.dtype, np.float64)
def testIndicesIndexingTiles(self):
t = ones((10, 20, 30), chunks=(2, 20, 30))
t2 = t[3]
t2.tiles()
self.assertEqual(len(t2.chunks), 1)
self.assertIs(t2.chunks[0].inputs[0], t.cix[1, 0, 0].data)
self.assertEqual(t2.chunks[0].op.indexes[0], 1)
t3 = t[4]
t3.tiles()
self.assertEqual(len(t3.chunks), 1)
self.assertIs(t3.chunks[0].inputs[0], t.cix[2, 0, 0].data)
self.assertEqual(t3.chunks[0].op.indexes[0], 0)
def testBaseExecution(self):
arr = ones((10, 8), chunk_size=2)
arr2 = arr + 1
res = self.executor.execute_tensor(arr2)
self.assertTrue((res[0] == np.ones((2, 2)) + 1).all())
data = np.random.random((10, 8, 3))
arr = tensor(data, chunk_size=2)
arr2 = arr + 1
res = self.executor.execute_tensor(arr2)
self.assertTrue((res[0] == data[:2, :2, :2] + 1).all())
def testDtypeExecution(self):
a = ones((10, 20), dtype='f4', chunks=5)
c = truediv(a, 2, dtype='f8')
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertEqual(res.dtype, np.float64)
c = truediv(a, 0, dtype='f8')
res = self.executor.execute_tensor(c, concat=True)[0]
self.assertTrue(np.isinf(res[0, 0]))
with self.assertRaises(FloatingPointError):
with np.errstate(divide='raise'):
c = truediv(a, 0, dtype='f8')
_ = self.executor.execute_tensor(c, concat=True)[0]
def testStandardFFT(self):
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = fft.fft(t)
self.assertEqual(t1.shape, (10, 20, 30))
self.assertEqual(calc_shape(t1), t1.shape)
t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape)
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = fft.ifft(t)
self.assertEqual(t1.shape, (10, 20, 30))
self.assertEqual(calc_shape(t1), t1.shape)
t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape)
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = fft.fft2(t, s=(23, 21))
self.assertEqual(calc_shape(t1), t1.shape)
self.assertEqual(t1.shape, (10, 23, 21))
t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape)
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = fft.ifft2(t, s=(11, 9), axes=(1, 2))
self.assertEqual(calc_shape(t1), t1.shape)
self.assertEqual(t1.shape, (10, 11, 9))
t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape)
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = fft.fftn(t, s=(11, 9), axes=(1, 2))
self.assertEqual(calc_shape(t1), t1.shape)
self.assertEqual(t1.shape, (10, 11, 9))
t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape)
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = fft.ifftn(t, s=(11, 9), axes=(1, 2))
self.assertEqual(calc_shape(t1), t1.shape)
self.assertEqual(t1.shape, (10, 11, 9))
t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
self.assertEqual(calc_shape(t1.chunks[0]), t1.chunks[0].shape)
def testRechunk(self):
tensor = ones((12, 9), chunk_size=4)
new_tensor = tensor.rechunk(3)
new_tensor.tiles()
self.assertEqual(len(new_tensor.chunks), 12)
self.assertEqual(new_tensor.chunks[0].inputs[0], tensor.chunks[0].data)
self.assertEqual(len(new_tensor.chunks[1].inputs), 2)
self.assertEqual(new_tensor.chunks[1].inputs[0].op.slices,
(slice(None, 3, None), slice(3, None, None)))
self.assertEqual(new_tensor.chunks[1].inputs[1].op.slices,
(slice(None, 3, None), slice(None, 2, None)))
self.assertEqual(len(new_tensor.chunks[-1].inputs), 2)
self.assertEqual(new_tensor.chunks[-1].inputs[0].op.slices,
(slice(1, None, None), slice(2, None, None)))
self.assertEqual(new_tensor.chunks[-1].inputs[1].op.slices,
(slice(1, None, None), slice(None, None, None)))
def testLogWithOutWhere(self):
t1 = ones((3, 4), chunk_size=2)
t2 = log(t1, out=t1)
self.assertIsInstance(t2.op, TensorLog)
self.assertEqual(t1.op.out.key, t1.op.input.key)
self.assertIs(t2, t1)
self.assertEqual(t2.op.input.extra_params.raw_chunk_size, 2)
self.assertNotEqual(t2.key, t2.op.input.key)
t3 = empty((3, 4), chunk_size=2)
t4 = log(t1, out=t3, where=t1 > 0)
self.assertIsInstance(t4.op, TensorLog)
self.assertIs(t4, t3)
self.assertEqual(t2.op.input.extra_params.raw_chunk_size, 2)
self.assertNotEqual(t2.key, t2.op.input.key)
def testArgReduction(self):
argmax = lambda x, *args, **kwargs: x.argmax(*args, **kwargs).tiles()
argmin = lambda x, *args, **kwargs: x.argmin(*args, **kwargs).tiles()
res1 = argmax(ones((10, 8, 10), chunk_size=3))
res2 = argmin(ones((10, 8, 10), chunk_size=3))
self.assertEqual(res1.shape, ())
self.assertIsNotNone(res1.dtype)
self.assertEqual(res2.shape, ())
self.assertIsInstance(res1.chunks[0].op, TensorArgmax)
self.assertIsInstance(res2.chunks[0].op, TensorArgmin)
self.assertIsInstance(res1.chunks[0].inputs[0].op, TensorConcatenate)
self.assertIsInstance(res2.chunks[0].inputs[0].op, TensorConcatenate)
self.assertIsInstance(res1.chunks[0].inputs[0].inputs[0].op,
TensorArgmaxCombine)
self.assertIsInstance(res2.chunks[0].inputs[0].inputs[0].op,
TensorArgminCombine)
res1 = argmax(ones((10, 8), chunk_size=3), axis=1, keepdims=True)
res2 = argmin(ones((10, 8), chunk_size=3), axis=1, keepdims=True)
self.assertEqual(res1.shape, (10, 1))
self.assertEqual(res2.shape, (10, 1))
self.assertIsInstance(res1.chunks[0].op, TensorArgmax)
self.assertIsInstance(res2.chunks[0].op, TensorArgmin)
self.assertIsInstance(res1.chunks[0].inputs[0].op, TensorConcatenate)
self.assertIsInstance(res2.chunks[0].inputs[0].op, TensorConcatenate)
self.assertIsInstance(res1.chunks[0].inputs[0].inputs[0].op,
TensorArgmaxChunk)
self.assertIsInstance(res2.chunks[0].inputs[0].inputs[0].op,
TensorArgminChunk)
self.assertRaises(
TypeError,
lambda: argmax(ones((10, 8, 10), chunk_size=3), axis=(0, 1)))
self.assertRaises(
TypeError,
lambda: argmin(ones((10, 8, 10), chunk_size=3), axis=(0, 1)))
self.assertRaises(
np.AxisError,
lambda: argmin(ones((10, 8, 10), chunk_size=3), axis=3))
self.assertRaises(
np.AxisError,
lambda: argmin(ones((10, 8, 10), chunk_size=3), axis=-4))
def testSliceTiles(self):
t = ones((100, 200, 300), chunks=30)
t2 = t[10:40, 199:, -30:303]
t2.tiles()
self.assertEqual(t2.chunk_shape, (2, 1, 1))
self.assertEqual(t2.chunks[0].inputs[0], t.cix[0, -1, -1].data)
self.assertEqual(
t2.chunks[0].op.indexes,
[slice(10, 30, 1), slice(19, 20, 1),
slice(None)])
self.assertEqual(t2.chunks[0].index, (0, 0, 0))
self.assertEqual(t2.chunks[1].inputs[0], t.cix[1, -1, -1].data)
self.assertEqual(
t2.chunks[1].op.indexes,
[slice(0, 10, 1), slice(19, 20, 1),
slice(None)])
self.assertEqual(t2.chunks[1].index, (1, 0, 0))
def testRealFFT(self):
t = ones((10, 20, 30), chunks=(3, 20, 30))
t1 = fft.rfft(t)
self.assertEqual(t1.shape, np.fft.rfft(np.ones(t.shape)).shape)
t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
t = ones((10, 20, 30), chunks=(3, 20, 30))
t1 = fft.irfft(t)
self.assertEqual(t1.shape, np.fft.irfft(np.ones(t.shape)).shape)
t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
t = ones((10, 20, 30), chunks=(3, 20, 30))
t1 = fft.rfft2(t, s=(23, 21))
self.assertEqual(t1.shape,
np.fft.rfft2(np.ones(t.shape), s=(23, 21)).shape)
t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
t = ones((10, 20, 30), chunks=(3, 20, 30))
t1 = fft.irfft2(t, s=(11, 9), axes=(1, 2))
self.assertEqual(
t1.shape,
np.fft.irfft2(np.ones(t.shape), s=(11, 9), axes=(1, 2)).shape)
t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
t = ones((10, 20, 30), chunks=(3, 20, 30))
t1 = fft.rfftn(t, s=(11, 30), axes=(1, 2))
self.assertEqual(
t1.shape,
np.fft.rfftn(np.ones(t.shape), s=(11, 30), axes=(1, 2)).shape)
t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
t = ones((10, 20, 30), chunks=(3, 20, 30))
t1 = fft.irfftn(t, s=(11, 9), axes=(1, 2))
self.assertEqual(
t1.shape,
np.fft.irfftn(np.ones(t.shape), s=(11, 9), axes=(1, 2)).shape)
t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
def testComputeRechunk(self):
tensor = ones((12, 8), chunk_size=((4, 4, 3, 1), (3, 3, 2)))
tensor.tiles()
new_tensor = compute_rechunk(tensor, ((9, 2, 1), (2, 1, 4, 1)))
self.assertEqual(len(new_tensor.chunks), 12)
self.assertEqual(len(new_tensor.chunks[0].inputs), 3)
self.assertIsInstance(new_tensor.chunks[0].inputs[0].op, TensorSlice)
self.assertIs(new_tensor.chunks[0].inputs[0].inputs[0], tensor.chunks[0].data)
self.assertEqual(new_tensor.chunks[0].inputs[0].op.slices,
(slice(None, None, None), slice(None, 2, None)))
self.assertIs(new_tensor.chunks[0].inputs[1].inputs[0], tensor.chunks[3].data)
self.assertEqual(new_tensor.chunks[0].inputs[1].op.slices,
(slice(None, None, None), slice(None, 2, None)))
self.assertIs(new_tensor.chunks[0].inputs[2].inputs[0], tensor.chunks[6].data)
self.assertEqual(new_tensor.chunks[0].inputs[2].op.slices,
(slice(None, 1, None), slice(None, 2, None)))
self.assertIs(new_tensor.chunks[-1].inputs[0], tensor.chunks[-1].data)
self.assertEqual(new_tensor.chunks[-1].op.slices,
(slice(None, None, None), slice(1, None, None)))
def testCreateView(self):
arr = ones((10, 20, 30), chunk_size=[4, 3, 5])
arr2 = transpose(arr)
self.assertTrue(arr2.op.create_view)
arr3 = transpose(arr)
self.assertTrue(arr3.op.create_view)
arr4 = arr.swapaxes(0, 1)
self.assertTrue(arr4.op.create_view)
arr5 = moveaxis(arr, 1, 0)
self.assertTrue(arr5.op.create_view)
arr6 = atleast_1d(1)
self.assertTrue(arr6.op.create_view)
arr7 = atleast_2d([1, 1])
self.assertTrue(arr7.op.create_view)
arr8 = atleast_3d([1, 1])
self.assertTrue(arr8.op.create_view)
arr9 = arr[:3, [1, 2, 3]]
# no view cuz of fancy indexing
self.assertFalse(arr9.op.create_view)
arr10 = arr[:3, None, :5]
self.assertTrue(arr10.op.create_view)
data = np.array([[[0], [1], [2]]])
x = tensor(data)
t = squeeze(x)
self.assertTrue(t.op.create_view)
y = x.reshape(3)
self.assertTrue(y.op.create_view)
def testFrexp(self):
t1 = ones((3, 4, 5), chunk_size=2)
t2 = empty((3, 4, 5), dtype=np.float_, chunk_size=2)
op_type = type(t1.op)
o1, o2 = frexp(t1)
self.assertIs(o1.op, o2.op)
self.assertNotEqual(o1.dtype, o2.dtype)
o1, o2 = frexp(t1, t1)
self.assertIs(o1, t1)
self.assertIsNot(o1.inputs[0], t1)
self.assertIsInstance(o1.inputs[0].op, op_type)
self.assertIsNot(o2.inputs[0], t1)
o1, o2 = frexp(t1, t2, where=t1 > 0)
op_type = type(t2.op)
self.assertIs(o1, t2)
self.assertIsNot(o1.inputs[0], t1)
self.assertIsInstance(o1.inputs[0].op, op_type)
self.assertIsNot(o2.inputs[0], t1)
def testMeanReduction(self):
mean = lambda x, *args, **kwargs: x.mean(*args, **kwargs).tiles()
res = mean(ones((10, 8), chunk_size=3))
self.assertEqual(res.shape, ())
self.assertIsNotNone(res.dtype)
self.assertIsInstance(res.chunks[0].op, Mean)
self.assertIsInstance(res.chunks[0].inputs[0].op, Concatenate)
self.assertIsInstance(res.chunks[0].inputs[0].inputs[0].op,
MeanCombine)
res = mean(ones((8, 8), chunk_size=8))
self.assertEqual(res.shape, ())
self.assertEqual(calc_shape(res), res.shape)
res = mean(ones((10, 8), chunk_size=3), axis=0)
self.assertEqual(res.shape, (8, ))
self.assertEqual(calc_shape(res), res.shape)
res = mean(ones((10, 8), chunk_size=3), axis=1)
self.assertEqual(res.shape, (10, ))
self.assertEqual(calc_shape(res), res.shape)
res = mean(ones((10, 8), chunk_size=3), keepdims=True)
self.assertEqual(res.shape, (1, 1))
self.assertEqual(calc_shape(res), res.shape)
res = mean(ones((10, 8), chunk_size=3), axis=0, keepdims=True)
self.assertEqual(res.shape, (1, 8))
self.assertEqual(calc_shape(res), res.shape)
res = mean(ones((10, 8), chunk_size=3), axis=1, keepdims=True)
self.assertEqual(res.shape, (10, 1))
self.assertEqual(calc_shape(res), res.shape)
self.assertIsInstance(res.chunks[0].op, Mean)
self.assertIsInstance(res.chunks[0].inputs[0].op, Concatenate)
self.assertIsInstance(res.chunks[0].inputs[0].inputs[0].op, MeanChunk)
self.assertEqual(calc_shape(res.chunks[0]), res.chunks[0].shape)
def testCopyto(self):
a = ones((10, 20), chunk_size=3)
b = ones(10, chunk_size=4)
with self.assertRaises(ValueError):
copyto(a, b)
tp = type(a.op)
b = ones(20, chunk_size=4)
copyto(a, b)
self.assertIsInstance(a.op, CopyTo)
self.assertIs(a.inputs[0], b.data)
self.assertIsInstance(a.inputs[1].op, tp)
self.assertEqual(calc_shape(a), a.shape)
a.tiles()
self.assertIsInstance(a.chunks[0].op, CopyTo)
self.assertEqual(len(a.chunks[0].inputs), 2)
self.assertEqual(calc_shape(a.chunks[0]), a.chunks[0].shape)
a = ones((10, 20), chunk_size=3, dtype='i4')
b = ones(20, chunk_size=4, dtype='f8')
with self.assertRaises(TypeError):
copyto(a, b)
b = ones(20, chunk_size=4, dtype='i4')
copyto(a, b, where=b > 0)
self.assertIsNotNone(a.op.where)
self.assertEqual(calc_shape(a), a.shape)
a.tiles()
self.assertIsInstance(a.chunks[0].op, CopyTo)
self.assertEqual(len(a.chunks[0].inputs), 3)
self.assertEqual(calc_shape(a.chunks[0]), a.chunks[0].shape)
with self.assertRaises(ValueError):
copyto(a, a, where=np.ones(30, dtype='?'))
def testBaseReduction(self):
sum = lambda x, *args, **kwargs: x.sum(*args, **kwargs).tiles()
prod = lambda x, *args, **kwargs: x.prod(*args, **kwargs).tiles()
max = lambda x, *args, **kwargs: x.max(*args, **kwargs).tiles()
min = lambda x, *args, **kwargs: x.min(*args, **kwargs).tiles()
all = lambda x, *args, **kwargs: x.all(*args, **kwargs).tiles()
any = lambda x, *args, **kwargs: x.any(*args, **kwargs).tiles()
for f in [sum, prod, max, min, all, any]:
res = f(ones((8, 8), chunk_size=8))
self.assertEqual(res.shape, ())
res = f(ones((10, 8), chunk_size=3))
self.assertIsNotNone(res.dtype)
self.assertEqual(res.shape, ())
res = f(ones((10, 8), chunk_size=3), axis=0)
self.assertEqual(res.shape, (8, ))
res = f(ones((10, 8), chunk_size=3), axis=1)
self.assertEqual(res.shape, (10, ))
with self.assertRaises(np.AxisError):
f(ones((10, 8), chunk_size=3), axis=2)
res = f(ones((10, 8), chunk_size=3), axis=-1)
self.assertEqual(res.shape, (10, ))
with self.assertRaises(np.AxisError):
f(ones((10, 8), chunk_size=3), axis=-3)
res = f(ones((10, 8), chunk_size=3), keepdims=True)
self.assertEqual(res.shape, (1, 1))
res = f(ones((10, 8), chunk_size=3), axis=0, keepdims=True)
self.assertEqual(res.shape, (1, 8))
res = f(ones((10, 8), chunk_size=3), axis=1, keepdims=True)
self.assertEqual(res.shape, (10, 1))
res = f(ones((10, 8, 10), chunk_size=3), axis=1)
self.assertEqual(res.shape, (10, 10))
res = f(ones((10, 8, 10), chunk_size=3), axis=1, keepdims=True)
self.assertEqual(res.shape, (10, 1, 10))
res = f(ones((10, 8, 10), chunk_size=3), axis=(0, 2))
self.assertEqual(res.shape, (8, ))
res = f(ones((10, 8, 10), chunk_size=3),
axis=(0, 2),
keepdims=True)
self.assertEqual(res.shape, (1, 8, 1))
def testPrefetch(self):
t1 = ones((10, 8), chunk_size=10)
t2 = ones((1, 8), chunk_size=10)
t3 = t1 + t2
self.executor.execute_tensor(t3)
def testAdd(self):
t1 = ones((3, 4), chunk_size=2)
t2 = ones(4, chunk_size=2)
t3 = t1 + t2
k1 = t3.key
t3.tiles()
self.assertNotEqual(t3.key, k1)
self.assertEqual(t3.shape, (3, 4))
self.assertEqual(len(t3.chunks), 4)
self.assertEqual(calc_shape(t3), t3.shape)
self.assertEqual(t3.chunks[0].inputs,
[t1.chunks[0].data, t2.chunks[0].data])
self.assertEqual(t3.chunks[1].inputs,
[t1.chunks[1].data, t2.chunks[1].data])
self.assertEqual(t3.chunks[2].inputs,
[t1.chunks[2].data, t2.chunks[0].data])
self.assertEqual(t3.chunks[3].inputs,
[t1.chunks[3].data, t2.chunks[1].data])
self.assertEqual(t3.op.dtype, np.dtype('f8'))
self.assertEqual(t3.chunks[0].op.dtype, np.dtype('f8'))
self.assertEqual(calc_shape(t3.chunks[0]), t3.chunks[0].shape)
t4 = t1 + 1
t4.tiles()
self.assertEqual(t4.shape, (3, 4))
self.assertEqual(len(t3.chunks), 4)
self.assertEqual(calc_shape(t4), t4.shape)
self.assertEqual(t4.chunks[0].inputs, [t1.chunks[0].data])
self.assertEqual(t4.chunks[0].op.constant[0], 1)
self.assertEqual(t4.chunks[1].inputs, [t1.chunks[1].data])
self.assertEqual(t4.chunks[1].op.constant[0], 1)
self.assertEqual(t4.chunks[2].inputs, [t1.chunks[2].data])
self.assertEqual(t4.chunks[2].op.constant[0], 1)
self.assertEqual(t4.chunks[3].inputs, [t1.chunks[3].data])
self.assertEqual(t4.chunks[3].op.constant[0], 1)
self.assertEqual(calc_shape(t4.chunks[0]), t4.chunks[0].shape)
# sparse tests
t5 = add([1, 2, 3, 4], 1)
t5.tiles()
self.assertEqual(t4.chunks[0].inputs, [t1.chunks[0].data])
t1 = tensor([[0, 1, 0], [1, 0, 0]], chunk_size=2).tosparse()
t = t1 + 1
self.assertFalse(t.issparse())
self.assertIs(type(t), Tensor)
t.tiles()
self.assertFalse(t.chunks[0].op.sparse)
t = t1 + 0
self.assertTrue(t.issparse())
self.assertIs(type(t), SparseTensor)
t2 = tensor([[1, 0, 0]], chunk_size=2).tosparse()
t = t1 + t2
self.assertTrue(t.issparse())
self.assertIs(type(t), SparseTensor)
t.tiles()
self.assertTrue(t.chunks[0].op.sparse)
t3 = tensor([1, 1, 1], chunk_size=2)
t = t1 + t3
self.assertFalse(t.issparse())
self.assertIs(type(t), Tensor)
t.tiles()
self.assertFalse(t.chunks[0].op.sparse)
def testTensordotExecution(self):
size_executor = Executor(
sync_provider_type=Executor.SyncProviderType.MOCK)
a_data = np.arange(60).reshape(3, 4, 5)
a = tensor(a_data, chunk_size=2)
b_data = np.arange(24).reshape(4, 3, 2)
b = tensor(b_data, chunk_size=2)
axes = ([1, 0], [0, 1])
c = tensordot(a, b, axes=axes)
size_res = size_executor.execute_tensor(c, mock=True)
res = self.executor.execute_tensor(c)
expected = np.tensordot(a_data, b_data, axes=axes)
self.assertEqual(sum(s[0] for s in size_res), c.nbytes)
self.assertTrue(np.array_equal(res[0], expected[:2, :]))
self.assertTrue(np.array_equal(res[1], expected[2:4, :]))
self.assertTrue(np.array_equal(res[2], expected[4:, :]))
a = ones((1000, 2000), chunk_size=500)
b = ones((2000, 100), chunk_size=500)
c = dot(a, b)
res = self.executor.execute_tensor(c)
expected = np.dot(np.ones((1000, 2000)), np.ones((2000, 100)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((10, 8), chunk_size=2)
b = ones((8, 10), chunk_size=2)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertEqual(len(res), 25)
for r in res:
self.assertTrue(np.array_equal(r, np.tile([8], [2, 2])))
a = ones((500, 500), chunk_size=500)
b = ones((500, 100), chunk_size=500)
c = a.dot(b)
res = self.executor.execute_tensor(c)
self.assertTrue(np.array_equal(res[0], np.tile([500], [500, 100])))
raw_a = np.random.random((100, 200, 50))
raw_b = np.random.random((200, 10, 100))
a = tensor(raw_a, chunk_size=50)
b = tensor(raw_b, chunk_size=33)
c = tensordot(a, b, axes=((0, 1), (2, 0)))
res = self.executor.execute_tensor(c, concat=True)
expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes))
self.assertTrue(np.allclose(res[0], expected))
a = ones((1000, 2000), chunk_size=500)
b = ones((100, 2000), chunk_size=500)
c = inner(a, b)
res = self.executor.execute_tensor(c)
expected = np.inner(np.ones((1000, 2000)), np.ones((100, 2000)))
self.assertEqual(len(res), 2)
self.assertTrue(np.array_equal(res[0], expected[:500, :]))
self.assertTrue(np.array_equal(res[1], expected[500:, :]))
a = ones((100, 100), chunk_size=30)
b = ones((100, 100), chunk_size=30)
c = a.dot(b)
res = self.executor.execute_tensor(c, concat=True)[0]
np.testing.assert_array_equal(res, np.ones((100, 100)) * 100)
def testBuildMode(self):
t1 = ones((2, 3), chunk_size=2)
self.assertTrue(t1 == 2)
with build_mode():
self.assertFalse(t1 == 2)
def testMatmul(self):
a_data = [[1, 0], [0, 1]]
b_data = [[4, 1], [2, 2]]
a = tensor(a_data, chunk_size=1)
b = tensor(b_data, chunk_size=1)
t = matmul(a, b)
self.assertEqual(t.shape, (2, 2))
self.assertEqual(calc_shape(t), t.shape)
t.tiles()
self.assertEqual(t.shape, tuple(sum(s) for s in t.nsplits))
self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape)
b_data = [1, 2]
b = tensor(b_data, chunk_size=1)
t = matmul(a, b)
self.assertEqual(t.shape, (2, ))
self.assertEqual(calc_shape(t), t.shape)
t.tiles()
self.assertEqual(t.shape, tuple(sum(s) for s in t.nsplits))
self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape)
t = matmul(b, a)
self.assertEqual(t.shape, (2, ))
self.assertEqual(calc_shape(t), t.shape)
t.tiles()
self.assertEqual(t.shape, tuple(sum(s) for s in t.nsplits))
self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape)
a_data = np.arange(2 * 2 * 4).reshape((2, 2, 4))
b_data = np.arange(2 * 2 * 4).reshape((2, 4, 2))
a = tensor(a_data, chunk_size=1)
b = tensor(b_data, chunk_size=1)
t = matmul(a, b)
self.assertEqual(t.shape, (2, 2, 2))
self.assertEqual(calc_shape(t), t.shape)
t.tiles()
self.assertEqual(t.shape, tuple(sum(s) for s in t.nsplits))
self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape)
t = matmul(tensor([2j, 3j], chunk_size=1),
tensor([2j, 3j], chunk_size=1))
self.assertEqual(t.shape, ())
self.assertEqual(calc_shape(t), t.shape)
t.tiles()
self.assertEqual(t.shape, tuple(sum(s) for s in t.nsplits))
self.assertEqual(calc_shape(t.chunks[0]), t.chunks[0].shape)
with self.assertRaises(ValueError):
matmul([1, 2], 3)
with self.assertRaises(ValueError):
matmul(np.random.randn(2, 3, 4), np.random.randn(3, 4, 3))
t = matmul(tensor(np.random.randn(2, 3, 4), chunk_size=2),
tensor(np.random.randn(3, 1, 4, 3), chunk_size=3))
self.assertEqual(t.shape, (3, 2, 3, 3))
v = ones((100, 100), chunk_size=10)
tv = matmul(v, v)
self.assertEqual(tv.shape, (100, 100))
self.assertEqual(calc_shape(tv), tv.shape)
tv.tiles()
self.assertEqual(tv.shape, tuple(sum(s) for s in tv.nsplits))
self.assertEqual(calc_shape(tv.chunks[0]), tv.chunks[0].shape)
