def testSetItem(self):
shape = (10, 20, 30, 40)
t = ones(shape, chunk_size=5, dtype='i4')
t[5:20:3, 5, ..., :-5] = 2.2
self.assertIsInstance(t.op, TensorIndexSetValue)
self.assertEqual(t.shape, shape)
self.assertIsInstance(t.inputs[0].op.outputs[0].op, TensorOnes)
t = t.tiles()
self.assertIsInstance(t.chunks[0].op, TensorOnes)
self.assertIsInstance(t.cix[1, 1, 0, 0].op, TensorIndexSetValue)
self.assertEqual(t.cix[1, 1, 0, 0].op.value, 2.2)
t2 = ones(shape, chunk_size=5, dtype='i4')
shape = t2[5:20:3, 5, ..., :-5].shape
t2[5:20:3, 5, ..., :-5] = ones(shape, chunk_size=4, dtype='i4') * 2
t2 = t2.tiles()
self.assertIsInstance(t2.chunks[0].op, TensorOnes)
self.assertIsInstance(t2.cix[1, 1, 0, 0].op, TensorIndexSetValue)
self.assertIsInstance(t2.cix[1, 1, 0, 0].op.value.op,
TensorConcatenate)
with self.assertRaises(ValueError):
t[0, 0, 0, 0] = ones(2, chunk_size=10)
def testReshape(self):
a = ones((10, 20, 30), chunk_size=5)
b = a.reshape(10, 600)
b = b.tiles()
self.assertEqual(tuple(sum(s) for s in b.nsplits), (10, 600))
a = ones((10, 600), chunk_size=5)
b = a.reshape(10, 30, 20)
b = b.tiles()
self.assertEqual(tuple(sum(s) for s in b.nsplits), (10, 30, 20))
a = ones((10, 600), chunk_size=5)
a.shape = [10, 30, 20]
a = a.tiles()
self.assertEqual(tuple(sum(s) for s in a.nsplits), (10, 30, 20))
# test reshape unknown shape
c = a[a > 0]
d = c.reshape(10, 600)
self.assertEqual(d.shape, (10, 600))
d = c.reshape(-1, 10)
self.assertEqual(len(d.shape), 2)
self.assertTrue(np.isnan(d.shape[0]))
self.assertTrue(d.shape[1], 10)
with self.assertRaises(TypeError):
a.reshape((10, 30, 20), other_argument=True)
def testHermitianFFT(self):
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = hfft(t)
self.assertEqual(t1.shape, np.fft.hfft(np.ones(t.shape)).shape)
t1 = t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = hfft(t, n=100)
self.assertEqual(t1.shape, np.fft.hfft(np.ones(t.shape), n=100).shape)
t1 = t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = ihfft(t)
self.assertEqual(t1.shape, np.fft.ihfft(np.ones(t.shape)).shape)
t1 = t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = ihfft(t, n=100)
self.assertEqual(t1.shape, np.fft.ihfft(np.ones(t.shape), n=100).shape)
t1 = t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
t1 = ihfft(t, n=101)
self.assertEqual(t1.shape, np.fft.ihfft(np.ones(t.shape), n=101).shape)
t1 = t1.tiles()
self.assertEqual(t1.shape, tuple(sum(ns) for ns in t1.nsplits))
def test_sum_prod_execution(setup):
arr = ones((10, 8), chunk_size=6)
assert 80 == arr.sum().execute().fetch()
np.testing.assert_array_equal(
arr.sum(axis=0).execute().fetch(), np.full((8, ), fill_value=10))
arr = ones((3, 3), chunk_size=2)
assert 512 == (arr * 2).prod().execute().fetch()
np.testing.assert_array_equal((arr * 2).prod(axis=0).execute().fetch(),
np.full((3, ), fill_value=8))
raw = sps.random(10, 20, density=.1)
arr = tensor(raw, chunk_size=3)
res = arr.sum().execute().fetch()
assert pytest.approx(res) == raw.sum()
# test order
raw = np.asfortranarray(np.random.rand(10, 20, 30))
arr = tensor(raw, chunk_size=13)
arr2 = arr.sum(axis=-1)
res = arr2.execute().fetch()
expected = raw.sum(axis=-1)
np.testing.assert_allclose(res, expected)
assert res.flags['C_CONTIGUOUS'] == expected.flags['C_CONTIGUOUS']
assert res.flags['F_CONTIGUOUS'] == expected.flags['F_CONTIGUOUS']
# test string dtype
a = tensor(list('abcdefghi'), dtype=object)
assert a.sum().execute().fetch() == 'abcdefghi'
a = tensor(list('abcdefghi'), dtype=object, chunk_size=2)
assert a.sum().execute().fetch() == 'abcdefghi'
def testBoolIndexingTiles(self):
t = ones((100, 200, 300), chunk_size=30)
indexed = t[t < 2]
indexed = indexed.tiles()
t = get_tiled(t)
self.assertEqual(len(indexed.chunks), 280)
self.assertEqual(indexed.chunks[0].index, (0, ))
self.assertEqual(indexed.chunks[20].index, (20, ))
self.assertIs(indexed.chunks[20].inputs[0], t.cix[(0, 2, 0)].data)
self.assertIs(indexed.chunks[20].inputs[1],
indexed.op.indexes[0].cix[0, 2, 0].data)
t2 = ones((100, 200), chunk_size=30)
indexed2 = t[t2 < 2]
indexed2 = indexed2.tiles()
t = get_tiled(t)
self.assertEqual(len(indexed2.chunks), 280)
self.assertEqual(len(indexed2.chunks[0].shape), 2)
self.assertTrue(np.isnan(indexed2.chunks[0].shape[0]))
self.assertEqual(indexed2.chunks[0].shape[1], 30)
self.assertEqual(indexed2.chunks[20].inputs[0], t.cix[(0, 2, 0)].data)
self.assertEqual(indexed2.chunks[20].inputs[1],
indexed2.op.indexes[0].cix[0, 2].data)
def testMixedIndexingTiles(self):
t = ones((100, 200, 300, 400), chunk_size=24)
cmp = ones(400, chunk_size=24) < 2
t2 = t[10:90:3, 5, ..., None, cmp]
t2 = t2.tiles()
cmp = get_tiled(cmp)
self.assertEqual(t2.shape[:-1], (27, 300, 1))
self.assertTrue(np.isnan(t2.shape[-1]))
self.assertEqual(t2.chunk_shape, (4, 13, 1, 17))
self.assertEqual(
t2.chunks[0].op.indexes,
[slice(10, 24, 3), 5,
slice(None), None, cmp.cix[0, ].data])
# multiple tensor type as indexes
t3 = ones((100, 200, 300, 400), chunk_size=24)
cmp2 = ones((100, 200), chunk_size=24) < 2
cmp3 = ones(400, chunk_size=24) < 2
t4 = t3[cmp2, 5, None, cmp3]
t4 = t4.tiles()
cmp2, cmp3 = get_tiled(cmp2), get_tiled(cmp3)
self.assertEqual(t4.shape[1], 1)
self.assertTrue(np.isnan(t4.shape[0]))
self.assertTrue(np.isnan(t4.shape[-1]))
self.assertEqual(t4.chunk_shape, (45, 1, 17))
self.assertEqual(t4.chunks[0].op.indexes,
[cmp2.cix[0, 0].data, 5, None, cmp3.cix[0, ].data])
def testSumProdExecution(self):
arr = ones((10, 8), chunk_size=3)
self.assertEqual([80], self.executor.execute_tensor(arr.sum()))
self.assertEqual((10,) * 8,
tuple(np.concatenate(self.executor.execute_tensor(arr.sum(axis=0)))))
arr = ones((3, 3), chunk_size=2)
self.assertEqual([512], self.executor.execute_tensor((arr * 2).prod()))
self.assertEqual((8,) * 3,
tuple(np.concatenate(self.executor.execute_tensor((arr * 2).prod(axis=0)))))
raw = sps.random(10, 20, density=.1)
arr = tensor(raw, chunk_size=3)
res = self.executor.execute_tensor(arr.sum())[0]
self.assertAlmostEqual(res, raw.sum())
# test order
raw = np.asfortranarray(np.random.rand(10, 20, 30))
arr = tensor(raw, chunk_size=13)
arr2 = arr.sum(axis=-1)
res = self.executor.execute_tensor(arr2, concat=True)[0]
expected = raw.sum(axis=-1)
np.testing.assert_allclose(res, expected)
self.assertEqual(res.flags['C_CONTIGUOUS'], expected.flags['C_CONTIGUOUS'])
self.assertEqual(res.flags['F_CONTIGUOUS'], expected.flags['F_CONTIGUOUS'])
def test_reshape():
a = ones((10, 20, 30), chunk_size=5)
b = a.reshape(10, 600)
b = tile(b)
assert tuple(sum(s) for s in b.nsplits) == (10, 600)
a = ones((10, 600), chunk_size=5)
b = a.reshape(10, 30, 20)
b = tile(b)
assert tuple(sum(s) for s in b.nsplits) == (10, 30, 20)
a = ones((10, 600), chunk_size=5)
a.shape = [10, 30, 20]
a = tile(a)
assert tuple(sum(s) for s in a.nsplits) == (10, 30, 20)
# test reshape unknown shape
c = a[a > 0]
d = c.reshape(10, 600)
assert d.shape == (10, 600)
d = c.reshape(-1, 10)
assert len(d.shape) == 2
assert np.isnan(d.shape[0])
assert d.shape[1]
with pytest.raises(TypeError):
a.reshape((10, 30, 20), other_argument=True)
def test_add_with_out():
t1 = ones((3, 4), chunk_size=2)
t2 = ones(4, chunk_size=2)
t3 = add(t1, t2, out=t1)
assert isinstance(t1.op, TensorAdd)
assert t1.op.out.key == t1.op.lhs.key
assert t3 is t1
assert t3.shape == (3, 4)
assert t3.op.lhs.extra_params.raw_chunk_size == 2
assert t3.op.rhs is t2.data
assert t3.key != t3.op.lhs.key
t1, t3 = tile(t1, t3)
assert isinstance(t1.chunks[0].op, TensorAdd)
assert t1.chunks[0].op.out.key == t1.chunks[0].op.lhs.key
with pytest.raises(TypeError):
add(t1, t2, out=1)
with pytest.raises(ValueError):
add(t1, t2, out=t2)
with pytest.raises(TypeError):
truediv(t1, t2, out=t1.astype('i8'))
t1 = ones((3, 4), chunk_size=2, dtype=float)
t2 = ones(4, chunk_size=2, dtype=int)
t3 = add(t2, 1, out=t1)
assert t3.shape == (3, 4)
assert t3.dtype == np.float64
def testAddWithOut(self):
t1 = ones((3, 4), chunk_size=2)
t2 = ones(4, chunk_size=2)
t3 = add(t1, t2, out=t1)
self.assertIsInstance(t1.op, TensorAdd)
self.assertEqual(t1.op.out.key, t1.op.lhs.key)
self.assertIs(t3, t1)
self.assertEqual(t3.shape, (3, 4))
self.assertEqual(t3.op.lhs.extra_params.raw_chunk_size, 2)
self.assertIs(t3.op.rhs, t2.data)
self.assertNotEqual(t3.key, t3.op.lhs.key)
t3.tiles()
t1 = get_tiled(t1)
self.assertIsInstance(t1.chunks[0].op, TensorAdd)
self.assertEqual(t1.chunks[0].op.out.key, t1.chunks[0].op.lhs.key)
with self.assertRaises(TypeError):
add(t1, t2, out=1)
with self.assertRaises(ValueError):
add(t1, t2, out=t2)
with self.assertRaises(TypeError):
truediv(t1, t2, out=t1.astype('i8'))
t1 = ones((3, 4), chunk_size=2, dtype=float)
t2 = ones(4, chunk_size=2, dtype=int)
t3 = add(t2, 1, out=t1)
self.assertEqual(t3.shape, (3, 4))
self.assertEqual(t3.dtype, np.float64)
def testConcatenate(self):
a = ones((10, 20, 30), chunk_size=10)
b = ones((20, 20, 30), chunk_size=20)
c = concatenate([a, b])
self.assertEqual(c.shape, (30, 20, 30))
a = ones((10, 20, 30), chunk_size=10)
b = ones((10, 20, 40), chunk_size=20)
c = concatenate([a, b], axis=-1)
self.assertEqual(c.shape, (10, 20, 70))
with self.assertRaises(ValueError):
a = ones((10, 20, 30), chunk_size=10)
b = ones((20, 30, 30), chunk_size=20)
concatenate([a, b])
with self.assertRaises(ValueError):
a = ones((10, 20, 30), chunk_size=10)
b = ones((20, 20), chunk_size=20)
concatenate([a, b])
a = ones((10, 20, 30), chunk_size=5)
b = ones((20, 20, 30), chunk_size=10)
c = concatenate([a, b]).tiles()
self.assertEqual(c.chunk_shape[0], 4)
self.assertEqual(c.chunk_shape[1], 4)
self.assertEqual(c.chunk_shape[2], 6)
self.assertEqual(c.nsplits, ((5, 5, 10, 10), (5, ) * 4, (5, ) * 6))
self.assertEqual(c.cix[0, 0, 0].key, a.cix[0, 0, 0].key)
self.assertEqual(c.cix[1, 0, 0].key, a.cix[1, 0, 0].key)
def test_broadcast_to():
arr = ones((10, 5), chunk_size=2)
arr2 = broadcast_to(arr, (20, 10, 5))
arr, arr2 = tile(arr, arr2)
assert arr2.shape == (20, 10, 5)
assert len(arr2.chunks) == len(arr.chunks)
assert arr2.chunks[0].shape == (20, 2, 2)
arr = ones((10, 5, 1), chunk_size=2)
arr3 = broadcast_to(arr, (5, 10, 5, 6))
arr, arr3 = tile(arr, arr3)
assert arr3.shape == (5, 10, 5, 6)
assert len(arr3.chunks) == len(arr.chunks)
assert arr3.nsplits == ((5,), (2, 2, 2, 2, 2), (2, 2, 1), (6,))
assert arr3.chunks[0].shape == (5, 2, 2, 6)
arr = ones((10, 1), chunk_size=2)
arr4 = broadcast_to(arr, (20, 10, 5))
arr, arr4 = tile(arr, arr4)
assert arr4.shape == (20, 10, 5)
assert len(arr4.chunks) == len(arr.chunks)
assert arr4.chunks[0].shape == (20, 2, 5)
with pytest.raises(ValueError):
broadcast_to(arr, (10,))
with pytest.raises(ValueError):
broadcast_to(arr, (5, 1))
arr = ones((4, 5), chunk_size=2)
with pytest.raises((ValueError)):
broadcast_to(arr[arr < 2], (3, 20))
def testCompare(self):
t1 = ones(4, chunk_size=2) * 2
t2 = ones(4, chunk_size=2)
t3 = t1 > t2
t3 = t3.tiles()
self.assertEqual(len(t3.chunks), 2)
self.assertIsInstance(t3.op, TensorGreaterThan)
def test_concatenate():
a = ones((10, 20, 30), chunk_size=10)
b = ones((20, 20, 30), chunk_size=20)
c = concatenate([a, b])
assert c.shape == (30, 20, 30)
a = ones((10, 20, 30), chunk_size=10)
b = ones((10, 20, 40), chunk_size=20)
c = concatenate([a, b], axis=-1)
assert c.shape == (10, 20, 70)
with pytest.raises(ValueError):
a = ones((10, 20, 30), chunk_size=10)
b = ones((20, 30, 30), chunk_size=20)
concatenate([a, b])
with pytest.raises(ValueError):
a = ones((10, 20, 30), chunk_size=10)
b = ones((20, 20), chunk_size=20)
concatenate([a, b])
a = ones((10, 20, 30), chunk_size=5)
b = ones((20, 20, 30), chunk_size=10)
a, c = tile(a, concatenate([a, b]))
assert c.chunk_shape[0] == 4
assert c.chunk_shape[1] == 4
assert c.chunk_shape[2] == 6
assert c.nsplits == ((5, 5, 10, 10), (5,) * 4, (5,) * 6)
assert c.cix[0, 0, 0].key == a.cix[0, 0, 0].key
assert c.cix[1, 0, 0].key == a.cix[1, 0, 0].key
def test_array_equal(setup):
a = ones((10, 5), chunk_size=4)
b = ones((10, 5), chunk_size=5)
c = array_equal(a, b)
assert c.execute().fetch()
def test_hermitian_fft():
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = hfft(t)
assert t1.shape == np.fft.hfft(np.ones(t.shape)).shape
t1 = tile(t1)
assert t1.shape == tuple(sum(ns) for ns in t1.nsplits)
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = hfft(t, n=100)
assert t1.shape == np.fft.hfft(np.ones(t.shape), n=100).shape
t1 = tile(t1)
assert t1.shape == tuple(sum(ns) for ns in t1.nsplits)
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = ihfft(t)
assert t1.shape == np.fft.ihfft(np.ones(t.shape)).shape
t1 = tile(t1)
assert t1.shape == tuple(sum(ns) for ns in t1.nsplits)
t = ones((10, 20, 30), chunk_size=(3, 20, 30))
t1 = ihfft(t, n=100)
assert t1.shape == np.fft.ihfft(np.ones(t.shape), n=100).shape
t1 = tile(t1)
assert t1.shape == tuple(sum(ns) for ns in t1.nsplits)
t1 = ihfft(t, n=101)
assert t1.shape == np.fft.ihfft(np.ones(t.shape), n=101).shape
t1 = tile(t1)
assert t1.shape == tuple(sum(ns) for ns in t1.nsplits)
def test_compare():
t1 = ones(4, chunk_size=2) * 2
t2 = ones(4, chunk_size=2)
t3 = t1 > t2
t3 = tile(t3)
assert len(t3.chunks) == 2
assert isinstance(t3.op, TensorGreaterThan)
def testBroadcastTo(self):
arr = ones((10, 5), chunk_size=2)
arr2 = broadcast_to(arr, (20, 10, 5))
arr2.tiles()
self.assertEqual(arr2.shape, (20, 10, 5))
self.assertEqual(len(arr2.chunks), len(arr.chunks))
self.assertEqual(arr2.chunks[0].shape, (20, 2, 2))
arr = ones((10, 5, 1), chunk_size=2)
arr3 = broadcast_to(arr, (5, 10, 5, 6))
arr3.tiles()
self.assertEqual(arr3.shape, (5, 10, 5, 6))
self.assertEqual(len(arr3.chunks), len(arr.chunks))
self.assertEqual(arr3.nsplits,
((5, ), (2, 2, 2, 2, 2), (2, 2, 1), (6, )))
self.assertEqual(arr3.chunks[0].shape, (5, 2, 2, 6))
arr = ones((10, 1), chunk_size=2)
arr4 = broadcast_to(arr, (20, 10, 5))
arr4.tiles()
self.assertEqual(arr4.shape, (20, 10, 5))
self.assertEqual(len(arr4.chunks), len(arr.chunks))
self.assertEqual(arr4.chunks[0].shape, (20, 2, 5))
with self.assertRaises(ValueError):
broadcast_to(arr, (10, ))
with self.assertRaises(ValueError):
broadcast_to(arr, (5, 1))
def testBoolIndexing(self):
t = ones((100, 200, 300))
indexed = t[t < 2]
self.assertEqual(len(indexed.shape), 1)
self.assertTrue(np.isnan(indexed.shape[0]))
t2 = ones((100, 200))
indexed = t[t2 < 2]
self.assertEqual(len(indexed.shape), 2)
self.assertTrue(np.isnan(indexed.shape[0]))
self.assertEqual(indexed.shape[1], 300)
t2 = ones((100, 200))
indexed = t[t2 < 2] + 1
self.assertEqual(len(indexed.shape), 2)
self.assertTrue(np.isnan(indexed.shape[0]))
self.assertEqual(indexed.shape[1], 300)
t3 = ones((101, 200))
with self.assertRaises(IndexError) as cm:
_ = t[t3 < 2] # noqa: F841
e = str(cm.exception)
self.assertIn('along dimension 0', str(e))
self.assertIn(
'dimension is 100 but corresponding boolean dimension is 101',
str(e))
t4 = ones((100, 201))
with self.assertRaises(IndexError) as cm:
_ = t[t4 < 2] # noqa: F841
e = str(cm.exception)
self.assertIn('along dimension 1', str(e))
self.assertIn(
'dimension is 200 but corresponding boolean dimension is 201',
str(e))
def testArrayEqual(self):
a = ones((10, 5), chunk_size=1)
b = ones((10, 5), chunk_size=2)
c = array_equal(a, b)
res = bool(self.executor.execute_tensor(c)[0])
self.assertTrue(res)
def testVarReduction(self):
var = lambda x, *args, **kwargs: x.var(*args, **kwargs).tiles()
res1 = var(ones((10, 8), chunk_size=3), ddof=2)
self.assertEqual(res1.shape, ())
self.assertEqual(res1.op.ddof, 2)
res1 = var(ones((10, 8, 8), chunk_size=3), axis=1)
self.assertEqual(res1.shape, (10, 8))
def test_var_reduction():
var = lambda x, *args, **kwargs: tile(x.var(*args, **kwargs))
res1 = var(ones((10, 8), chunk_size=3), ddof=2)
assert res1.shape == ()
assert res1.op.ddof == 2
res1 = var(ones((10, 8, 8), chunk_size=3), axis=1)
assert res1.shape == (10, 8)
def testMixedIndexing(self):
t = ones((100, 200, 300, 400))
with self.assertRaises(IndexError):
_ = t[ones((100, 200), dtype=float)] # noqa: F841
t2 = t[ones(100) < 2, ..., 20::101, 2]
self.assertEqual(len(t2.shape), 3)
self.assertTrue(np.isnan(t2.shape[0]))
def test_tensordot_execution(setup):
rs = np.random.RandomState(0)
# size_executor = ExecutorForTest(sync_provider_type=ExecutorForTest.SyncProviderType.MOCK)
#
# a_data = np.arange(60).reshape(3, 4, 5)
# a = tensor(a_data, chunk_size=2)
# b_data = np.arange(24).reshape(4, 3, 2)
# b = tensor(b_data, chunk_size=2)
#
# axes = ([1, 0], [0, 1])
# c = tensordot(a, b, axes=axes)
# size_res = size_executor.execute_tensor(c, mock=True)
# assert sum(s[0] for s in size_res) == c.nbytes
# assert sum(s[1] for s in size_res) == c.nbytes
a = ones((100, 200), chunk_size=50)
b = ones((200, 10), chunk_size=50)
c = dot(a, b)
res = c.execute().fetch()
expected = np.dot(np.ones((100, 200)), np.ones((200, 10)))
np.testing.assert_array_equal(res, expected)
a = ones((10, 8), chunk_size=4)
b = ones((8, 10), chunk_size=4)
c = a.dot(b)
res = c.execute().fetch()
np.testing.assert_array_equal(res, np.tile([8], [10, 10]))
a = ones((500, 500), chunk_size=500)
b = ones((500, 100), chunk_size=500)
c = a.dot(b)
res = c.execute().fetch()
np.testing.assert_array_equal(res, np.tile([500], [500, 100]))
raw_a = rs.random((100, 200, 50))
raw_b = rs.random((200, 10, 100))
a = tensor(raw_a, chunk_size=50)
b = tensor(raw_b, chunk_size=33)
c = tensordot(a, b, axes=((0, 1), (2, 0)))
res = c.execute().fetch()
expected = np.tensordot(raw_a, raw_b, axes=(c.op.a_axes, c.op.b_axes))
np.testing.assert_array_almost_equal(res, expected)
a = ones((100, 200), chunk_size=50)
b = ones((10, 200), chunk_size=50)
c = inner(a, b)
res = c.execute().fetch()
expected = np.inner(np.ones((100, 200)), np.ones((10, 200)))
np.testing.assert_array_equal(res, expected)
a = ones((100, 100), chunk_size=30)
b = ones((100, 100), chunk_size=30)
c = a.dot(b)
res = c.execute().fetch()
np.testing.assert_array_equal(res, np.ones((100, 100)) * 100)
def test_mixed_indexing_tiles():
t = ones((100, 200, 300, 400), chunk_size=24)
cmp = ones(400, chunk_size=24) < 2
t2 = t[10:90:3, 5, ..., None, cmp]
t2, cmp = tile(t2, cmp)
assert t2.shape[:-1] == (27, 300, 1)
assert np.isnan(t2.shape[-1])
assert t2.chunk_shape == (4, 13, 1, 17)
assert t2.chunks[0].op.indexes == [slice(10, 24, 3), 5, slice(None), None, cmp.cix[0, ].data]
def testUnifyChunkAdd(self):
t1 = ones(4, chunk_size=2)
t2 = ones(1, chunk_size=1)
t3 = t1 + t2
t3.tiles()
self.assertEqual(len(t3.chunks), 2)
self.assertEqual(t3.chunks[0].inputs[0], t1.chunks[0].data)
self.assertEqual(t3.chunks[0].inputs[1], t2.chunks[0].data)
self.assertEqual(t3.chunks[1].inputs[0], t1.chunks[1].data)
self.assertEqual(t3.chunks[1].inputs[1], t2.chunks[0].data)
def testElementwise(self):
t1 = ones((10000, 5000), chunk_size=500, gpu=True)
t2 = ones(5000, chunk_size=500, gpu=True)
t = (t1 - t2) / sqrt(t2 * (1 - t2) * len(t2))
g = t.build_graph(tiled=True)
RuntimeOptimizer(g, self.executor._engine).optimize([], False)
self.assertTrue(any(n.op.__class__.__name__ == 'TensorCpFuseChunk' for n in g))
c = next(n for n in g if n.op.__class__.__name__ == 'TensorCpFuseChunk')
self.assertGreater(len(_evaluate(c)), 1)
def testElementwise(self):
t1 = ones((10000, 5000), chunk_size=500, gpu=True)
t2 = ones(5000, chunk_size=500, gpu=True)
t = (t1 - t2) / sqrt(t2 * (1 - t2) * len(t2))
g = t.build_graph(tiled=True)
graph = self.executor._preprocess(g, [])
self.assertTrue(any(n.op.__class__.__name__ == 'TensorCpFuseChunk' for n in graph))
c = next(n for n in graph if n.op.__class__.__name__ == 'TensorCpFuseChunk')
print(_evaluate(c))
def testAtleast3dExecution(self):
x = 1
y = ones(3, chunk_size=2)
z = ones((3, 4), chunk_size=2)
t = atleast_3d(x, y, z)
res = [self.executor.execute_tensor(i, concat=True)[0] for i in t]
self.assertTrue(np.array_equal(res[0], np.atleast_3d(x)))
self.assertTrue(np.array_equal(res[1], np.atleast_3d(np.ones(3))))
self.assertTrue(np.array_equal(res[2], np.atleast_3d(np.ones((3, 4)))))
def test_unify_chunk_add():
t1 = ones(4, chunk_size=2)
t2 = ones(1, chunk_size=1)
t3 = t1 + t2
t1, t2, t3 = tile(t1, t2, t3)
assert len(t3.chunks) == 2
assert t3.chunks[0].inputs[0] == t1.chunks[0].data
assert t3.chunks[0].inputs[1] == t2.chunks[0].data
assert t3.chunks[1].inputs[0] == t1.chunks[1].data
assert t3.chunks[1].inputs[1] == t2.chunks[0].data
