def testAddOrder(self):
raw_a = np.random.rand(4, 2)
raw_b = np.asfortranarray(np.random.rand(4, 2))
t1 = tensor(raw_a)
t2 = tensor(raw_b)
out = tensor(raw_b)
# C + scalar
self.assertEqual((t1 + 1).flags['C_CONTIGUOUS'], (raw_a + 1).flags['C_CONTIGUOUS'])
self.assertEqual((t1 + 1).flags['F_CONTIGUOUS'], (raw_a + 1).flags['F_CONTIGUOUS'])
# C + C
self.assertEqual((t1 + t1).flags['C_CONTIGUOUS'], (raw_a + raw_a).flags['C_CONTIGUOUS'])
self.assertEqual((t1 + t1).flags['F_CONTIGUOUS'], (raw_a + raw_a).flags['F_CONTIGUOUS'])
# F + scalar
self.assertEqual((t2 + 1).flags['C_CONTIGUOUS'], (raw_b + 1).flags['C_CONTIGUOUS'])
self.assertEqual((t2 + 1).flags['F_CONTIGUOUS'], (raw_b + 1).flags['F_CONTIGUOUS'])
# F + F
self.assertEqual((t2 + t2).flags['C_CONTIGUOUS'], (raw_b + raw_b).flags['C_CONTIGUOUS'])
self.assertEqual((t2 + t2).flags['F_CONTIGUOUS'], (raw_b + raw_b).flags['F_CONTIGUOUS'])
# C + F
self.assertEqual((t1 + t2).flags['C_CONTIGUOUS'], (raw_a + raw_b).flags['C_CONTIGUOUS'])
self.assertEqual((t1 + t2).flags['F_CONTIGUOUS'], (raw_a + raw_b).flags['F_CONTIGUOUS'])
# C + C + out
self.assertEqual(add(t1, t1, out=out).flags['C_CONTIGUOUS'],
np.add(raw_a, raw_a, out=np.empty((4, 2), order='F')).flags['C_CONTIGUOUS'])
self.assertEqual(add(t1, t1, out=out).flags['F_CONTIGUOUS'],
np.add(raw_a, raw_a, out=np.empty((4, 2), order='F')).flags['F_CONTIGUOUS'])
with self.assertRaises(TypeError):
add(t1, 1, order='B')
def testDatatimeArith(self):
t1 = array([np.datetime64('2005-02-02'), np.datetime64('2005-02-03')])
t2 = t1 + np.timedelta64(1)
self.assertIsInstance(t2.op, TensorAdd)
t3 = t1 - np.datetime64('2005-02-02')
self.assertIsInstance(t3.op, TensorSubtract)
self.assertEqual(t3.dtype,
(np.array(['2005-02-02', '2005-02-03'], dtype=np.datetime64) -
np.datetime64('2005-02-02')).dtype)
t1 = array([np.datetime64('2005-02-02'), np.datetime64('2005-02-03')])
subtract(t1, np.datetime64('2005-02-02'), out=empty(t1.shape, dtype=t3.dtype))
t1 = array([np.datetime64('2005-02-02'), np.datetime64('2005-02-03')])
add(t1, np.timedelta64(1, 'D'), out=t1)
def test_datatime_arith():
t1 = array([np.datetime64('2005-02-02'), np.datetime64('2005-02-03')])
t2 = t1 + np.timedelta64(1)
assert isinstance(t2.op, TensorAdd)
t3 = t1 - np.datetime64('2005-02-02')
assert isinstance(t3.op, TensorSubtract)
assert t3.dtype == (
np.array(['2005-02-02', '2005-02-03'], dtype=np.datetime64) -
np.datetime64('2005-02-02')).dtype
t1 = array([np.datetime64('2005-02-02'), np.datetime64('2005-02-03')])
subtract(t1,
np.datetime64('2005-02-02'),
out=empty(t1.shape, dtype=t3.dtype))
t1 = array([np.datetime64('2005-02-02'), np.datetime64('2005-02-03')])
add(t1, np.timedelta64(1, 'D'), out=t1)
def testAddWithOut(self):
t1 = ones((3, 4), chunk_size=2)
t2 = ones(4, chunk_size=2)
t3 = add(t1, t2, out=t1)
self.assertIsInstance(t1.op, TensorAdd)
self.assertEqual(t1.op.out.key, t1.op.lhs.key)
self.assertIs(t3, t1)
self.assertEqual(t3.shape, (3, 4))
self.assertEqual(t3.op.lhs.extra_params.raw_chunk_size, 2)
self.assertIs(t3.op.rhs, t2.data)
self.assertNotEqual(t3.key, t3.op.lhs.key)
t3.tiles()
self.assertIsInstance(t1.chunks[0].op, TensorAdd)
self.assertEqual(t1.chunks[0].op.out.key, t1.chunks[0].op.lhs.key)
with self.assertRaises(TypeError):
add(t1, t2, out=1)
with self.assertRaises(ValueError):
add(t1, t2, out=t2)
with self.assertRaises(TypeError):
truediv(t1, t2, out=t1.astype('i8'))
t1 = ones((3, 4), chunk_size=2, dtype=float)
t2 = ones(4, chunk_size=2, dtype=int)
t3 = add(t2, 1, out=t1)
self.assertEqual(t3.shape, (3, 4))
self.assertEqual(t3.dtype, np.float64)
def test_add_with_out():
t1 = ones((3, 4), chunk_size=2)
t2 = ones(4, chunk_size=2)
t3 = add(t1, t2, out=t1)
assert isinstance(t1.op, TensorAdd)
assert t1.op.out.key == t1.op.lhs.key
assert t3 is t1
assert t3.shape == (3, 4)
assert t3.op.lhs.extra_params.raw_chunk_size == 2
assert t3.op.rhs is t2.data
assert t3.key != t3.op.lhs.key
t1, t3 = tile(t1, t3)
assert isinstance(t1.chunks[0].op, TensorAdd)
assert t1.chunks[0].op.out.key == t1.chunks[0].op.lhs.key
with pytest.raises(TypeError):
add(t1, t2, out=1)
with pytest.raises(ValueError):
add(t1, t2, out=t2)
with pytest.raises(TypeError):
truediv(t1, t2, out=t1.astype('i8'))
t1 = ones((3, 4), chunk_size=2, dtype=float)
t2 = ones(4, chunk_size=2, dtype=int)
t3 = add(t2, 1, out=t1)
assert t3.shape == (3, 4)
assert t3.dtype == np.float64
def testBaseOrderExecution(self):
raw = np.asfortranarray(np.random.rand(5, 6))
arr = tensor(raw, chunk_size=3)
res = self.executor.execute_tensor(arr + 1, concat=True)[0]
np.testing.assert_array_equal(res, raw + 1)
self.assertFalse(res.flags['C_CONTIGUOUS'])
self.assertTrue(res.flags['F_CONTIGUOUS'])
res2 = self.executor.execute_tensor(add(arr, 1, order='C'), concat=True)[0]
np.testing.assert_array_equal(res2, np.add(raw, 1, order='C'))
self.assertTrue(res2.flags['C_CONTIGUOUS'])
self.assertFalse(res2.flags['F_CONTIGUOUS'])
def test_base_order_execution(setup):
raw = np.asfortranarray(np.random.rand(5, 6))
arr = tensor(raw, chunk_size=3)
res = (arr + 1).execute().fetch()
np.testing.assert_array_equal(res, raw + 1)
assert res.flags['C_CONTIGUOUS'] is False
assert res.flags['F_CONTIGUOUS'] is True
res2 = add(arr, 1, order='C').execute().fetch()
np.testing.assert_array_equal(res2, np.add(raw, 1, order='C'))
assert res2.flags['C_CONTIGUOUS'] is True
assert res2.flags['F_CONTIGUOUS'] is False
def test_add_order():
raw_a = np.random.rand(4, 2)
raw_b = np.asfortranarray(np.random.rand(4, 2))
t1 = tensor(raw_a)
t2 = tensor(raw_b)
out = tensor(raw_b)
# C + scalar
assert (t1 + 1).flags['C_CONTIGUOUS'] == (raw_a + 1).flags['C_CONTIGUOUS']
assert (t1 + 1).flags['F_CONTIGUOUS'] == (raw_a + 1).flags['F_CONTIGUOUS']
# C + C
assert (t1 + t1).flags['C_CONTIGUOUS'] == (raw_a +
raw_a).flags['C_CONTIGUOUS']
assert (t1 + t1).flags['F_CONTIGUOUS'] == (raw_a +
raw_a).flags['F_CONTIGUOUS']
# F + scalar
assert (t2 + 1).flags['C_CONTIGUOUS'] == (raw_b + 1).flags['C_CONTIGUOUS']
assert (t2 + 1).flags['F_CONTIGUOUS'] == (raw_b + 1).flags['F_CONTIGUOUS']
# F + F
assert (t2 + t2).flags['C_CONTIGUOUS'] == (raw_b +
raw_b).flags['C_CONTIGUOUS']
assert (t2 + t2).flags['F_CONTIGUOUS'] == (raw_b +
raw_b).flags['F_CONTIGUOUS']
# C + F
assert (t1 + t2).flags['C_CONTIGUOUS'] == (raw_a +
raw_b).flags['C_CONTIGUOUS']
assert (t1 + t2).flags['F_CONTIGUOUS'] == (raw_a +
raw_b).flags['F_CONTIGUOUS']
# C + C + out
assert add(t1, t1, out=out).flags['C_CONTIGUOUS'] == np.add(
raw_a, raw_a, out=np.empty((4, 2), order='F')).flags['C_CONTIGUOUS']
assert add(t1, t1, out=out).flags['F_CONTIGUOUS'] == np.add(
raw_a, raw_a, out=np.empty((4, 2), order='F')).flags['F_CONTIGUOUS']
with pytest.raises(TypeError):
add(t1, 1, order='B')
def testAddWithOutExecution(self):
data1 = np.random.random((5, 9, 4))
data2 = np.random.random((9, 4))
arr1 = tensor(data1.copy(), chunk_size=3)
arr2 = tensor(data2.copy(), chunk_size=3)
add(arr1, arr2, out=arr1)
res = self.executor.execute_tensor(arr1, concat=True)[0]
self.assertTrue(np.array_equal(res, data1 + data2))
arr1 = tensor(data1.copy(), chunk_size=3)
arr2 = tensor(data2.copy(), chunk_size=3)
arr3 = add(arr1, arr2, out=arr1.astype('i4'), casting='unsafe')
res = self.executor.execute_tensor(arr3, concat=True)[0]
np.testing.assert_array_equal(res, (data1 + data2).astype('i4'))
arr1 = tensor(data1.copy(), chunk_size=3)
arr2 = tensor(data2.copy(), chunk_size=3)
arr3 = truediv(arr1, arr2, out=arr1, where=arr2 > .5)
res = self.executor.execute_tensor(arr3, concat=True)[0]
self.assertTrue(
np.array_equal(
res,
np.true_divide(data1,
data2,
out=data1.copy(),
where=data2 > .5)))
arr1 = tensor(data1.copy(), chunk_size=4)
arr2 = tensor(data2.copy(), chunk_size=4)
arr3 = add(arr1, arr2, where=arr1 > .5)
res = self.executor.execute_tensor(arr3, concat=True)[0]
expected = np.add(data1, data2, where=data1 > .5)
self.assertTrue(np.array_equal(res[data1 > .5], expected[data1 > .5]))
arr1 = tensor(data1.copy(), chunk_size=4)
arr3 = add(arr1, 1, where=arr1 > .5)
res = self.executor.execute_tensor(arr3, concat=True)[0]
expected = np.add(data1, 1, where=data1 > .5)
self.assertTrue(np.array_equal(res[data1 > .5], expected[data1 > .5]))
arr1 = tensor(data2.copy(), chunk_size=3)
arr3 = add(arr1[:5, :], 1, out=arr1[-5:, :])
res = self.executor.execute_tensor(arr3, concat=True)[0]
expected = np.add(data2[:5, :], 1)
self.assertTrue(np.array_equal(res, expected))
def test_add_with_out_execution(setup):
data1 = np.random.random((5, 9, 4))
data2 = np.random.random((9, 4))
arr1 = tensor(data1.copy(), chunk_size=3)
arr2 = tensor(data2.copy(), chunk_size=3)
add(arr1, arr2, out=arr1)
res = arr1.execute().fetch()
np.testing.assert_array_equal(res, data1 + data2)
arr1 = tensor(data1.copy(), chunk_size=3)
arr2 = tensor(data2.copy(), chunk_size=3)
arr3 = add(arr1, arr2, out=arr1.astype('i4'), casting='unsafe')
res = arr3.execute().fetch()
np.testing.assert_array_equal(res, (data1 + data2).astype('i4'))
arr1 = tensor(data1.copy(), chunk_size=3)
arr2 = tensor(data2.copy(), chunk_size=3)
arr3 = truediv(arr1, arr2, out=arr1, where=arr2 > .5)
res = arr3.execute().fetch()
np.testing.assert_array_equal(
res, np.true_divide(data1, data2, out=data1.copy(), where=data2 > .5))
arr1 = tensor(data1.copy(), chunk_size=4)
arr2 = tensor(data2.copy(), chunk_size=4)
arr3 = add(arr1, arr2, where=arr1 > .5)
res = arr3.execute().fetch()
expected = np.add(data1, data2, where=data1 > .5)
np.testing.assert_array_equal(
res[data1 > .5], expected[data1 > .5])
arr1 = tensor(data1.copy(), chunk_size=4)
arr3 = add(arr1, 1, where=arr1 > .5)
res = arr3.execute().fetch()
expected = np.add(data1, 1, where=data1 > .5)
np.testing.assert_array_equal(res[data1 > .5], expected[data1 > .5])
arr1 = tensor(data2.copy(), chunk_size=3)
arr3 = add(arr1[:5, :], 1, out=arr1[-5:, :])
res = arr3.execute().fetch()
expected = np.add(data2[:5, :], 1)
np.testing.assert_array_equal(res, expected)
def testAdd(self):
t1 = ones((3, 4), chunk_size=2)
t2 = ones(4, chunk_size=2)
t3 = t1 + t2
k1 = t3.key
t3.tiles()
self.assertNotEqual(t3.key, k1)
self.assertEqual(t3.shape, (3, 4))
self.assertEqual(len(t3.chunks), 4)
self.assertEqual(t3.chunks[0].inputs,
[t1.chunks[0].data, t2.chunks[0].data])
self.assertEqual(t3.chunks[1].inputs,
[t1.chunks[1].data, t2.chunks[1].data])
self.assertEqual(t3.chunks[2].inputs,
[t1.chunks[2].data, t2.chunks[0].data])
self.assertEqual(t3.chunks[3].inputs,
[t1.chunks[3].data, t2.chunks[1].data])
self.assertEqual(t3.op.dtype, np.dtype('f8'))
self.assertEqual(t3.chunks[0].op.dtype, np.dtype('f8'))
t4 = t1 + 1
t4.tiles()
self.assertEqual(t4.shape, (3, 4))
self.assertEqual(len(t3.chunks), 4)
self.assertEqual(t4.chunks[0].inputs, [t1.chunks[0].data])
self.assertEqual(t4.chunks[0].op.rhs, 1)
self.assertEqual(t4.chunks[1].inputs, [t1.chunks[1].data])
self.assertEqual(t4.chunks[1].op.rhs, 1)
self.assertEqual(t4.chunks[2].inputs, [t1.chunks[2].data])
self.assertEqual(t4.chunks[2].op.rhs, 1)
self.assertEqual(t4.chunks[3].inputs, [t1.chunks[3].data])
self.assertEqual(t4.chunks[3].op.rhs, 1)
# sparse tests
t5 = add([1, 2, 3, 4], 1)
t5.tiles()
self.assertEqual(t4.chunks[0].inputs, [t1.chunks[0].data])
t1 = tensor([[0, 1, 0], [1, 0, 0]], chunk_size=2).tosparse()
t = t1 + 1
self.assertTrue(t.issparse())
self.assertIs(type(t), SparseTensor)
t.tiles()
self.assertTrue(t.chunks[0].op.sparse)
t = t1 + 0
self.assertTrue(t.issparse())
self.assertIs(type(t), SparseTensor)
t2 = tensor([[1, 0, 0]], chunk_size=2).tosparse()
t = t1 + t2
self.assertTrue(t.issparse())
self.assertIs(type(t), SparseTensor)
t.tiles()
self.assertTrue(t.chunks[0].op.sparse)
t3 = tensor([1, 1, 1], chunk_size=2)
t = t1 + t3
self.assertFalse(t.issparse())
self.assertIs(type(t), Tensor)
t.tiles()
self.assertFalse(t.chunks[0].op.sparse)
def test_add():
t1 = ones((3, 4), chunk_size=2)
t2 = ones(4, chunk_size=2)
t3 = t1 + t2
k1 = t3.key
assert t3.op.gpu is False
t1, t2, t3 = tile(t1, t2, t3)
assert t3.key != k1
assert t3.shape == (3, 4)
assert len(t3.chunks) == 4
assert t3.chunks[0].inputs == [t1.chunks[0].data, t2.chunks[0].data]
assert t3.chunks[1].inputs == [t1.chunks[1].data, t2.chunks[1].data]
assert t3.chunks[2].inputs == [t1.chunks[2].data, t2.chunks[0].data]
assert t3.chunks[3].inputs == [t1.chunks[3].data, t2.chunks[1].data]
assert t3.op.dtype == np.dtype('f8')
assert t3.chunks[0].op.dtype == np.dtype('f8')
t1 = ones((3, 4), chunk_size=2)
t4 = t1 + 1
t1, t4 = tile(t1, t4)
assert t4.shape == (3, 4)
assert len(t3.chunks) == 4
assert t4.chunks[0].inputs == [t1.chunks[0].data]
assert t4.chunks[0].op.rhs == 1
assert t4.chunks[1].inputs == [t1.chunks[1].data]
assert t4.chunks[1].op.rhs == 1
assert t4.chunks[2].inputs == [t1.chunks[2].data]
assert t4.chunks[2].op.rhs == 1
assert t4.chunks[3].inputs == [t1.chunks[3].data]
assert t4.chunks[3].op.rhs == 1
t5 = add([1, 2, 3, 4], 1)
tile(t5)
assert t4.chunks[0].inputs == [t1.chunks[0].data]
t2 = ones(4, chunk_size=2)
t6 = ones((3, 4), chunk_size=2, gpu=True)
t7 = ones(4, chunk_size=2, gpu=True)
t8 = t6 + t7
t9 = t6 + t2
assert t8.op.gpu is True
t8, t9 = tile(t8, t9)
assert t8.chunks[0].op.gpu is True
assert t9.op.gpu is None
assert t9.chunks[0].op.gpu is None
# sparse tests
t1 = tensor([[0, 1, 0], [1, 0, 0]], chunk_size=2).tosparse()
t = t1 + 1
assert t.op.gpu is False
assert t.issparse() is True
assert type(t) is SparseTensor
t = tile(t)
assert t.chunks[0].op.sparse is True
t = t1 + 0
assert t.issparse() is True
assert type(t) is SparseTensor
t2 = tensor([[1, 0, 0]], chunk_size=2).tosparse()
t = t1 + t2
assert t.issparse() is True
assert type(t) is SparseTensor
t = tile(t)
assert t.chunks[0].op.sparse is True
t3 = tensor([1, 1, 1], chunk_size=2)
t = t1 + t3
assert t.issparse() is False
assert type(t) is Tensor
t = tile(t)
assert t.chunks[0].op.sparse is False
def testAdd(self):
t1 = ones((3, 4), chunk_size=2)
t2 = ones(4, chunk_size=2)
t3 = t1 + t2
k1 = t3.key
self.assertFalse(t3.op.gpu)
t3 = t3.tiles()
t1, t2 = get_tiled(t1), get_tiled(t2)
self.assertNotEqual(t3.key, k1)
self.assertEqual(t3.shape, (3, 4))
self.assertEqual(len(t3.chunks), 4)
self.assertEqual(t3.chunks[0].inputs,
[t1.chunks[0].data, t2.chunks[0].data])
self.assertEqual(t3.chunks[1].inputs,
[t1.chunks[1].data, t2.chunks[1].data])
self.assertEqual(t3.chunks[2].inputs,
[t1.chunks[2].data, t2.chunks[0].data])
self.assertEqual(t3.chunks[3].inputs,
[t1.chunks[3].data, t2.chunks[1].data])
self.assertEqual(t3.op.dtype, np.dtype('f8'))
self.assertEqual(t3.chunks[0].op.dtype, np.dtype('f8'))
t4 = t1 + 1
t4 = t4.tiles()
self.assertEqual(t4.shape, (3, 4))
self.assertEqual(len(t3.chunks), 4)
self.assertEqual(t4.chunks[0].inputs, [t1.chunks[0].data])
self.assertEqual(t4.chunks[0].op.rhs, 1)
self.assertEqual(t4.chunks[1].inputs, [t1.chunks[1].data])
self.assertEqual(t4.chunks[1].op.rhs, 1)
self.assertEqual(t4.chunks[2].inputs, [t1.chunks[2].data])
self.assertEqual(t4.chunks[2].op.rhs, 1)
self.assertEqual(t4.chunks[3].inputs, [t1.chunks[3].data])
self.assertEqual(t4.chunks[3].op.rhs, 1)
t5 = add([1, 2, 3, 4], 1)
t5 = t5.tiles()
self.assertEqual(t4.chunks[0].inputs, [t1.chunks[0].data])
t6 = ones((3, 4), chunk_size=2, gpu=True)
t7 = ones(4, chunk_size=2, gpu=True)
t8 = t6 + t7
self.assertTrue(t8.op.gpu)
t8 = t8.tiles()
self.assertTrue(t8.chunks[0].op.gpu)
t9 = t6 + t2
self.assertIsNone(t9.op.gpu)
t9 = t9.tiles()
self.assertIsNone(t9.chunks[0].op.gpu)
# sparse tests
t1 = tensor([[0, 1, 0], [1, 0, 0]], chunk_size=2).tosparse()
t = t1 + 1
self.assertFalse(t.op.gpu)
self.assertTrue(t.issparse())
self.assertIs(type(t), SparseTensor)
t = t.tiles()
self.assertTrue(t.chunks[0].op.sparse)
t = t1 + 0
self.assertTrue(t.issparse())
self.assertIs(type(t), SparseTensor)
t2 = tensor([[1, 0, 0]], chunk_size=2).tosparse()
t = t1 + t2
self.assertTrue(t.issparse())
self.assertIs(type(t), SparseTensor)
t = t.tiles()
self.assertTrue(t.chunks[0].op.sparse)
t3 = tensor([1, 1, 1], chunk_size=2)
t = t1 + t3
self.assertFalse(t.issparse())
self.assertIs(type(t), Tensor)
t = t.tiles()
self.assertFalse(t.chunks[0].op.sparse)
