def testTensorSerialize(self):
from mars.tensor import split
t = ones((10, 10, 8), chunk_size=(3, 3, 5))
serials = self._pb_serial(t)
dt = self._pb_deserial(serials)[t.data]
self.assertEqual(dt.extra_params.raw_chunk_size, (3, 3, 5))
serials = self._json_serial(t)
dt = self._json_deserial(serials)[t.data]
self.assertEqual(dt.extra_params.raw_chunk_size, (3, 3, 5))
t2, _ = split(t, 2)
serials = self._pb_serial(t2)
dt = self._pb_deserial(serials)[t2.data]
self.assertEqual(dt.op.indices_or_sections, 2)
t2, _, _ = split(t, ones(2, chunk_size=2))
serials = self._pb_serial(t2)
dt = self._pb_deserial(serials)[t2.data]
with enter_mode(build=True):
self.assertIn(dt.op.indices_or_sections, dt.inputs)
def testSetTensorInputs(self):
t1 = tensor([1, 2], chunk_size=2)
t2 = tensor([2, 3], chunk_size=2)
t3 = t1 + t2
t1c = copy(t1)
t2c = copy(t2)
self.assertIsNot(t1c, t1)
self.assertIsNot(t2c, t2)
self.assertIs(t3.op.lhs, t1.data)
self.assertIs(t3.op.rhs, t2.data)
self.assertEqual(t3.op.inputs, [t1.data, t2.data])
self.assertEqual(t3.inputs, [t1.data, t2.data])
with self.assertRaises(StopIteration):
t3.inputs = []
t1 = tensor([1, 2], chunk_size=2)
t2 = tensor([True, False], chunk_size=2)
t3 = t1[t2]
t1c = copy(t1)
t2c = copy(t2)
t3c = copy(t3)
t3c.inputs = [t1c, t2c]
with enter_mode(build=True):
self.assertIs(t3c.op.input, t1c.data)
self.assertIs(t3c.op.indexes[0], t2c.data)
def wrapped2():
wrapped()
with option_context({'eager_mode': True}):
assert options.eager_mode
assert not utils.is_eager_mode()
with utils.enter_mode(kernel=False):
assert not utils.is_kernel_mode()
assert utils.is_kernel_mode()
def wrapped2():
wrapped()
with option_context({'eager_mode': True}):
self.assertTrue(options.eager_mode)
self.assertFalse(utils.is_eager_mode())
with utils.enter_mode(kernel=False):
self.assertFalse(utils.is_kernel_mode())
self.assertTrue(utils.is_kernel_mode())
def wrapped3():
wrapped()
with option_context({'eager_mode': True}):
assert options.eager_mode
assert not utils.is_kernel_mode()
with utils.enter_mode(kernel=True, build=True):
assert utils.is_kernel_mode()
assert utils.is_build_mode()
assert not utils.is_kernel_mode()
assert not utils.is_build_mode()
with pytest.raises(ValueError):
with utils.enter_mode(kernel=True, build=True):
raise ValueError('meant to raise error')
assert not utils.is_kernel_mode()
assert not utils.is_build_mode()
@utils.enter_mode(kernel=True)
def wrapped4():
raise ValueError('meant to raise error')
with pytest.raises(ValueError):
wrapped4()
assert not utils.is_kernel_mode()
assert not utils.is_build_mode()
def wrapped3():
wrapped()
with option_context({'eager_mode': True}):
self.assertTrue(options.eager_mode)
self.assertFalse(utils.is_kernel_mode())
with utils.enter_mode(kernel=True, build=True):
self.assertTrue(utils.is_kernel_mode())
self.assertTrue(utils.is_build_mode())
self.assertFalse(utils.is_kernel_mode())
self.assertFalse(utils.is_build_mode())
with self.assertRaises(ValueError):
with utils.enter_mode(kernel=True, build=True):
raise ValueError('meant to raise error')
self.assertFalse(utils.is_kernel_mode())
self.assertFalse(utils.is_build_mode())
@utils.enter_mode(kernel=True)
def wrapped4():
raise ValueError('meant to raise error')
with self.assertRaises(ValueError):
wrapped4()
self.assertFalse(utils.is_kernel_mode())
self.assertFalse(utils.is_build_mode())
def testArrowListArrayCreation(self):
# create from pandas Series
series = pd.Series([['a', 'b'], ['c'], ['d', 'e']])
array = ArrowListArray(series)
self.assertIsInstance(array.dtype, ArrowListDtype)
self.assertIsInstance(array.dtype.value_type, ArrowStringDtype)
self.assertIsInstance(array._arrow_array, pa.ChunkedArray)
# create from list
lst = [['a'], ['b', 'c'], ['d', 'e']]
array = ArrowListArray(lst)
self.assertIsInstance(array.dtype, ArrowListDtype)
self.assertIsInstance(array.dtype.value_type, ArrowStringDtype)
self.assertIsInstance(array._arrow_array, pa.ChunkedArray)
# create from pyarrow Array
a = pa.array([[1.], [2., 3.], [4.]])
array = ArrowListArray(a)
self.assertIsInstance(array.dtype, ArrowListDtype)
self.assertEqual(array.dtype.value_type, np.float64)
self.assertIsInstance(array._arrow_array, pa.ChunkedArray)
# create from ArrowListArray
array2 = ArrowListArray(array)
self.assertIsInstance(array2._arrow_array, pa.ChunkedArray)
# test _from_sequence
array = ArrowListArray._from_sequence([[1, 2], [3, 4], [5]])
self.assertIsInstance(array.dtype, ArrowListDtype)
self.assertEqual(array.dtype.value_type, np.int64)
self.assertIsInstance(array._arrow_array, pa.ChunkedArray)
# test pandas_only
with option_context({'dataframe.arrow_array.pandas_only': True}):
array = ArrowListArray._from_sequence([[1, 2], [3, 4], [5]])
self.assertIsInstance(array.dtype, ArrowListDtype)
self.assertIsInstance(array._ndarray, np.ndarray)
# test pandas_only and in kernel mode
with enter_mode(kernel=True), \
option_context({'dataframe.arrow_array.pandas_only': True}), \
self.assertRaises(ImportError):
ArrowListArray._from_sequence([[1, 2], [3, 4], [5]])
def testComp(self):
df1 = DataFrame(pd.DataFrame(np.random.rand(4, 3)))
df2 = DataFrame(pd.DataFrame(np.random.rand(4, 3)))
with enter_mode(build=True):
self.assertFalse(df1.data == df2.data)
self.assertTrue(df1.data == df1.data)
for op in [
operator.eq, operator.ne, operator.lt, operator.gt,
operator.le, operator.ge
]:
eq_df = op(df1, df2)
pd.testing.assert_index_equal(eq_df.index_value.to_pandas(),
df1.index_value.to_pandas())
# index not identical
df3 = DataFrame(
pd.DataFrame(np.random.rand(4, 3), index=[1, 2, 3, 4]))
with self.assertRaises(ValueError):
op(df1, df3)
# columns not identical
df4 = DataFrame(
pd.DataFrame(np.random.rand(4, 3), columns=['a', 'b', 'c']))
with self.assertRaises(ValueError):
op(df1, df4)
# test datetime
df = DataFrame(pd.DataFrame(pd.date_range('20130101', periods=6)))
for op in [
operator.eq, operator.ne, operator.lt, operator.gt,
operator.le, operator.ge
]:
r_df = op(df, datetime(2013, 1, 2))
pd.testing.assert_index_equal(r_df.index_value.to_pandas(),
df.index_value.to_pandas())
def testBuildMode(self):
t1 = ones((2, 3), chunk_size=2)
self.assertTrue(t1 == 2)
with enter_mode(build=True):
self.assertFalse(t1 == 2)