def testFromRecordsExecution(self):
dtype = np.dtype([('x', 'int'), ('y', 'double'), ('z', '<U16')])
ndarr = np.ones((10,), dtype=dtype)
pdf_expected = pd.DataFrame.from_records(ndarr, index=pd.RangeIndex(10))
# from structured array of mars
tensor = mt.ones((10,), dtype=dtype, chunk_size=3)
df1 = from_records(tensor)
df1_result = self.executor.execute_dataframe(df1, concat=True)[0]
pd.testing.assert_frame_equal(df1_result, pdf_expected)
# from structured array of numpy
df2 = from_records(ndarr)
df2_result = self.executor.execute_dataframe(df2, concat=True)[0]
pd.testing.assert_frame_equal(df2_result, pdf_expected)
def testFromRecords(self):
dtype = np.dtype([('x', 'int'), ('y', 'double'), ('z', '<U16')])
tensor = mt.ones((10,), dtype=dtype, chunk_size=3)
df = from_records(tensor)
df.tiles()
self.assertEqual(df.chunk_shape, (4, 1))
self.assertEqual(df.chunks[0].shape, (3, 3))
self.assertEqual(df.chunks[1].shape, (3, 3))
self.assertEqual(df.chunks[2].shape, (3, 3))
self.assertEqual(df.chunks[3].shape, (1, 3))
self.assertEqual(df.chunks[0].inputs[0].shape, (3,))
self.assertEqual(df.chunks[1].inputs[0].shape, (3,))
self.assertEqual(df.chunks[2].inputs[0].shape, (3,))
self.assertEqual(df.chunks[3].inputs[0].shape, (1,))
self.assertEqual(df.chunks[0].op.extra_params, {'begin_index': 0, 'end_index': 3})
self.assertEqual(df.chunks[1].op.extra_params, {'begin_index': 3, 'end_index': 6})
self.assertEqual(df.chunks[2].op.extra_params, {'begin_index': 6, 'end_index': 9})
self.assertEqual(df.chunks[3].op.extra_params, {'begin_index': 9, 'end_index': 10})
names = pd.Index(['x', 'y', 'z'])
dtypes = pd.Series({'x': np.dtype('int'), 'y': np.dtype('double'), 'z': np.dtype('<U16')})
for chunk in df.chunks:
pd.testing.assert_index_equal(chunk.columns_value.to_pandas(), names)
pd.testing.assert_series_equal(chunk.dtypes, dtypes)
pd.testing.assert_index_equal(df.chunks[0].index_value.to_pandas(), pd.RangeIndex(0, 3))
pd.testing.assert_index_equal(df.chunks[1].index_value.to_pandas(), pd.RangeIndex(3, 6))
pd.testing.assert_index_equal(df.chunks[2].index_value.to_pandas(), pd.RangeIndex(6, 9))
pd.testing.assert_index_equal(df.chunks[3].index_value.to_pandas(), pd.RangeIndex(9, 10))
def test_from_records_execution(setup):
dtype = np.dtype([('x', 'int'), ('y', 'double'), ('z', '<U16')])
ndarr = np.ones((10,), dtype=dtype)
pdf_expected = pd.DataFrame.from_records(ndarr, index=pd.RangeIndex(10))
# from structured array of mars
tensor = mt.ones((10,), dtype=dtype, chunk_size=3)
df1 = from_records(tensor)
df1_result = df1.execute().fetch()
pd.testing.assert_frame_equal(df1_result, pdf_expected)
# from structured array of numpy
df2 = from_records(ndarr)
df2_result = df2.execute().fetch()
pd.testing.assert_frame_equal(df2_result, pdf_expected)
