def testDataFrameGraphSerialize(self):
df = from_pandas_df(pd.DataFrame(np.random.rand(10, 10),
columns=pd.timedelta_range(start='1 day', periods=10),
index=pd.date_range('2020-1-1', periods=10)))
graph = df.build_graph(tiled=False)
pb = graph.to_pb()
graph2 = DAG.from_pb(pb)
self.assertEqual(len(graph), len(graph2))
t = next(iter(graph))
t2 = next(iter(graph2))
self.assertTrue(t2.op.outputs[0], ReferenceType) # make sure outputs are all weak reference
self.assertBaseEqual(t.op, t2.op)
self.assertEqual(t.shape, t2.shape)
self.assertEqual(sorted(i.key for i in t.inputs), sorted(i.key for i in t2.inputs))
pd.testing.assert_index_equal(t2.index_value.to_pandas(), t.index_value.to_pandas())
pd.testing.assert_index_equal(t2.columns_value.to_pandas(), t.columns_value.to_pandas())
jsn = graph.to_json()
graph2 = DAG.from_json(jsn)
self.assertEqual(len(graph), len(graph2))
t = next(iter(graph))
t2 = next(iter(graph2))
self.assertTrue(t2.op.outputs[0], ReferenceType) # make sure outputs are all weak reference
self.assertBaseEqual(t.op, t2.op)
self.assertEqual(t.shape, t2.shape)
self.assertEqual(sorted(i.key for i in t.inputs), sorted(i.key for i in t2.inputs))
pd.testing.assert_index_equal(t2.index_value.to_pandas(), t.index_value.to_pandas())
pd.testing.assert_index_equal(t2.columns_value.to_pandas(), t.columns_value.to_pandas())
# test graph with tiled DataFrame
t2 = from_pandas_df(pd.DataFrame(np.random.rand(10, 10)), chunk_size=(5, 4)).tiles()
graph = DAG()
graph.add_node(t2)
pb = graph.to_pb()
graph2 = DAG.from_pb(pb)
self.assertEqual(len(graph), len(graph2))
chunks = next(iter(graph2)).chunks
self.assertEqual(len(chunks), 6)
self.assertIsInstance(chunks[0], DataFrameChunk)
self.assertEqual(chunks[0].index, t2.chunks[0].index)
self.assertBaseEqual(chunks[0].op, t2.chunks[0].op)
pd.testing.assert_index_equal(chunks[0].index_value.to_pandas(), t2.chunks[0].index_value.to_pandas())
pd.testing.assert_index_equal(chunks[0].columns_value.to_pandas(), t2.chunks[0].columns_value.to_pandas())
jsn = graph.to_json()
graph2 = DAG.from_json(jsn)
self.assertEqual(len(graph), len(graph2))
chunks = next(iter(graph2)).chunks
self.assertEqual(len(chunks), 6)
self.assertIsInstance(chunks[0], DataFrameChunk)
self.assertEqual(chunks[0].index, t2.chunks[0].index)
self.assertBaseEqual(chunks[0].op, t2.chunks[0].op)
pd.testing.assert_index_equal(chunks[0].index_value.to_pandas(), t2.chunks[0].index_value.to_pandas())
pd.testing.assert_index_equal(chunks[0].columns_value.to_pandas(), t2.chunks[0].columns_value.to_pandas())
def testTensorGraphSerialize(self):
t = ones((10, 3), chunk_size=(5, 2)) + tensor(np.random.random(
(10, 3)),
chunk_size=(5, 2))
graph = t.build_graph(tiled=False)
pb = graph.to_pb()
graph2 = DAG.from_pb(pb)
self.assertEqual(len(graph), len(graph2))
t = next(c for c in graph if c.inputs)
t2 = next(c for c in graph2 if c.key == t.key)
self.assertTrue(
t2.op.outputs[0],
ReferenceType) # make sure outputs are all weak reference
self.assertBaseEqual(t.op, t2.op)
self.assertEqual(t.shape, t2.shape)
self.assertEqual(sorted(i.key for i in t.inputs),
sorted(i.key for i in t2.inputs))
jsn = graph.to_json()
graph2 = DAG.from_json(jsn)
self.assertEqual(len(graph), len(graph2))
t = next(c for c in graph if c.inputs)
t2 = next(c for c in graph2 if c.key == t.key)
self.assertTrue(
t2.op.outputs[0],
ReferenceType) # make sure outputs are all weak reference
self.assertBaseEqual(t.op, t2.op)
self.assertEqual(t.shape, t2.shape)
self.assertEqual(sorted(i.key for i in t.inputs),
sorted(i.key for i in t2.inputs))
# test graph with tiled tensor
t2 = ones((10, 10), chunk_size=(5, 4)).tiles()
graph = DAG()
graph.add_node(t2)
pb = graph.to_pb()
graph2 = DAG.from_pb(pb)
self.assertEqual(len(graph), len(graph2))
chunks = next(iter(graph2)).chunks
self.assertEqual(len(chunks), 6)
self.assertIsInstance(chunks[0], TensorChunk)
self.assertEqual(chunks[0].index, t2.chunks[0].index)
self.assertBaseEqual(chunks[0].op, t2.chunks[0].op)
jsn = graph.to_json()
graph2 = DAG.from_json(jsn)
self.assertEqual(len(graph), len(graph2))
chunks = next(iter(graph2)).chunks
self.assertEqual(len(chunks), 6)
self.assertIsInstance(chunks[0], TensorChunk)
self.assertEqual(chunks[0].index, t2.chunks[0].index)
self.assertBaseEqual(chunks[0].op, t2.chunks[0].op)
def testDataFrameGraphSerialize(self):
df = from_pandas(pd.DataFrame(np.random.rand(10, 10)))
graph = df.build_graph(tiled=False)
pb = graph.to_pb()
graph2 = DAG.from_pb(pb)
self.assertEqual(len(graph), len(graph2))
t = next(iter(graph))
t2 = next(iter(graph2))
self.assertTrue(
t2.op.outputs[0],
ReferenceType) # make sure outputs are all weak reference
self.assertBaseEqual(t.op, t2.op)
self.assertEqual(t.shape, t2.shape)
self.assertEqual(sorted(i.key for i in t.inputs),
sorted(i.key for i in t2.inputs))
jsn = graph.to_json()
graph2 = DAG.from_json(jsn)
self.assertEqual(len(graph), len(graph2))
t = next(iter(graph))
t2 = next(iter(graph2))
self.assertTrue(
t2.op.outputs[0],
ReferenceType) # make sure outputs are all weak reference
self.assertBaseEqual(t.op, t2.op)
self.assertEqual(t.shape, t2.shape)
self.assertEqual(sorted(i.key for i in t.inputs),
sorted(i.key for i in t2.inputs))
# test graph with tiled DataFrame
t2 = from_pandas(pd.DataFrame(np.random.rand(10, 10)),
chunk_size=(5, 4)).tiles()
graph = DAG()
graph.add_node(t2)
pb = graph.to_pb()
graph2 = DAG.from_pb(pb)
self.assertEqual(len(graph), len(graph2))
chunks = next(iter(graph2)).chunks
self.assertEqual(len(chunks), 6)
self.assertIsInstance(chunks[0], DataFrameChunk)
self.assertEqual(chunks[0].index, t2.chunks[0].index)
self.assertBaseEqual(chunks[0].op, t2.chunks[0].op)
jsn = graph.to_json()
graph2 = DAG.from_json(jsn)
self.assertEqual(len(graph), len(graph2))
chunks = next(iter(graph2)).chunks
self.assertEqual(len(chunks), 6)
self.assertIsInstance(chunks[0], DataFrameChunk)
self.assertEqual(chunks[0].index, t2.chunks[0].index)
self.assertBaseEqual(chunks[0].op, t2.chunks[0].op)
