コード例 #1
0
class Test(TestBase):
    def setup(self):
        datatypes = lambda *types: [validate_data_type(t) for t in types]
        schema = Schema.from_lists(['name', 'id', 'fid'],
                                   datatypes('string', 'int64', 'float64'))
        table = MockTable(name='pyodps_test_expr_table', schema=schema)

        self.expr = CollectionExpr(_source_data=table, _schema=schema)
        self.ctx = ExecuteContext()

    def testBuildDAG(self):
        expr = self.expr[self.expr.name, self.expr.id + 1]
        expr2 = self.expr[self.expr.name, self.expr.id + 2]
        expr3 = self.expr[self.expr.name, self.expr.id + 3]
        self.ctx.build_dag(expr, expr.copy_tree())
        self.ctx.build_dag(expr2, expr2.copy_tree())
        self.ctx.build_dag(expr3, expr3.copy_tree())

        self.assertEqual(len(self.ctx._expr_to_dag), 3)
        self.assertGreater(len(list(self.ctx._expr_to_dag.values())[0].nodes()), 0)
        self.assertGreater(len(list(self.ctx._expr_to_dag.values())[1].nodes()), 0)
        self.assertGreater(len(list(self.ctx._expr_to_dag.values())[2].nodes()), 0)
        self.assertTrue(all(l is r for l, r in zip(expr.traverse(unique=True),
                                                   self.ctx.get_dag(expr).traverse(expr))))

        del expr
        self.assertEqual(len(self.ctx._expr_to_dag), 2)
        del expr2
        del expr3
        self.assertEqual(len(self.ctx._expr_to_dag), 0)
コード例 #2
0
class Test(TestBase):
    def setup(self):
        datatypes = lambda *types: [validate_data_type(t) for t in types]
        schema = Schema.from_lists(["name", "id", "fid"], datatypes("string", "int64", "float64"))
        table = MockTable(name="pyodps_test_expr_table", schema=schema)

        self.expr = CollectionExpr(_source_data=table, _schema=schema)
        self.ctx = ExecuteContext()

    def testBuildDAG(self):
        expr = self.expr[self.expr.name, self.expr.id + 1]
        expr2 = self.expr[self.expr.name, self.expr.id + 2]
        expr3 = self.expr[self.expr.name, self.expr.id + 3]
        self.ctx.build_dag(expr, expr.copy_tree())
        self.ctx.build_dag(expr2, expr2.copy_tree())
        self.ctx.build_dag(expr3, expr3.copy_tree())

        self.assertEqual(len(self.ctx._expr_to_dag), 3)
        self.assertGreater(len(list(self.ctx._expr_to_dag.values())[0].nodes()), 0)
        self.assertGreater(len(list(self.ctx._expr_to_dag.values())[1].nodes()), 0)
        self.assertGreater(len(list(self.ctx._expr_to_dag.values())[2].nodes()), 0)
        self.assertTrue(all(l is r for l, r in zip(expr.traverse(unique=True), self.ctx.get_dag(expr).traverse(expr))))

        del expr
        self.assertEqual(len(self.ctx._expr_to_dag), 2)
        del expr2
        del expr3
        self.assertEqual(len(self.ctx._expr_to_dag), 0)
コード例 #3
0
    def setup(self):
        datatypes = lambda *types: [validate_data_type(t) for t in types]
        schema = Schema.from_lists(['name', 'id', 'fid'],
                                   datatypes('string', 'int64', 'float64'))
        table = MockTable(name='pyodps_test_expr_table', schema=schema)

        self.expr = CollectionExpr(_source_data=table, _schema=schema)
        self.ctx = ExecuteContext()
コード例 #4
0
    def setup(self):
        datatypes = lambda *types: [validate_data_type(t) for t in types]
        schema = Schema.from_lists(["name", "id", "fid"], datatypes("string", "int64", "float64"))
        table = MockTable(name="pyodps_test_expr_table", schema=schema)

        self.expr = CollectionExpr(_source_data=table, _schema=schema)
        self.ctx = ExecuteContext()
コード例 #5
0
    def testPandasCompilation(self):
        import pandas as pd
        import numpy as np

        df = pd.DataFrame(np.arange(9).reshape(3, 3), columns=list('abc'))

        schema = Schema.from_lists(list('abc'), [types.int8] * 3)
        expr = CollectionExpr(_source_data=df, _schema=schema)

        expr = expr['a', 'b']
        ctx = ExecuteContext()

        compiler = PandasCompiler(ctx.build_dag(expr, expr))
        dag = compiler.compile(expr)

        self.assertEqual(len(dag._graph), 4)
        topos = dag.topological_sort()
        self.assertIsInstance(topos[0][0], CollectionExpr)
        self.assertIsInstance(topos[1][0], Column)
        self.assertIsInstance(topos[2][0], Column)
        self.assertIsInstance(topos[3][0], ProjectCollectionExpr)
コード例 #6
0
    def testPandasCompilation(self):
        import pandas as pd
        import numpy as np

        df = pd.DataFrame(np.arange(9).reshape(3, 3), columns=list('abc'))

        schema = Schema.from_lists(list('abc'), [types.int8] * 3)
        expr = CollectionExpr(_source_data=df, _schema=schema)

        expr = expr['a', 'b']
        ctx = ExecuteContext()

        compiler = PandasCompiler(expr.to_dag())
        dag = compiler.compile(expr)

        self.assertEqual(len(dag._graph), 4)
        topos = dag.topological_sort()
        self.assertIsInstance(topos[0][0], CollectionExpr)
        self.assertIsInstance(topos[1][0], Column)
        self.assertIsInstance(topos[2][0], Column)
        self.assertIsInstance(topos[3][0], ProjectCollectionExpr)