def testSVGFormatter(self):
t = MockTable(name='pyodps_test_svg', schema=self.schema, _client=self.odps.rest)
expr = CollectionExpr(_source_data=t, _schema=self.schema)
expr1 = expr.groupby('name').agg(id=expr['id'].sum())
expr2 = expr1['name', expr1.id + 3]
engine = MixedEngine(self.odps)
dag = engine.compile(expr2)
nodes = dag.nodes()
self.assertEqual(len(nodes), 1)
expr3 = nodes[0].expr
self.assertIsInstance(expr3, GroupByCollectionExpr)
dot = ExprExecutionGraphFormatter(dag)._to_dot()
self.assertNotIn('Projection', dot)
expr1 = expr.groupby('name').agg(id=expr['id'].sum()).cache()
expr2 = expr1['name', expr1.id + 3]
engine = MixedEngine(self.odps)
dag = engine.compile(expr2)
nodes = dag.nodes()
self.assertEqual(len(nodes), 2)
dot = ExprExecutionGraphFormatter(dag)._to_dot()
self.assertIn('Projection', dot)
def setup(self):
import pandas as pd
odps_data = [
['name1', 1],
['name2', 2],
['name1', 3],
]
pd_data = [['name1', 5], ['name2', 6]]
names = ['name', 'id']
types = ['string', 'bigint']
table = tn('pyodps_df_mixed_%d' % os.getpid())
if self.odps.exist_table(table):
self.t = self.odps.get_table(table)
else:
self.t = self.odps.create_table(table,
Schema.from_lists(names, types),
lifecycle=1)
with self.t.open_writer() as w:
w.write([self.t.new_record(r) for r in odps_data])
self.odps_df = DataFrame(self.t)
self.pd_df = DataFrame(pd.DataFrame(pd_data, columns=names))
self.engine = MixedEngine(self.odps)
self.pd_engine = PandasEngine(self.odps)
def setup(self):
import pandas as pd
odps_data = [
['name1', 1],
['name2', 2],
['name1', 3],
]
pd_data = [
['name1', 5],
['name2', 6]
]
names = ['name', 'id']
types = ['string', 'bigint']
table = 'pyodps_df_mixed'
self.odps.delete_table(table, if_exists=True)
self.t = self.odps.create_table(table, Schema.from_lists(names, types))
with self.t.open_writer() as w:
w.write([self.t.new_record(r) for r in odps_data])
self.odps_df = DataFrame(self.t)
self.pd_df = DataFrame(pd.DataFrame(pd_data, columns=names))
self.engine = MixedEngine(self.odps)
self.pd_engine = PandasEngine(self.odps)
def testHeadAndTail(self):
res = self.odps_df.head(2)
self.assertEqual(len(res), 2)
df = self.odps_df[self.odps_df['name'] == 'name1']
res = df.head(1)
self.assertEqual(len(res), 1)
self.assertTrue(context.is_cached(df))
res = self.odps_df.tail(2)
self.assertEqual(len(res), 2)
self.assertTrue(all(it > 1 for it in res.values['id']))
self.assertEqual(len(self.odps_df.name.head(2)), 2)
self.assertEqual(len(self.odps_df.name.tail(2)), 2)
res = self.pd_df.head(1)
self.assertEqual(len(res), 1)
df = self.pd_df[self.pd_df['name'] == 'name1']
res = df.head(1)
self.assertEqual(len(res), 1)
self.assertTrue(context.is_cached(df))
res = self.pd_df.tail(1)
self.assertEqual(len(res), 1)
self.assertEqual(res.values['id'][0], 6)
self.assertEqual(len(self.pd_df.name.head(1)), 1)
self.assertEqual(len(self.pd_df.name.tail(1)), 1)
class TunnelOnlyODPSEngine(ODPSSQLEngine):
def _do_execute(self, *args, **kwargs):
kwargs['_force_tunnel'] = True
return super(TunnelOnlyODPSEngine,
self)._do_execute(*args, **kwargs)
engine = MixedEngine(self.odps)
engine._odpssql_engine = TunnelOnlyODPSEngine(self.odps)
res = engine.execute(self.odps_df['id'], head=3)
self.assertIsNotNone(res)
self.assertEqual(sum(res.values['id']), 6)
table_name = tn('pyodps_df_mixed2')
self.odps.delete_table(table_name, if_exists=True)
table = next(self.odps_df.data_source())
table2 = self.odps.create_table(table_name, table.schema)
try:
res = DataFrame(table2).head(10)
self.assertEqual(len(res), 0)
finally:
table2.drop()
def setup(self):
datatypes = lambda *types: [validate_data_type(t) for t in types]
schema = Schema.from_lists(['name', 'id', 'fid', 'isMale', 'scale', 'birth'],
datatypes('string', 'bigint', 'double', 'boolean', 'decimal', 'datetime'))
table = MockTable(name='pyodps_test_expr_table', schema=schema)
self.tb = DataFrame(table)
import pandas as pd
df = pd.DataFrame([['name1', 2, 3.14], ['name2', 100, 2.7]], columns=['name', 'id', 'fid'])
self.pd = DataFrame(df)
self.expr = self.tb.join(self.pd, on='name')
self.engine = MixedEngine(self.odps)