def setUp(self):
super(RunnerTestBase, self).setUp()
RunnerContext.reset()
from odps.runner.df.adapter import _df_endpoint_dict, _df_link_maintainer
_df_endpoint_dict.clear()
_df_link_maintainer.clear()
# Force to false
options.runner.dry_run = False
options.lifecycle = 3
options.verbose = 'CI_MODE' not in os.environ
options.interactive = False
# Disable warnings
warnings.filterwarnings('ignore', category=DeprecationWarning, module=r'.*widget\.py.*')
def setUp(self):
super(RunnerTestBase, self).setUp()
RunnerContext.reset()
from odps.runner.df.adapter import _df_endpoint_dict, _df_link_maintainer
_df_endpoint_dict.clear()
_df_link_maintainer.clear()
# Force to false
options.runner.dry_run = False
options.lifecycle = 3
options.verbose = 'CI_MODE' not in os.environ
options.interactive = False
# Disable warnings
warnings.filterwarnings('ignore',
category=DeprecationWarning,
module=r'.*widget\.py.*')
def create_test_pmml_model(self, model_name):
if self.odps.exist_offline_model(model_name):
return
old_node_id = RunnerContext.instance()._dag._node_seq_id
old_dry_run = options.runner.dry_run
options.runner.dry_run = False
self.create_ionosphere(IONOSPHERE_TABLE)
from odps.ml import classifiers
df = DataFrame(self.odps.get_table(IONOSPHERE_TABLE)).roles(label='class')
lr = classifiers.LogisticRegression(epsilon=0.001).set_max_iter(50)
lr.train(df).persist(model_name)
options.runner.dry_run = old_dry_run
RunnerContext.instance()._dag._node_seq_id = old_node_id
def actual_exec(self):
in_tables = []
for in_name, in_port in six.iteritems(self._node.inputs):
if in_port.type != PortType.DATA:
continue
ep = RunnerContext.instance()._obj_container.get(in_port.obj_uuid)
if ep:
in_tables.append((in_name, ep.table))
msg = 'Message: %s Input tables: %s' % (self._node.message, ', '.join('%s<-%s' % ti for ti in in_tables))
if self._node.action is not None:
self._node.action(self._node)
else:
print(msg)
def actual_exec(self):
in_tables = []
for in_name, in_port in six.iteritems(self._node.inputs):
if in_port.type != PortType.DATA:
continue
ep = RunnerContext.instance()._obj_container.get(in_port.obj_uuid)
if ep:
in_tables.append((in_name, ep.table))
msg = 'Message: %s Input tables: %s' % (self._node.message, ', '.join(
'%s<-%s' % ti for ti in in_tables))
if self._node.action is not None:
self._node.action(self._node)
else:
print(msg)
def test_construct(self):
if utils.is_secret_mode():
get_odps_tuple = lambda o: (o.account.access_id, o.project, o.
endpoint)
else:
get_odps_tuple = lambda o: (o.account.access_id, o.account.
secret_access_key, o.project, o.
endpoint)
with_odps = RunnerContext(self.odps)
self.assertTupleEqual(get_odps_tuple(self.odps),
get_odps_tuple(with_odps._odps))
if not utils.is_secret_mode():
inter.teardown(TEST_CONTEXT_ROOM)
inter.setup(*get_odps_tuple(self.odps), room=TEST_CONTEXT_ROOM)
inter.enter(TEST_CONTEXT_ROOM)
without_odps = RunnerContext()
self.assertTupleEqual(get_odps_tuple(self.odps),
get_odps_tuple(without_odps._odps))
inter.teardown(TEST_CONTEXT_ROOM)
def setUp(self):
super(Test, self).setUp()
self.ml_context = RunnerContext.instance()
self.create_ionosphere(IONOSPHERE_TABLE)
self.df = DataFrame(self.odps.get_table(IONOSPHERE_TABLE))
def setUp(self):
super(TestCore, self).setUp()
self.ml_context = RunnerContext.instance()
def setUp(self):
super(TestCore, self).setUp()
self.ml_context = RunnerContext.instance()
def after_exec(self):
from odps.runner import RunnerContext
context = RunnerContext.instance()
self._node.after_exec(self._odps, True)
context._node_outputs[self._node_hash] = dict((pn, self.get_output_object(p))
for pn, p in six.iteritems(self._node.outputs))