def setUp(self):
        super(RunnerTestBase, self).setUp()

        RunnerContext.reset()
        from odps.runner.df.adapter import _df_endpoint_dict, _df_link_maintainer
        _df_endpoint_dict.clear()
        _df_link_maintainer.clear()

        # Force to false
        options.runner.dry_run = False
        options.lifecycle = 3
        options.verbose = 'CI_MODE' not in os.environ
        options.interactive = False
        # Disable warnings
        warnings.filterwarnings('ignore', category=DeprecationWarning, module=r'.*widget\.py.*')
Exemple #2
0
    def setUp(self):
        super(RunnerTestBase, self).setUp()

        RunnerContext.reset()
        from odps.runner.df.adapter import _df_endpoint_dict, _df_link_maintainer
        _df_endpoint_dict.clear()
        _df_link_maintainer.clear()

        # Force to false
        options.runner.dry_run = False
        options.lifecycle = 3
        options.verbose = 'CI_MODE' not in os.environ
        options.interactive = False
        # Disable warnings
        warnings.filterwarnings('ignore',
                                category=DeprecationWarning,
                                module=r'.*widget\.py.*')
    def create_test_pmml_model(self, model_name):
        if self.odps.exist_offline_model(model_name):
            return

        old_node_id = RunnerContext.instance()._dag._node_seq_id
        old_dry_run = options.runner.dry_run
        options.runner.dry_run = False

        self.create_ionosphere(IONOSPHERE_TABLE)

        from odps.ml import classifiers

        df = DataFrame(self.odps.get_table(IONOSPHERE_TABLE)).roles(label='class')
        lr = classifiers.LogisticRegression(epsilon=0.001).set_max_iter(50)
        lr.train(df).persist(model_name)

        options.runner.dry_run = old_dry_run
        RunnerContext.instance()._dag._node_seq_id = old_node_id
 def actual_exec(self):
     in_tables = []
     for in_name, in_port in six.iteritems(self._node.inputs):
         if in_port.type != PortType.DATA:
             continue
         ep = RunnerContext.instance()._obj_container.get(in_port.obj_uuid)
         if ep:
             in_tables.append((in_name, ep.table))
     msg = 'Message: %s Input tables: %s' % (self._node.message, ', '.join('%s<-%s' % ti for ti in in_tables))
     if self._node.action is not None:
         self._node.action(self._node)
     else:
         print(msg)
Exemple #5
0
 def actual_exec(self):
     in_tables = []
     for in_name, in_port in six.iteritems(self._node.inputs):
         if in_port.type != PortType.DATA:
             continue
         ep = RunnerContext.instance()._obj_container.get(in_port.obj_uuid)
         if ep:
             in_tables.append((in_name, ep.table))
     msg = 'Message: %s Input tables: %s' % (self._node.message, ', '.join(
         '%s<-%s' % ti for ti in in_tables))
     if self._node.action is not None:
         self._node.action(self._node)
     else:
         print(msg)
    def test_construct(self):
        if utils.is_secret_mode():
            get_odps_tuple = lambda o: (o.account.access_id, o.project, o.
                                        endpoint)
        else:
            get_odps_tuple = lambda o: (o.account.access_id, o.account.
                                        secret_access_key, o.project, o.
                                        endpoint)

        with_odps = RunnerContext(self.odps)
        self.assertTupleEqual(get_odps_tuple(self.odps),
                              get_odps_tuple(with_odps._odps))

        if not utils.is_secret_mode():
            inter.teardown(TEST_CONTEXT_ROOM)
            inter.setup(*get_odps_tuple(self.odps), room=TEST_CONTEXT_ROOM)
            inter.enter(TEST_CONTEXT_ROOM)

            without_odps = RunnerContext()
            self.assertTupleEqual(get_odps_tuple(self.odps),
                                  get_odps_tuple(without_odps._odps))

            inter.teardown(TEST_CONTEXT_ROOM)
 def setUp(self):
     super(Test, self).setUp()
     self.ml_context = RunnerContext.instance()
     self.create_ionosphere(IONOSPHERE_TABLE)
     self.df = DataFrame(self.odps.get_table(IONOSPHERE_TABLE))
 def setUp(self):
     super(TestCore, self).setUp()
     self.ml_context = RunnerContext.instance()
 def setUp(self):
     super(TestCore, self).setUp()
     self.ml_context = RunnerContext.instance()
Exemple #10
0
 def after_exec(self):
     from odps.runner import RunnerContext
     context = RunnerContext.instance()
     self._node.after_exec(self._odps, True)
     context._node_outputs[self._node_hash] = dict((pn, self.get_output_object(p))
                                                   for pn, p in six.iteritems(self._node.outputs))