def test_update_actions(self):
db_actions = action_service.create_actions(ACTION_LIST)
self.assertEqual(2, len(db_actions))
action1_db = self._assert_single_item(db_actions, name='action1')
action1_spec = spec_parser.get_action_spec(action1_db.spec)
self.assertEqual('action1', action1_spec.get_name())
self.assertEqual('std.echo', action1_spec.get_base())
self.assertDictEqual({'output': 'Hi'}, action1_spec.get_base_input())
self.assertDictEqual({}, action1_spec.get_input())
db_actions = action_service.update_actions(UPDATED_ACTION_LIST)
# Action 1.
action1_db = self._assert_single_item(db_actions, name='action1')
action1_spec = spec_parser.get_action_spec(action1_db.spec)
self.assertEqual('action1', action1_spec.get_name())
self.assertListEqual([], action1_spec.get_tags())
self.assertEqual('std.echo', action1_spec.get_base())
self.assertDictEqual({'output': 'Hi'}, action1_spec.get_base_input())
self.assertIn('param1', action1_spec.get_input())
self.assertIs(action1_spec.get_input().get('param1'), utils.NotDefined)
def test_update_actions(self):
db_actions = action_service.create_actions(ACTION_LIST)
self.assertEqual(2, len(db_actions))
action1_db = self._assert_single_item(db_actions, name='action1')
action1_spec = spec_parser.get_action_spec(action1_db.spec)
self.assertEqual('action1', action1_spec.get_name())
self.assertEqual('std.echo', action1_spec.get_base())
self.assertDictEqual({'output': 'Hi'}, action1_spec.get_base_input())
self.assertDictEqual({}, action1_spec.get_input())
db_actions = action_service.update_actions(UPDATED_ACTION_LIST)
# Action 1.
action1_db = self._assert_single_item(db_actions, name='action1')
action1_spec = spec_parser.get_action_spec(action1_db.spec)
self.assertEqual('action1', action1_spec.get_name())
self.assertListEqual([], action1_spec.get_tags())
self.assertEqual('std.echo', action1_spec.get_base())
self.assertDictEqual({'output': 'Hi'}, action1_spec.get_base_input())
self.assertIn('param1', action1_spec.get_input())
self.assertIs(
action1_spec.get_input().get('param1'),
utils.NotDefined
)
def _prepare_input(self, input_dict):
if self._prepared_input is not None:
return self._prepared_input
base_input_dict = input_dict
for action_def in self.adhoc_action_defs:
action_spec = spec_parser.get_action_spec(action_def.spec)
for k, v in action_spec.get_input().items():
if (k not in base_input_dict
or base_input_dict[k] is utils.NotDefined):
base_input_dict[k] = v
base_input_expr = action_spec.get_base_input()
if base_input_expr:
wf_ex = (self.task_ex.workflow_execution
if self.task_ex else None)
ctx_view = data_flow.ContextView(
base_input_dict, self.task_ctx,
data_flow.get_workflow_environment_dict(wf_ex),
self.wf_ctx)
base_input_dict = expr.evaluate_recursively(
base_input_expr, ctx_view)
else:
base_input_dict = {}
self._prepared_input = super(AdHocAction,
self)._prepare_input(base_input_dict)
return self._prepared_input
def __init__(self, action_def, action_ex=None, task_ex=None, task_ctx=None,
wf_ctx=None):
self.action_spec = spec_parser.get_action_spec(action_def.spec)
base_action_def = _find_action_definition_by_name(
self.action_spec.get_base()
)
if not base_action_def:
raise exc.InvalidActionException(
"Failed to find action [action_name=%s]" %
self.action_spec.get_base()
)
base_action_def = self._gather_base_actions(
action_def,
base_action_def
)
super(AdHocAction, self).__init__(
base_action_def,
action_ex,
task_ex
)
self.adhoc_action_def = action_def
self.task_ctx = task_ctx or {}
self.wf_ctx = wf_ctx or {}
def __init__(self, action_def, action_ex=None, task_ex=None, task_ctx=None,
wf_ctx=None):
self.action_spec = spec_parser.get_action_spec(action_def.spec)
base_action_def = lookup_utils.find_action_definition_by_name(
self.action_spec.get_base()
)
if not base_action_def:
raise exc.InvalidActionException(
"Failed to find action [action_name=%s]" %
self.action_spec.get_base()
)
base_action_def = self._gather_base_actions(
action_def,
base_action_def
)
super(AdHocAction, self).__init__(
base_action_def,
action_ex,
task_ex
)
self.adhoc_action_def = action_def
self.task_ctx = task_ctx or {}
self.wf_ctx = wf_ctx or {}
def _prepare_input(self, input_dict):
base_input_dict = input_dict
for action_def in self.adhoc_action_defs:
action_spec = spec_parser.get_action_spec(action_def.spec)
for k, v in action_spec.get_input().items():
if (k not in base_input_dict or
base_input_dict[k] is utils.NotDefined):
base_input_dict[k] = v
base_input_expr = action_spec.get_base_input()
if base_input_expr:
ctx_view = data_flow.ContextView(
base_input_dict,
self.task_ctx,
self.wf_ctx
)
base_input_dict = expr.evaluate_recursively(
base_input_expr,
ctx_view
)
else:
base_input_dict = {}
return super(AdHocAction, self)._prepare_input(base_input_dict)
def __init__(self, action_def, action_ex=None, task_ex=None, task_ctx=None,
wf_ctx=None):
self.action_spec = spec_parser.get_action_spec(action_def.spec)
try:
base_action_def = db_api.get_action_definition(
self.action_spec.get_base()
)
except exc.DBEntityNotFoundError:
raise exc.InvalidActionException(
"Failed to find action [action_name=%s]" %
self.action_spec.get_base()
)
base_action_def = self._gather_base_actions(
action_def, base_action_def
)
super(AdHocAction, self).__init__(
base_action_def,
action_ex,
task_ex
)
self.adhoc_action_def = action_def
self.task_ctx = task_ctx or {}
self.wf_ctx = wf_ctx or {}
def __init__(self, action_def):
super(AdHocActionDescriptor,
self).__init__(action_def.name, action_def.description,
action_def.input or '', action_def.namespace,
action_def.project_id, action_def.scope)
self._definition = action_def.definition
self._spec = spec_parser.get_action_spec(action_def.spec)
self._action_def = action_def
def test_create_actions(self):
db_actions = action_service.create_actions(ACTION_LIST)
self.assertEqual(2, len(db_actions))
# Action 1.
action1_db = self._assert_single_item(db_actions, name='action1')
action1_spec = spec_parser.get_action_spec(action1_db.spec)
self.assertEqual('action1', action1_spec.get_name())
self.assertListEqual(['test', 'v2'], action1_spec.get_tags())
self.assertEqual('std.echo', action1_spec.get_base())
self.assertDictEqual({'output': 'Hi'}, action1_spec.get_base_input())
# Action 2.
action2_db = self._assert_single_item(db_actions, name='action2')
action2_spec = spec_parser.get_action_spec(action2_db.spec)
self.assertEqual('action2', action2_spec.get_name())
self.assertEqual('std.echo', action1_spec.get_base())
self.assertDictEqual({'output': 'Hey'}, action2_spec.get_base_input())
def test_create_actions(self):
db_actions = action_service.create_actions(ACTION_LIST)
self.assertEqual(2, len(db_actions))
# Action 1.
action1_db = self._assert_single_item(db_actions, name='action1')
action1_spec = spec_parser.get_action_spec(action1_db.spec)
self.assertEqual('action1', action1_spec.get_name())
self.assertListEqual(['test', 'v2'], action1_spec.get_tags())
self.assertEqual('std.echo', action1_spec.get_base())
self.assertDictEqual({'output': 'Hi'}, action1_spec.get_base_input())
# Action 2.
action2_db = self._assert_single_item(db_actions, name='action2')
action2_spec = spec_parser.get_action_spec(action2_db.spec)
self.assertEqual('action2', action2_spec.get_name())
self.assertEqual('std.echo', action1_spec.get_base())
self.assertDictEqual({'output': 'Hey'}, action2_spec.get_base_input())
def test_create_workbook(self):
namespace = 'test_workbook_service_0123_namespace'
wb_db = wb_service.create_workbook_v2(WORKBOOK, namespace=namespace)
self.assertIsNotNone(wb_db)
self.assertEqual('my_wb', wb_db.name)
self.assertEqual(namespace, wb_db.namespace)
self.assertEqual(WORKBOOK, wb_db.definition)
self.assertIsNotNone(wb_db.spec)
self.assertListEqual(['test'], wb_db.tags)
db_actions = db_api.get_action_definitions(name='my_wb.concat',
namespace=namespace)
self.assertEqual(1, len(db_actions))
# Action.
action_db = self._assert_single_item(db_actions, name='my_wb.concat')
self.assertFalse(action_db.is_system)
action_spec = spec_parser.get_action_spec(action_db.spec)
self.assertEqual('concat', action_spec.get_name())
self.assertEqual('std.echo', action_spec.get_base())
self.assertEqual(ACTION_DEFINITION, action_db.definition)
db_wfs = db_api.get_workflow_definitions()
self.assertEqual(2, len(db_wfs))
# Workflow 1.
wf1_db = self._assert_single_item(db_wfs, name='my_wb.wf1')
wf1_spec = spec_parser.get_workflow_spec(wf1_db.spec)
self.assertEqual('wf1', wf1_spec.get_name())
self.assertEqual('reverse', wf1_spec.get_type())
self.assertListEqual(['wf_test'], wf1_spec.get_tags())
self.assertListEqual(['wf_test'], wf1_db.tags)
self.assertEqual(namespace, wf1_db.namespace)
self.assertEqual(WORKBOOK_WF1_DEFINITION, wf1_db.definition)
# Workflow 2.
wf2_db = self._assert_single_item(db_wfs, name='my_wb.wf2')
wf2_spec = spec_parser.get_workflow_spec(wf2_db.spec)
self.assertEqual('wf2', wf2_spec.get_name())
self.assertEqual('direct', wf2_spec.get_type())
self.assertEqual(namespace, wf2_db.namespace)
self.assertEqual(WORKBOOK_WF2_DEFINITION, wf2_db.definition)
def _prepare_output(self, result):
# In case of error, we don't transform a result.
if not result.is_error():
adhoc_action_spec = spec_parser.get_action_spec(
self.adhoc_action_def.spec)
transformer = adhoc_action_spec.get_output()
if transformer is not None:
result = ml_actions.Result(data=expr.evaluate_recursively(
transformer, result.data),
error=result.error)
return result
def test_create_workbook(self):
namespace = 'test_workbook_service_0123_namespace'
wb_db = wb_service.create_workbook_v2(WORKBOOK, namespace=namespace)
self.assertIsNotNone(wb_db)
self.assertEqual('my_wb', wb_db.name)
self.assertEqual(namespace, wb_db.namespace)
self.assertEqual(WORKBOOK, wb_db.definition)
self.assertIsNotNone(wb_db.spec)
self.assertListEqual(['test'], wb_db.tags)
db_actions = db_api.get_action_definitions(name='my_wb.concat')
self.assertEqual(1, len(db_actions))
# Action.
action_db = self._assert_single_item(db_actions, name='my_wb.concat')
self.assertFalse(action_db.is_system)
action_spec = spec_parser.get_action_spec(action_db.spec)
self.assertEqual('concat', action_spec.get_name())
self.assertEqual('std.echo', action_spec.get_base())
self.assertEqual(ACTION_DEFINITION, action_db.definition)
db_wfs = db_api.get_workflow_definitions()
self.assertEqual(2, len(db_wfs))
# Workflow 1.
wf1_db = self._assert_single_item(db_wfs, name='my_wb.wf1')
wf1_spec = spec_parser.get_workflow_spec(wf1_db.spec)
self.assertEqual('wf1', wf1_spec.get_name())
self.assertEqual('reverse', wf1_spec.get_type())
self.assertListEqual(['wf_test'], wf1_spec.get_tags())
self.assertListEqual(['wf_test'], wf1_db.tags)
self.assertEqual(namespace, wf1_db.namespace)
self.assertEqual(WORKBOOK_WF1_DEFINITION, wf1_db.definition)
# Workflow 2.
wf2_db = self._assert_single_item(db_wfs, name='my_wb.wf2')
wf2_spec = spec_parser.get_workflow_spec(wf2_db.spec)
self.assertEqual('wf2', wf2_spec.get_name())
self.assertEqual('direct', wf2_spec.get_type())
self.assertEqual(namespace, wf2_db.namespace)
self.assertEqual(WORKBOOK_WF2_DEFINITION, wf2_db.definition)
def __init__(self,
action_def,
action_ex=None,
task_ex=None,
task_ctx=None,
wf_ctx=None):
self.action_spec = spec_parser.get_action_spec(action_def.spec)
base_action_def = db_api.get_action_definition(
self.action_spec.get_base())
base_action_def = self._gather_base_actions(action_def,
base_action_def)
super(AdHocAction, self).__init__(base_action_def, action_ex, task_ex)
self.adhoc_action_def = action_def
self.task_ctx = task_ctx or {}
self.wf_ctx = wf_ctx or {}
def _prepare_output(self, result):
# In case of error, we don't transform a result.
if not result.is_error():
for action_def in reversed(self.adhoc_action_defs):
adhoc_action_spec = spec_parser.get_action_spec(
action_def.spec
)
transformer = adhoc_action_spec.get_output()
if transformer is not None:
result = ml_actions.Result(
data=expr.evaluate_recursively(
transformer,
result.data
),
error=result.error
)
return result
def _prepare_input(self, input_dict):
if self._prepared_input is not None:
return self._prepared_input
base_input_dict = input_dict
for action_def in self.adhoc_action_defs:
action_spec = spec_parser.get_action_spec(action_def.spec)
for k, v in action_spec.get_input().items():
if (k not in base_input_dict or
base_input_dict[k] is utils.NotDefined):
base_input_dict[k] = v
base_input_expr = action_spec.get_base_input()
if base_input_expr:
wf_ex = (
self.task_ex.workflow_execution if self.task_ex else None
)
ctx_view = data_flow.ContextView(
base_input_dict,
self.task_ctx,
data_flow.get_workflow_environment_dict(wf_ex),
self.wf_ctx
)
base_input_dict = expr.evaluate_recursively(
base_input_expr,
ctx_view
)
else:
base_input_dict = {}
self._prepared_input = super(AdHocAction, self)._prepare_input(
base_input_dict
)
return self._prepared_input