def test_workflow_spec_cache_update_via_workbook_service(self):
wb_text = """
version: '2.0'
name: wb
workflows:
wf:
tasks:
task1:
action: std.echo output="Echo"
"""
wb_service.create_workbook_v2(wb_text)
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(0, spec_parser.get_wf_definition_spec_cache_size())
wf = db_api.get_workflow_definition('wb.wf')
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf.id, wf.updated_at)
self.assertEqual(1, len(wf_spec.get_tasks()))
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size())
# Now update workflow definition and check that cache is updated too.
wb_text = """
version: '2.0'
name: wb
workflows:
wf:
tasks:
task1:
action: std.echo output="1"
task2:
action: std.echo output="2"
"""
wb_service.update_workbook_v2(wb_text)
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size())
wf = db_api.get_workflow_definition(wf.id)
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf.id, wf.updated_at)
self.assertEqual(2, len(wf_spec.get_tasks()))
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(2, spec_parser.get_wf_definition_spec_cache_size())
def test_workflow_spec_cache_update_via_workflow_service(self):
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.echo output="Echo"
"""
wfs = wf_service.create_workflows(wf_text)
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(0, spec_parser.get_wf_definition_spec_cache_size())
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wfs[0].id,
wfs[0].updated_at
)
self.assertEqual(1, len(wf_spec.get_tasks()))
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size())
# Now update workflow definition and check that cache is updated too.
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.echo output="1"
task2:
action: std.echo output="2"
"""
wfs = wf_service.update_workflows(wf_text)
self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size())
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wfs[0].id,
wfs[0].updated_at
)
self.assertEqual(2, len(wf_spec.get_tasks()))
self.assertEqual(2, spec_parser.get_wf_definition_spec_cache_size())
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
def schedule(self, input_dict, target, index=0, desc='', safe_rerun=False):
assert not self.action_ex
parent_wf_ex = self.task_ex.workflow_execution
parent_wf_spec = spec_parser.get_workflow_spec_by_execution_id(
parent_wf_ex.id)
task_spec = spec_parser.get_task_spec(self.task_ex.spec)
wf_spec_name = task_spec.get_workflow_name()
wf_def = e_utils.resolve_workflow_definition(
parent_wf_ex.workflow_name, parent_wf_spec.get_name(),
wf_spec_name)
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id, wf_def.updated_at)
wf_params = {'task_execution_id': self.task_ex.id, 'index': index}
if 'env' in parent_wf_ex.params:
wf_params['env'] = parent_wf_ex.params['env']
for k, v in list(input_dict.items()):
if k not in wf_spec.get_input():
wf_params[k] = v
del input_dict[k]
wf_handler.start_workflow(wf_def.id, input_dict,
"sub-workflow execution", wf_params)
def start(self, wf_def, input_dict, desc='', params=None):
"""Start workflow.
:param wf_def: Workflow definition.
:param input_dict: Workflow input.
:param desc: Workflow execution description.
:param params: Workflow type specific parameters.
"""
assert not self.wf_ex
# New workflow execution.
self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id, wf_def.updated_at)
wf_trace.info(
self.wf_ex, "Starting workflow [name=%s, input=%s]" %
(wf_def.name, utils.cut(input_dict)))
# TODO(rakhmerov): This call implicitly changes input_dict! Fix it!
# After fix we need to move validation after adding risky fields.
eng_utils.validate_input(wf_def, input_dict, self.wf_spec)
self._create_execution(wf_def, input_dict, desc, params)
self.set_state(states.RUNNING)
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
cmds = wf_ctrl.continue_workflow()
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def __init__(self, wf_def, wf_ex=None):
self.wf_def = wf_def
self.wf_ex = wf_ex
if wf_ex:
# We're processing a workflow that's already in progress.
self.wf_spec = spec_parser.get_workflow_spec_by_execution_id(
wf_ex.id)
else:
# New workflow execution.
self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id, wf_def.updated_at)
def _prepare_test(self, wf_text):
wfs = wf_service.create_workflows(wf_text)
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wfs[0].id, wfs[0].updated_at)
wf_ex = models.WorkflowExecution(id='1-2-3-4',
spec=wf_spec.to_dict(),
state=states.RUNNING,
workflow_id=wfs[0].id)
self.wf_ex = wf_ex
self.wf_spec = wf_spec
return wf_ex
def __init__(self, wf_def, wf_ex=None):
self.wf_def = wf_def
self.wf_ex = wf_ex
if wf_ex:
# We're processing a workflow that's already in progress.
self.wf_spec = spec_parser.get_workflow_spec_by_execution_id(
wf_ex.id
)
else:
# New workflow execution.
self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id,
wf_def.updated_at
)
def _prepare_test(self, wf_text):
wfs = wf_service.create_workflows(wf_text)
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wfs[0].id,
wfs[0].updated_at
)
wf_ex = models.WorkflowExecution(
id='1-2-3-4',
spec=wf_spec.to_dict(),
state=states.RUNNING,
workflow_id=wfs[0].id,
input={},
context={}
)
self.wf_ex = wf_ex
self.wf_spec = wf_spec
return wf_ex
def test_workflow_spec_caching(self):
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.echo output="Echo"
"""
wfs = wf_service.create_workflows(wf_text)
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(0, spec_parser.get_wf_definition_spec_cache_size())
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wfs[0].id, wfs[0].updated_at)
self.assertIsNotNone(wf_spec)
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size())
def create_event_trigger(name,
exchange,
topic,
event,
workflow_id,
workflow_input=None,
workflow_params=None):
with db_api.transaction():
wf_def = db_api.get_workflow_definition_by_id(workflow_id)
eng_utils.validate_input(
wf_def, workflow_input or {},
parser.get_workflow_spec_by_definition_id(wf_def.id,
wf_def.updated_at))
values = {
'name': name,
'workflow_id': workflow_id,
'workflow_input': workflow_input or {},
'workflow_params': workflow_params or {},
'exchange': exchange,
'topic': topic,
'event': event,
}
security.add_trust_id(values)
trig = db_api.create_event_trigger(values)
trigs = db_api.get_event_triggers(insecure=True,
exchange=exchange,
topic=topic)
events = [t.event for t in trigs]
# NOTE(kong): Send RPC message within the db transaction, rollback if
# any error occurs.
rpc.get_event_engine_client().create_event_trigger(
trig.to_dict(), events)
return trig
def create_event_trigger(name, exchange, topic, event, workflow_id,
workflow_input=None, workflow_params=None):
with db_api.transaction():
wf_def = db_api.get_workflow_definition_by_id(workflow_id)
eng_utils.validate_input(
wf_def,
workflow_input or {},
parser.get_workflow_spec_by_definition_id(
wf_def.id,
wf_def.updated_at
)
)
values = {
'name': name,
'workflow_id': workflow_id,
'workflow_input': workflow_input or {},
'workflow_params': workflow_params or {},
'exchange': exchange,
'topic': topic,
'event': event,
}
security.add_trust_id(values)
trig = db_api.create_event_trigger(values)
trigs = db_api.get_event_triggers(insecure=True, exchange=exchange,
topic=topic)
events = [t.event for t in trigs]
# NOTE(kong): Send RPC message within the db transaction, rollback if
# any error occurs.
rpc.get_event_engine_client().create_event_trigger(
trig.to_dict(),
events
)
return trig
def test_workflow_spec_caching(self):
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.echo output="Echo"
"""
wfs = wf_service.create_workflows(wf_text)
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(0, spec_parser.get_wf_definition_spec_cache_size())
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wfs[0].id,
wfs[0].updated_at
)
self.assertIsNotNone(wf_spec)
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size())
def test_workflow_spec_cache_update_via_workbook_service(self):
wb_text = """
version: '2.0'
name: wb
workflows:
wf:
tasks:
task1:
action: std.echo output="Echo"
"""
wb_service.create_workbook_v2(wb_text)
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(0, spec_parser.get_wf_definition_spec_cache_size())
wf = db_api.get_workflow_definition('wb.wf')
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf.id,
wf.updated_at
)
self.assertEqual(1, len(wf_spec.get_tasks()))
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size())
# Now update workflow definition and check that cache is updated too.
wb_text = """
version: '2.0'
name: wb
workflows:
wf:
tasks:
task1:
action: std.echo output="1"
task2:
action: std.echo output="2"
"""
wb_service.update_workbook_v2(wb_text)
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size())
wf = db_api.get_workflow_definition(wf.id)
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf.id,
wf.updated_at
)
self.assertEqual(2, len(wf_spec.get_tasks()))
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(2, spec_parser.get_wf_definition_spec_cache_size())
def test_cache_workflow_spec_by_execution_id(self):
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.echo output="Echo"
"""
wfs = wf_service.create_workflows(wf_text)
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(0, spec_parser.get_wf_definition_spec_cache_size())
wf_def = wfs[0]
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id,
wf_def.updated_at
)
self.assertEqual(1, len(wf_spec.get_tasks()))
self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size())
self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size())
wf_ex = db_api.create_workflow_execution({
'id': '1-2-3-4',
'name': 'wf',
'workflow_id': wf_def.id,
'spec': wf_spec.to_dict(),
'state': states.RUNNING
})
# Check that we can get a valid spec by execution id.
wf_spec_by_exec_id = spec_parser.get_workflow_spec_by_execution_id(
wf_ex.id
)
self.assertEqual(1, len(wf_spec_by_exec_id.get_tasks()))
# Now update workflow definition and check that cache is updated too.
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.echo output="1"
task2:
action: std.echo output="2"
"""
wfs = wf_service.update_workflows(wf_text)
self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size())
wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wfs[0].id,
wfs[0].updated_at
)
self.assertEqual(2, len(wf_spec.get_tasks()))
self.assertEqual(2, spec_parser.get_wf_definition_spec_cache_size())
self.assertEqual(1, spec_parser.get_wf_execution_spec_cache_size())
# Now finally update execution cache and check that we can
# get a valid spec by execution id.
spec_parser.cache_workflow_spec_by_execution_id(wf_ex.id, wf_spec)
wf_spec_by_exec_id = spec_parser.get_workflow_spec_by_execution_id(
wf_ex.id
)
self.assertEqual(2, len(wf_spec_by_exec_id.get_tasks()))
def __init__(self, wf_def, wf_ex=None):
self.wf_def = wf_def
self.wf_ex = wf_ex
self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id, wf_def.updated_at)