def _get_adhoc_action_input(action_def, input_dict,
wf_name=None, wf_spec=None):
action_spec = spec_parser.get_action_spec(action_def.spec)
base_name = action_spec.get_base()
action_def = resolve_action_definition(
base_name,
wf_name if wf_name else None,
wf_spec.get_name() if wf_spec else None
)
_inject_action_ctx_for_validating(action_def, input_dict)
e_utils.validate_input(action_def, input_dict, action_spec)
base_input = action_spec.get_base_input()
if base_input:
input_dict = expr.evaluate_recursively(
base_input,
input_dict
)
else:
input_dict = {}
return input_dict
def get_action_input(action_name, input_dict, wf_name=None, wf_spec=None):
action_def = resolve_action_definition(
action_name,
wf_name,
wf_spec.get_name() if wf_spec else None
)
if action_def.action_class:
_inject_action_ctx_for_validating(action_def, input_dict)
# NOTE(xylan): Don't validate action input if action initialization method
# contains ** argument.
if '**' not in action_def.input:
e_utils.validate_input(action_def, input_dict)
if action_def.spec:
# Ad-hoc action.
return _get_adhoc_action_input(
action_def,
input_dict,
wf_name,
wf_spec
)
return input_dict
def start(self, wf_def, input_dict, desc='', params=None):
"""Start workflow.
:param wf_def: Workflow definition.
:param input_dict: Workflow input.
:param desc: Workflow execution description.
:param params: Workflow type specific parameters.
"""
assert not self.wf_ex
# New workflow execution.
self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id, wf_def.updated_at)
wf_trace.info(
self.wf_ex, "Starting workflow [name=%s, input=%s]" %
(wf_def.name, utils.cut(input_dict)))
# TODO(rakhmerov): This call implicitly changes input_dict! Fix it!
# After fix we need to move validation after adding risky fields.
eng_utils.validate_input(wf_def, input_dict, self.wf_spec)
self._create_execution(wf_def, input_dict, desc, params)
self.set_state(states.RUNNING)
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
cmds = wf_ctrl.continue_workflow()
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def validate_input(self, input_dict):
engine_utils.validate_input(
self.wf_spec.get_input(),
input_dict,
self.wf_spec.get_name(),
self.wf_spec.__class__.__name__
)
def start_workflow(self, wf_name, wf_input, description='', **params):
wf_exec_id = None
try:
params = self._canonize_workflow_params(params)
with db_api.transaction():
wf_def = db_api.get_workflow_definition(wf_name)
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
eng_utils.validate_input(wf_def, wf_input, wf_spec)
wf_ex = self._create_workflow_execution(
wf_def, wf_spec, wf_input, description, params)
wf_exec_id = wf_ex.id
wf_trace.info(wf_ex, "Starting workflow: '%s'" % wf_name)
wf_ctrl = wf_base.WorkflowController.get_controller(
wf_ex, wf_spec)
self._dispatch_workflow_commands(wf_ex,
wf_ctrl.continue_workflow())
return wf_ex.get_clone()
except Exception as e:
LOG.error("Failed to start workflow '%s' id=%s: %s\n%s", wf_name,
wf_exec_id, e, traceback.format_exc())
self._fail_workflow(wf_exec_id, e)
raise e
def start(self, input_dict, desc='', params=None):
"""Start workflow.
:param input_dict: Workflow input.
:param desc: Workflow execution description.
:param params: Workflow type specific parameters.
"""
assert not self.wf_ex
wf_trace.info(self.wf_ex, "Starting workflow: %s" % self.wf_def)
# TODO(rakhmerov): This call implicitly changes input_dict! Fix it!
# After fix we need to move validation after adding risky fields.
eng_utils.validate_input(self.wf_def, input_dict, self.wf_spec)
self._create_execution(input_dict, desc, params)
self.set_state(states.RUNNING)
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
cmds = wf_ctrl.continue_workflow()
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def start(self, input_dict, desc='', params=None):
"""Start workflow.
:param input_dict: Workflow input.
:param desc: Workflow execution description.
:param params: Workflow type specific parameters.
"""
assert not self.wf_ex
wf_trace.info(self.wf_ex, "Starting workflow: %s" % self.wf_def)
# TODO(rakhmerov): This call implicitly changes input_dict! Fix it!
# After fix we need to move validation after adding risky fields.
eng_utils.validate_input(self.wf_def, input_dict, self.wf_spec)
self._create_execution(input_dict, desc, params)
self.set_state(states.RUNNING)
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
cmds = wf_ctrl.continue_workflow()
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def validate_input(self, input_dict):
engine_utils.validate_input(
self.wf_spec.get_input(),
input_dict,
self.wf_spec.get_name(),
self.wf_spec.__class__.__name__
)
def _get_adhoc_action_input(action_def, input_dict,
wf_name=None, wf_spec=None):
action_spec = spec_parser.get_action_spec(action_def.spec)
base_name = action_spec.get_base()
action_def = resolve_action_definition(
base_name,
wf_name if wf_name else None,
wf_spec.get_name() if wf_spec else None
)
_inject_action_ctx_for_validating(action_def, input_dict)
e_utils.validate_input(action_def, input_dict, action_spec)
base_input = action_spec.get_base_input()
if base_input:
input_dict = expr.evaluate_recursively(
base_input,
input_dict
)
else:
input_dict = {}
return input_dict
def get_action_input(action_name, input_dict, wf_name=None, wf_spec=None):
action_def = resolve_action_definition(
action_name,
wf_name,
wf_spec.get_name() if wf_spec else None
)
if action_def.action_class:
_inject_action_ctx_for_validating(action_def, input_dict)
# NOTE(xylan): Don't validate action input if action initialization method
# contains ** argument.
if '**' not in action_def.input:
e_utils.validate_input(action_def, input_dict)
if action_def.spec:
# Ad-hoc action.
return _get_adhoc_action_input(
action_def,
input_dict,
wf_name,
wf_spec
)
return input_dict
def validate_input(self, input_dict):
if self.action_def.action_class:
self._inject_action_ctx_for_validating(input_dict)
# NOTE(xylan): Don't validate action input if action initialization
# method contains ** argument.
if '**' not in self.action_def.input:
e_utils.validate_input(self.action_def, input_dict)
def validate_input(self, input_dict):
if self.action_def.action_class:
self._inject_action_ctx_for_validating(input_dict)
# NOTE(xylan): Don't validate action input if action initialization
# method contains ** argument.
if '**' not in self.action_def.input:
e_utils.validate_input(self.action_def, input_dict)
def create_cron_trigger(name, workflow_name, workflow_input,
workflow_params=None, pattern=None, first_time=None,
count=None, start_time=None, workflow_id=None):
if not start_time:
start_time = datetime.datetime.now()
if isinstance(first_time, six.string_types):
try:
first_time = datetime.datetime.strptime(
first_time,
'%Y-%m-%d %H:%M'
)
except ValueError as e:
raise exc.InvalidModelException(e.message)
validate_cron_trigger_input(pattern, first_time, count)
first_utc_time = first_time
if first_time:
first_second = time.mktime(first_time.timetuple())
first_utc_time = datetime.datetime.utcfromtimestamp(first_second)
next_time = first_utc_time
if not (pattern or count):
count = 1
else:
next_time = get_next_execution_time(pattern, start_time)
with db_api.transaction():
wf_def = db_api.get_workflow_definition(
workflow_id if workflow_id else workflow_name
)
eng_utils.validate_input(
wf_def,
workflow_input or {},
parser.get_workflow_spec(wf_def.spec)
)
values = {
'name': name,
'pattern': pattern,
'first_execution_time': first_utc_time,
'next_execution_time': next_time,
'remaining_executions': count,
'workflow_name': wf_def.name,
'workflow_id': wf_def.id,
'workflow_input': workflow_input or {},
'workflow_params': workflow_params or {},
'scope': 'private'
}
security.add_trust_id(values)
trig = db_api.create_cron_trigger(values)
return trig
def validate_input(self, input_dict):
if self.action_def.action_class:
self._inject_action_ctx_for_validating(input_dict)
# TODO(rakhmerov): I'm not sure what this is for.
# NOTE(xylan): Don't validate action input if action initialization
# method contains ** argument.
if '**' not in self.action_def.input:
e_utils.validate_input(self.action_def, input_dict)
def validate_input(self, input_dict):
if self.action_def.action_class:
self._inject_action_ctx_for_validating(input_dict)
# TODO(rakhmerov): I'm not sure what this is for.
# NOTE(xylan): Don't validate action input if action initialization
# method contains ** argument.
if '**' not in self.action_def.input:
e_utils.validate_input(self.action_def, input_dict)
def validate_input(self, input_dict):
expected_input = self.action_spec.get_input()
engine_utils.validate_input(expected_input, input_dict,
self.adhoc_action_def.name,
self.action_spec.__class__.__name__)
super(AdHocAction,
self).validate_input(self._prepare_input(input_dict))
def validate_input(self, input_dict):
e_utils.validate_input(
self.adhoc_action_def,
input_dict,
self.action_spec
)
super(AdHocAction, self).validate_input(
self._prepare_input(input_dict)
)
def validate_input(self, input_dict):
e_utils.validate_input(
self.adhoc_action_def,
input_dict,
self.action_spec
)
super(AdHocAction, self).validate_input(
self._prepare_input(input_dict)
)
def validate_input(self, input_dict):
# NOTE(kong): Don't validate action input if action initialization
# method contains ** argument.
if '**' in self.action_def.input:
return
expected_input = utils.get_dict_from_string(self.action_def.input)
engine_utils.validate_input(expected_input, input_dict,
self.action_def.name,
self.action_def.action_class)
def validate_input(self, input_dict):
expected_input = self.action_spec.get_input()
engine_utils.validate_input(
expected_input,
input_dict,
self.adhoc_action_def.name,
self.action_spec.__class__.__name__
)
super(AdHocAction, self).validate_input(
self._prepare_input(input_dict)
)
def validate_input(self, input_dict):
# NOTE(kong): Don't validate action input if action initialization
# method contains ** argument.
if '**' in self.action_def.input:
return
expected_input = utils.get_dict_from_string(self.action_def.input)
engine_utils.validate_input(
expected_input,
input_dict,
self.action_def.name,
self.action_def.action_class
)
def create_workflow_execution(wf_identifier, wf_input, description, params):
params = canonize_workflow_params(params)
wf_def = db_api.get_workflow_definition(wf_identifier)
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
eng_utils.validate_input(wf_def, wf_input, wf_spec)
wf_ex = _create_workflow_execution(wf_def, wf_spec, wf_input, description,
params)
wf_trace.info(wf_ex, "Starting workflow: '%s'" % wf_identifier)
return wf_ex.id
def create_event_trigger(name, exchange, topic, event, workflow_id,
scope='private', workflow_input=None,
workflow_params=None):
with db_api.transaction():
wf_def = db_api.get_workflow_definition_by_id(workflow_id)
wf_spec = parser.get_workflow_spec_by_definition_id(
wf_def.id,
wf_def.updated_at
)
# TODO(rakhmerov): Use Workflow object here instead of utils.
eng_utils.validate_input(
wf_spec.get_input(),
workflow_input,
wf_spec.get_name(),
wf_spec.__class__.__name__
)
values = {
'name': name,
'workflow_id': workflow_id,
'workflow_input': workflow_input or {},
'workflow_params': workflow_params or {},
'exchange': exchange,
'topic': topic,
'event': event,
'scope': scope,
}
security.add_trust_id(values)
trig = db_api.create_event_trigger(values)
trigs = db_api.get_event_triggers(insecure=True, exchange=exchange,
topic=topic)
events = [t.event for t in trigs]
# NOTE(kong): Send RPC message within the db transaction, rollback if
# any error occurs.
trig_dict = trig.to_dict()
trig_dict['workflow_namespace'] = wf_def.namespace
rpc.get_event_engine_client().create_event_trigger(
trig_dict,
events
)
return trig
def create_event_trigger(name,
exchange,
topic,
event,
workflow_id,
scope='private',
workflow_input=None,
workflow_params=None):
with db_api.transaction():
wf_def = db_api.get_workflow_definition_by_id(workflow_id)
wf_spec = parser.get_workflow_spec_by_definition_id(
wf_def.id, wf_def.updated_at)
# TODO(rakhmerov): Use Workflow object here instead of utils.
eng_utils.validate_input(wf_spec.get_input(), workflow_input,
wf_spec.get_name(),
wf_spec.__class__.__name__)
values = {
'name': name,
'workflow_id': workflow_id,
'workflow_input': workflow_input or {},
'workflow_params': workflow_params or {},
'exchange': exchange,
'topic': topic,
'event': event,
'scope': scope,
}
security.add_trust_id(values)
trig = db_api.create_event_trigger(values)
trigs = db_api.get_event_triggers(insecure=True,
exchange=exchange,
topic=topic)
events = [t.event for t in trigs]
# NOTE(kong): Send RPC message within the db transaction, rollback if
# any error occurs.
trig_dict = trig.to_dict()
trig_dict['workflow_namespace'] = wf_def.namespace
rpc.get_event_engine_client().create_event_trigger(trig_dict, events)
return trig
def create_workflow_execution(wf_identifier, wf_input, description, params):
params = canonize_workflow_params(params)
wf_def = db_api.get_workflow_definition(wf_identifier)
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
eng_utils.validate_input(wf_def, wf_input, wf_spec)
wf_ex = _create_workflow_execution(
wf_def,
wf_spec,
wf_input,
description,
params
)
wf_trace.info(wf_ex, "Starting workflow: '%s'" % wf_identifier)
return wf_ex.id
def _get_action_input(wf_spec, task_ex, task_spec, ctx):
input_dict = expr.evaluate_recursively(task_spec.get_input(), ctx)
action_spec_name = task_spec.get_action_name()
action_def = e_utils.resolve_action_definition(
task_ex.workflow_name,
wf_spec.get_name(),
action_spec_name
)
input_dict = utils.merge_dicts(
input_dict,
_get_action_defaults(task_ex, task_spec),
overwrite=False
)
if action_def.spec:
# Ad-hoc action.
action_spec = spec_parser.get_action_spec(action_def.spec)
base_name = action_spec.get_base()
action_def = e_utils.resolve_action_definition(
task_ex.workflow_name,
wf_spec.get_name(),
base_name
)
e_utils.validate_input(action_def, action_spec, input_dict)
base_input = action_spec.get_base_input()
if base_input:
input_dict = expr.evaluate_recursively(
base_input,
input_dict
)
else:
input_dict = {}
return input_dict
def start_workflow(self, wf_name, wf_input, description='', **params):
wf_exec_id = None
try:
params = self._canonize_workflow_params(params)
with db_api.transaction():
wf_def = db_api.get_workflow_definition(wf_name)
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
eng_utils.validate_input(wf_def, wf_input, wf_spec)
wf_ex = self._create_workflow_execution(
wf_def,
wf_spec,
wf_input,
description,
params
)
wf_exec_id = wf_ex.id
wf_trace.info(wf_ex, "Starting workflow: '%s'" % wf_name)
wf_ctrl = wf_base.WorkflowController.get_controller(
wf_ex,
wf_spec
)
self._dispatch_workflow_commands(
wf_ex,
wf_ctrl.continue_workflow()
)
return wf_ex.get_clone()
except Exception as e:
LOG.error(
"Failed to start workflow '%s' id=%s: %s\n%s",
wf_name, wf_exec_id, e, traceback.format_exc()
)
self._fail_workflow(wf_exec_id, e)
raise e
def create_event_trigger(name, exchange, topic, event, workflow_id,
workflow_input=None, workflow_params=None):
with db_api.transaction():
wf_def = db_api.get_workflow_definition_by_id(workflow_id)
eng_utils.validate_input(
wf_def,
workflow_input or {},
parser.get_workflow_spec_by_definition_id(
wf_def.id,
wf_def.updated_at
)
)
values = {
'name': name,
'workflow_id': workflow_id,
'workflow_input': workflow_input or {},
'workflow_params': workflow_params or {},
'exchange': exchange,
'topic': topic,
'event': event,
}
security.add_trust_id(values)
trig = db_api.create_event_trigger(values)
trigs = db_api.get_event_triggers(insecure=True, exchange=exchange,
topic=topic)
events = [t.event for t in trigs]
# NOTE(kong): Send RPC message within the db transaction, rollback if
# any error occurs.
rpc.get_event_engine_client().create_event_trigger(
trig.to_dict(),
events
)
return trig
def create_event_trigger(name,
exchange,
topic,
event,
workflow_id,
workflow_input=None,
workflow_params=None):
with db_api.transaction():
wf_def = db_api.get_workflow_definition_by_id(workflow_id)
eng_utils.validate_input(
wf_def, workflow_input or {},
parser.get_workflow_spec_by_definition_id(wf_def.id,
wf_def.updated_at))
values = {
'name': name,
'workflow_id': workflow_id,
'workflow_input': workflow_input or {},
'workflow_params': workflow_params or {},
'exchange': exchange,
'topic': topic,
'event': event,
}
security.add_trust_id(values)
trig = db_api.create_event_trigger(values)
trigs = db_api.get_event_triggers(insecure=True,
exchange=exchange,
topic=topic)
events = [t.event for t in trigs]
# NOTE(kong): Send RPC message within the db transaction, rollback if
# any error occurs.
rpc.get_event_engine_client().create_event_trigger(
trig.to_dict(), events)
return trig
def create_delay_tolerant_workload(name, workflow_name, workflow_input,
workflow_params=None, deadline=None,
job_duration=None, workflow_id=None):
try:
deadline = date_parser.parse(deadline)
except ValueError as e:
raise exc.InvalidModelException(e.message)
if deadline < datetime.datetime.now() + datetime.timedelta(seconds=60):
raise exc.InvalidModelException(
'deadline must be at least 1 minute in the future.'
)
with db_api.transaction():
wf_def = db_api.get_workflow_definition(
workflow_id if workflow_id else workflow_name
)
eng_utils.validate_input(
wf_def,
workflow_input or {},
parser.get_workflow_spec(wf_def.spec)
)
values = {
'name': name,
'deadline': deadline,
'job_duration': job_duration,
'workflow_name': wf_def.name,
'workflow_id': wf_def.id,
'workflow_input': workflow_input or {},
'workflow_params': workflow_params or {},
'scope': 'private',
'executed': False
}
security.add_trust_id(values)
dtw = db_api.create_delay_tolerant_workload(values)
return dtw
def create_cron_trigger(name,
workflow_name,
workflow_input,
workflow_params=None,
pattern=None,
first_time=None,
count=None,
start_time=None,
workflow_id=None):
if not start_time:
start_time = datetime.datetime.utcnow()
if isinstance(first_time, six.string_types):
try:
first_time = datetime.datetime.strptime(first_time,
'%Y-%m-%d %H:%M')
except ValueError as e:
raise exc.InvalidModelException(str(e))
validate_cron_trigger_input(pattern, first_time, count)
if first_time:
next_time = first_time
if not (pattern or count):
count = 1
else:
next_time = get_next_execution_time(pattern, start_time)
with db_api.transaction():
wf_def = db_api.get_workflow_definition(
workflow_id if workflow_id else workflow_name)
wf_spec = parser.get_workflow_spec_by_definition_id(
wf_def.id, wf_def.updated_at)
# TODO(rakhmerov): Use Workflow object here instead of utils.
eng_utils.validate_input(wf_spec.get_input(), workflow_input,
wf_spec.get_name(),
wf_spec.__class__.__name__)
trigger_parameters = {
'name': name,
'pattern': pattern,
'first_execution_time': first_time,
'next_execution_time': next_time,
'remaining_executions': count,
'workflow_name': wf_def.name,
'workflow_id': wf_def.id,
'workflow_input': workflow_input or {},
'workflow_params': workflow_params or {},
'scope': 'private'
}
security.add_trust_id(trigger_parameters)
try:
trig = db_api.create_cron_trigger(trigger_parameters)
except Exception:
# Delete trust before raising exception.
security.delete_trust(trigger_parameters.get('trust_id'))
raise
return trig
def create_cron_trigger(name, workflow_name, workflow_input,
workflow_params=None, pattern=None, first_time=None,
count=None, start_time=None, workflow_id=None):
if not start_time:
start_time = datetime.datetime.utcnow()
if isinstance(first_time, six.string_types):
try:
first_time = datetime.datetime.strptime(
first_time,
'%Y-%m-%d %H:%M'
)
except ValueError as e:
raise exc.InvalidModelException(str(e))
validate_cron_trigger_input(pattern, first_time, count)
if first_time:
next_time = first_time
if not (pattern or count):
count = 1
else:
next_time = get_next_execution_time(pattern, start_time)
with db_api.transaction():
wf_def = db_api.get_workflow_definition(
workflow_id if workflow_id else workflow_name
)
wf_spec = parser.get_workflow_spec_by_definition_id(
wf_def.id,
wf_def.updated_at
)
# TODO(rakhmerov): Use Workflow object here instead of utils.
eng_utils.validate_input(
wf_spec.get_input(),
workflow_input,
wf_spec.get_name(),
wf_spec.__class__.__name__
)
trigger_parameters = {
'name': name,
'pattern': pattern,
'first_execution_time': first_time,
'next_execution_time': next_time,
'remaining_executions': count,
'workflow_name': wf_def.name,
'workflow_id': wf_def.id,
'workflow_input': workflow_input or {},
'workflow_params': workflow_params or {},
'scope': 'private'
}
security.add_trust_id(trigger_parameters)
try:
trig = db_api.create_cron_trigger(trigger_parameters)
except Exception:
# Delete trust before raising exception.
security.delete_trust(trigger_parameters.get('trust_id'))
raise
return trig
