def _get_adhoc_action_input(action_def, input_dict, wf_name=None, wf_spec=None): action_spec = spec_parser.get_action_spec(action_def.spec) base_name = action_spec.get_base() action_def = resolve_action_definition( base_name, wf_name if wf_name else None, wf_spec.get_name() if wf_spec else None ) _inject_action_ctx_for_validating(action_def, input_dict) e_utils.validate_input(action_def, input_dict, action_spec) base_input = action_spec.get_base_input() if base_input: input_dict = expr.evaluate_recursively( base_input, input_dict ) else: input_dict = {} return input_dict
def get_action_input(action_name, input_dict, wf_name=None, wf_spec=None): action_def = resolve_action_definition( action_name, wf_name, wf_spec.get_name() if wf_spec else None ) if action_def.action_class: _inject_action_ctx_for_validating(action_def, input_dict) # NOTE(xylan): Don't validate action input if action initialization method # contains ** argument. if '**' not in action_def.input: e_utils.validate_input(action_def, input_dict) if action_def.spec: # Ad-hoc action. return _get_adhoc_action_input( action_def, input_dict, wf_name, wf_spec ) return input_dict
def start(self, wf_def, input_dict, desc='', params=None): """Start workflow. :param wf_def: Workflow definition. :param input_dict: Workflow input. :param desc: Workflow execution description. :param params: Workflow type specific parameters. """ assert not self.wf_ex # New workflow execution. self.wf_spec = spec_parser.get_workflow_spec_by_definition_id( wf_def.id, wf_def.updated_at) wf_trace.info( self.wf_ex, "Starting workflow [name=%s, input=%s]" % (wf_def.name, utils.cut(input_dict))) # TODO(rakhmerov): This call implicitly changes input_dict! Fix it! # After fix we need to move validation after adding risky fields. eng_utils.validate_input(wf_def, input_dict, self.wf_spec) self._create_execution(wf_def, input_dict, desc, params) self.set_state(states.RUNNING) wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec) cmds = wf_ctrl.continue_workflow() dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def validate_input(self, input_dict): engine_utils.validate_input( self.wf_spec.get_input(), input_dict, self.wf_spec.get_name(), self.wf_spec.__class__.__name__ )
def start_workflow(self, wf_name, wf_input, description='', **params): wf_exec_id = None try: params = self._canonize_workflow_params(params) with db_api.transaction(): wf_def = db_api.get_workflow_definition(wf_name) wf_spec = spec_parser.get_workflow_spec(wf_def.spec) eng_utils.validate_input(wf_def, wf_input, wf_spec) wf_ex = self._create_workflow_execution( wf_def, wf_spec, wf_input, description, params) wf_exec_id = wf_ex.id wf_trace.info(wf_ex, "Starting workflow: '%s'" % wf_name) wf_ctrl = wf_base.WorkflowController.get_controller( wf_ex, wf_spec) self._dispatch_workflow_commands(wf_ex, wf_ctrl.continue_workflow()) return wf_ex.get_clone() except Exception as e: LOG.error("Failed to start workflow '%s' id=%s: %s\n%s", wf_name, wf_exec_id, e, traceback.format_exc()) self._fail_workflow(wf_exec_id, e) raise e
def start(self, input_dict, desc='', params=None): """Start workflow. :param input_dict: Workflow input. :param desc: Workflow execution description. :param params: Workflow type specific parameters. """ assert not self.wf_ex wf_trace.info(self.wf_ex, "Starting workflow: %s" % self.wf_def) # TODO(rakhmerov): This call implicitly changes input_dict! Fix it! # After fix we need to move validation after adding risky fields. eng_utils.validate_input(self.wf_def, input_dict, self.wf_spec) self._create_execution(input_dict, desc, params) self.set_state(states.RUNNING) wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec) cmds = wf_ctrl.continue_workflow() dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def start(self, input_dict, desc='', params=None): """Start workflow. :param input_dict: Workflow input. :param desc: Workflow execution description. :param params: Workflow type specific parameters. """ assert not self.wf_ex wf_trace.info(self.wf_ex, "Starting workflow: %s" % self.wf_def) # TODO(rakhmerov): This call implicitly changes input_dict! Fix it! # After fix we need to move validation after adding risky fields. eng_utils.validate_input(self.wf_def, input_dict, self.wf_spec) self._create_execution(input_dict, desc, params) self.set_state(states.RUNNING) wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec) cmds = wf_ctrl.continue_workflow() dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def validate_input(self, input_dict): engine_utils.validate_input( self.wf_spec.get_input(), input_dict, self.wf_spec.get_name(), self.wf_spec.__class__.__name__ )
def _get_adhoc_action_input(action_def, input_dict, wf_name=None, wf_spec=None): action_spec = spec_parser.get_action_spec(action_def.spec) base_name = action_spec.get_base() action_def = resolve_action_definition( base_name, wf_name if wf_name else None, wf_spec.get_name() if wf_spec else None ) _inject_action_ctx_for_validating(action_def, input_dict) e_utils.validate_input(action_def, input_dict, action_spec) base_input = action_spec.get_base_input() if base_input: input_dict = expr.evaluate_recursively( base_input, input_dict ) else: input_dict = {} return input_dict
def get_action_input(action_name, input_dict, wf_name=None, wf_spec=None): action_def = resolve_action_definition( action_name, wf_name, wf_spec.get_name() if wf_spec else None ) if action_def.action_class: _inject_action_ctx_for_validating(action_def, input_dict) # NOTE(xylan): Don't validate action input if action initialization method # contains ** argument. if '**' not in action_def.input: e_utils.validate_input(action_def, input_dict) if action_def.spec: # Ad-hoc action. return _get_adhoc_action_input( action_def, input_dict, wf_name, wf_spec ) return input_dict
def validate_input(self, input_dict): if self.action_def.action_class: self._inject_action_ctx_for_validating(input_dict) # NOTE(xylan): Don't validate action input if action initialization # method contains ** argument. if '**' not in self.action_def.input: e_utils.validate_input(self.action_def, input_dict)
def validate_input(self, input_dict): if self.action_def.action_class: self._inject_action_ctx_for_validating(input_dict) # NOTE(xylan): Don't validate action input if action initialization # method contains ** argument. if '**' not in self.action_def.input: e_utils.validate_input(self.action_def, input_dict)
def create_cron_trigger(name, workflow_name, workflow_input, workflow_params=None, pattern=None, first_time=None, count=None, start_time=None, workflow_id=None): if not start_time: start_time = datetime.datetime.now() if isinstance(first_time, six.string_types): try: first_time = datetime.datetime.strptime( first_time, '%Y-%m-%d %H:%M' ) except ValueError as e: raise exc.InvalidModelException(e.message) validate_cron_trigger_input(pattern, first_time, count) first_utc_time = first_time if first_time: first_second = time.mktime(first_time.timetuple()) first_utc_time = datetime.datetime.utcfromtimestamp(first_second) next_time = first_utc_time if not (pattern or count): count = 1 else: next_time = get_next_execution_time(pattern, start_time) with db_api.transaction(): wf_def = db_api.get_workflow_definition( workflow_id if workflow_id else workflow_name ) eng_utils.validate_input( wf_def, workflow_input or {}, parser.get_workflow_spec(wf_def.spec) ) values = { 'name': name, 'pattern': pattern, 'first_execution_time': first_utc_time, 'next_execution_time': next_time, 'remaining_executions': count, 'workflow_name': wf_def.name, 'workflow_id': wf_def.id, 'workflow_input': workflow_input or {}, 'workflow_params': workflow_params or {}, 'scope': 'private' } security.add_trust_id(values) trig = db_api.create_cron_trigger(values) return trig
def validate_input(self, input_dict): if self.action_def.action_class: self._inject_action_ctx_for_validating(input_dict) # TODO(rakhmerov): I'm not sure what this is for. # NOTE(xylan): Don't validate action input if action initialization # method contains ** argument. if '**' not in self.action_def.input: e_utils.validate_input(self.action_def, input_dict)
def validate_input(self, input_dict): if self.action_def.action_class: self._inject_action_ctx_for_validating(input_dict) # TODO(rakhmerov): I'm not sure what this is for. # NOTE(xylan): Don't validate action input if action initialization # method contains ** argument. if '**' not in self.action_def.input: e_utils.validate_input(self.action_def, input_dict)
def validate_input(self, input_dict): expected_input = self.action_spec.get_input() engine_utils.validate_input(expected_input, input_dict, self.adhoc_action_def.name, self.action_spec.__class__.__name__) super(AdHocAction, self).validate_input(self._prepare_input(input_dict))
def validate_input(self, input_dict): e_utils.validate_input( self.adhoc_action_def, input_dict, self.action_spec ) super(AdHocAction, self).validate_input( self._prepare_input(input_dict) )
def validate_input(self, input_dict): e_utils.validate_input( self.adhoc_action_def, input_dict, self.action_spec ) super(AdHocAction, self).validate_input( self._prepare_input(input_dict) )
def validate_input(self, input_dict): # NOTE(kong): Don't validate action input if action initialization # method contains ** argument. if '**' in self.action_def.input: return expected_input = utils.get_dict_from_string(self.action_def.input) engine_utils.validate_input(expected_input, input_dict, self.action_def.name, self.action_def.action_class)
def validate_input(self, input_dict): expected_input = self.action_spec.get_input() engine_utils.validate_input( expected_input, input_dict, self.adhoc_action_def.name, self.action_spec.__class__.__name__ ) super(AdHocAction, self).validate_input( self._prepare_input(input_dict) )
def validate_input(self, input_dict): # NOTE(kong): Don't validate action input if action initialization # method contains ** argument. if '**' in self.action_def.input: return expected_input = utils.get_dict_from_string(self.action_def.input) engine_utils.validate_input( expected_input, input_dict, self.action_def.name, self.action_def.action_class )
def create_workflow_execution(wf_identifier, wf_input, description, params): params = canonize_workflow_params(params) wf_def = db_api.get_workflow_definition(wf_identifier) wf_spec = spec_parser.get_workflow_spec(wf_def.spec) eng_utils.validate_input(wf_def, wf_input, wf_spec) wf_ex = _create_workflow_execution(wf_def, wf_spec, wf_input, description, params) wf_trace.info(wf_ex, "Starting workflow: '%s'" % wf_identifier) return wf_ex.id
def create_event_trigger(name, exchange, topic, event, workflow_id, scope='private', workflow_input=None, workflow_params=None): with db_api.transaction(): wf_def = db_api.get_workflow_definition_by_id(workflow_id) wf_spec = parser.get_workflow_spec_by_definition_id( wf_def.id, wf_def.updated_at ) # TODO(rakhmerov): Use Workflow object here instead of utils. eng_utils.validate_input( wf_spec.get_input(), workflow_input, wf_spec.get_name(), wf_spec.__class__.__name__ ) values = { 'name': name, 'workflow_id': workflow_id, 'workflow_input': workflow_input or {}, 'workflow_params': workflow_params or {}, 'exchange': exchange, 'topic': topic, 'event': event, 'scope': scope, } security.add_trust_id(values) trig = db_api.create_event_trigger(values) trigs = db_api.get_event_triggers(insecure=True, exchange=exchange, topic=topic) events = [t.event for t in trigs] # NOTE(kong): Send RPC message within the db transaction, rollback if # any error occurs. trig_dict = trig.to_dict() trig_dict['workflow_namespace'] = wf_def.namespace rpc.get_event_engine_client().create_event_trigger( trig_dict, events ) return trig
def create_event_trigger(name, exchange, topic, event, workflow_id, scope='private', workflow_input=None, workflow_params=None): with db_api.transaction(): wf_def = db_api.get_workflow_definition_by_id(workflow_id) wf_spec = parser.get_workflow_spec_by_definition_id( wf_def.id, wf_def.updated_at) # TODO(rakhmerov): Use Workflow object here instead of utils. eng_utils.validate_input(wf_spec.get_input(), workflow_input, wf_spec.get_name(), wf_spec.__class__.__name__) values = { 'name': name, 'workflow_id': workflow_id, 'workflow_input': workflow_input or {}, 'workflow_params': workflow_params or {}, 'exchange': exchange, 'topic': topic, 'event': event, 'scope': scope, } security.add_trust_id(values) trig = db_api.create_event_trigger(values) trigs = db_api.get_event_triggers(insecure=True, exchange=exchange, topic=topic) events = [t.event for t in trigs] # NOTE(kong): Send RPC message within the db transaction, rollback if # any error occurs. trig_dict = trig.to_dict() trig_dict['workflow_namespace'] = wf_def.namespace rpc.get_event_engine_client().create_event_trigger(trig_dict, events) return trig
def create_workflow_execution(wf_identifier, wf_input, description, params): params = canonize_workflow_params(params) wf_def = db_api.get_workflow_definition(wf_identifier) wf_spec = spec_parser.get_workflow_spec(wf_def.spec) eng_utils.validate_input(wf_def, wf_input, wf_spec) wf_ex = _create_workflow_execution( wf_def, wf_spec, wf_input, description, params ) wf_trace.info(wf_ex, "Starting workflow: '%s'" % wf_identifier) return wf_ex.id
def _get_action_input(wf_spec, task_ex, task_spec, ctx): input_dict = expr.evaluate_recursively(task_spec.get_input(), ctx) action_spec_name = task_spec.get_action_name() action_def = e_utils.resolve_action_definition( task_ex.workflow_name, wf_spec.get_name(), action_spec_name ) input_dict = utils.merge_dicts( input_dict, _get_action_defaults(task_ex, task_spec), overwrite=False ) if action_def.spec: # Ad-hoc action. action_spec = spec_parser.get_action_spec(action_def.spec) base_name = action_spec.get_base() action_def = e_utils.resolve_action_definition( task_ex.workflow_name, wf_spec.get_name(), base_name ) e_utils.validate_input(action_def, action_spec, input_dict) base_input = action_spec.get_base_input() if base_input: input_dict = expr.evaluate_recursively( base_input, input_dict ) else: input_dict = {} return input_dict
def start_workflow(self, wf_name, wf_input, description='', **params): wf_exec_id = None try: params = self._canonize_workflow_params(params) with db_api.transaction(): wf_def = db_api.get_workflow_definition(wf_name) wf_spec = spec_parser.get_workflow_spec(wf_def.spec) eng_utils.validate_input(wf_def, wf_input, wf_spec) wf_ex = self._create_workflow_execution( wf_def, wf_spec, wf_input, description, params ) wf_exec_id = wf_ex.id wf_trace.info(wf_ex, "Starting workflow: '%s'" % wf_name) wf_ctrl = wf_base.WorkflowController.get_controller( wf_ex, wf_spec ) self._dispatch_workflow_commands( wf_ex, wf_ctrl.continue_workflow() ) return wf_ex.get_clone() except Exception as e: LOG.error( "Failed to start workflow '%s' id=%s: %s\n%s", wf_name, wf_exec_id, e, traceback.format_exc() ) self._fail_workflow(wf_exec_id, e) raise e
def create_event_trigger(name, exchange, topic, event, workflow_id, workflow_input=None, workflow_params=None): with db_api.transaction(): wf_def = db_api.get_workflow_definition_by_id(workflow_id) eng_utils.validate_input( wf_def, workflow_input or {}, parser.get_workflow_spec_by_definition_id( wf_def.id, wf_def.updated_at ) ) values = { 'name': name, 'workflow_id': workflow_id, 'workflow_input': workflow_input or {}, 'workflow_params': workflow_params or {}, 'exchange': exchange, 'topic': topic, 'event': event, } security.add_trust_id(values) trig = db_api.create_event_trigger(values) trigs = db_api.get_event_triggers(insecure=True, exchange=exchange, topic=topic) events = [t.event for t in trigs] # NOTE(kong): Send RPC message within the db transaction, rollback if # any error occurs. rpc.get_event_engine_client().create_event_trigger( trig.to_dict(), events ) return trig
def create_event_trigger(name, exchange, topic, event, workflow_id, workflow_input=None, workflow_params=None): with db_api.transaction(): wf_def = db_api.get_workflow_definition_by_id(workflow_id) eng_utils.validate_input( wf_def, workflow_input or {}, parser.get_workflow_spec_by_definition_id(wf_def.id, wf_def.updated_at)) values = { 'name': name, 'workflow_id': workflow_id, 'workflow_input': workflow_input or {}, 'workflow_params': workflow_params or {}, 'exchange': exchange, 'topic': topic, 'event': event, } security.add_trust_id(values) trig = db_api.create_event_trigger(values) trigs = db_api.get_event_triggers(insecure=True, exchange=exchange, topic=topic) events = [t.event for t in trigs] # NOTE(kong): Send RPC message within the db transaction, rollback if # any error occurs. rpc.get_event_engine_client().create_event_trigger( trig.to_dict(), events) return trig
def create_delay_tolerant_workload(name, workflow_name, workflow_input, workflow_params=None, deadline=None, job_duration=None, workflow_id=None): try: deadline = date_parser.parse(deadline) except ValueError as e: raise exc.InvalidModelException(e.message) if deadline < datetime.datetime.now() + datetime.timedelta(seconds=60): raise exc.InvalidModelException( 'deadline must be at least 1 minute in the future.' ) with db_api.transaction(): wf_def = db_api.get_workflow_definition( workflow_id if workflow_id else workflow_name ) eng_utils.validate_input( wf_def, workflow_input or {}, parser.get_workflow_spec(wf_def.spec) ) values = { 'name': name, 'deadline': deadline, 'job_duration': job_duration, 'workflow_name': wf_def.name, 'workflow_id': wf_def.id, 'workflow_input': workflow_input or {}, 'workflow_params': workflow_params or {}, 'scope': 'private', 'executed': False } security.add_trust_id(values) dtw = db_api.create_delay_tolerant_workload(values) return dtw
def create_cron_trigger(name, workflow_name, workflow_input, workflow_params=None, pattern=None, first_time=None, count=None, start_time=None, workflow_id=None): if not start_time: start_time = datetime.datetime.utcnow() if isinstance(first_time, six.string_types): try: first_time = datetime.datetime.strptime(first_time, '%Y-%m-%d %H:%M') except ValueError as e: raise exc.InvalidModelException(str(e)) validate_cron_trigger_input(pattern, first_time, count) if first_time: next_time = first_time if not (pattern or count): count = 1 else: next_time = get_next_execution_time(pattern, start_time) with db_api.transaction(): wf_def = db_api.get_workflow_definition( workflow_id if workflow_id else workflow_name) wf_spec = parser.get_workflow_spec_by_definition_id( wf_def.id, wf_def.updated_at) # TODO(rakhmerov): Use Workflow object here instead of utils. eng_utils.validate_input(wf_spec.get_input(), workflow_input, wf_spec.get_name(), wf_spec.__class__.__name__) trigger_parameters = { 'name': name, 'pattern': pattern, 'first_execution_time': first_time, 'next_execution_time': next_time, 'remaining_executions': count, 'workflow_name': wf_def.name, 'workflow_id': wf_def.id, 'workflow_input': workflow_input or {}, 'workflow_params': workflow_params or {}, 'scope': 'private' } security.add_trust_id(trigger_parameters) try: trig = db_api.create_cron_trigger(trigger_parameters) except Exception: # Delete trust before raising exception. security.delete_trust(trigger_parameters.get('trust_id')) raise return trig
def create_cron_trigger(name, workflow_name, workflow_input, workflow_params=None, pattern=None, first_time=None, count=None, start_time=None, workflow_id=None): if not start_time: start_time = datetime.datetime.utcnow() if isinstance(first_time, six.string_types): try: first_time = datetime.datetime.strptime( first_time, '%Y-%m-%d %H:%M' ) except ValueError as e: raise exc.InvalidModelException(str(e)) validate_cron_trigger_input(pattern, first_time, count) if first_time: next_time = first_time if not (pattern or count): count = 1 else: next_time = get_next_execution_time(pattern, start_time) with db_api.transaction(): wf_def = db_api.get_workflow_definition( workflow_id if workflow_id else workflow_name ) wf_spec = parser.get_workflow_spec_by_definition_id( wf_def.id, wf_def.updated_at ) # TODO(rakhmerov): Use Workflow object here instead of utils. eng_utils.validate_input( wf_spec.get_input(), workflow_input, wf_spec.get_name(), wf_spec.__class__.__name__ ) trigger_parameters = { 'name': name, 'pattern': pattern, 'first_execution_time': first_time, 'next_execution_time': next_time, 'remaining_executions': count, 'workflow_name': wf_def.name, 'workflow_id': wf_def.id, 'workflow_input': workflow_input or {}, 'workflow_params': workflow_params or {}, 'scope': 'private' } security.add_trust_id(trigger_parameters) try: trig = db_api.create_cron_trigger(trigger_parameters) except Exception: # Delete trust before raising exception. security.delete_trust(trigger_parameters.get('trust_id')) raise return trig