Example #1
0
    def notify(self, rpc_ctx, ex_id, data, event, timestamp, publishers):
        """Receives calls over RPC to notify on notification server.

        :param rpc_ctx: RPC request context dictionary.
        :param ex_id: Workflow, task, or action execution id.
        :param data: Dictionary to include in the notification message.
        :param event: Event being notified on.
        :param timestamp: Datetime when this event occurred.
        :param publishers: The list of publishers to send the notification.
        """

        LOG.info(
            "Received RPC request 'notify'[ex_id=%s, event=%s, "
            "timestamp=%s, data=%s, publishers=%s]",
            ex_id,
            event,
            timestamp,
            data,
            utils.cut(publishers)
        )

        self.notifier.notify(
            ex_id,
            data,
            event,
            timestamp,
            publishers
        )
Example #2
0
    def start_workflow(self, rpc_ctx, wf_identifier, wf_namespace,
                       wf_ex_id, wf_input, description, params):
        """Receives calls over RPC to start workflows on engine.

        :param rpc_ctx: RPC request context.
        :param wf_identifier: Workflow definition identifier.
        :param wf_namespace: Workflow namespace.
        :param wf_input: Workflow input.
        :param wf_ex_id: Workflow execution id. If passed, it will be set
            in the new execution object.
        :param description: Workflow execution description.
        :param params: Additional workflow type specific parameters.
        :return: Workflow execution.
        """

        LOG.info(
            "Received RPC request 'start_workflow'[workflow_identifier=%s, "
            "workflow_input=%s, description=%s, params=%s]",
            wf_identifier,
            utils.cut(wf_input),
            description,
            params
        )

        return self.engine.start_workflow(
            wf_identifier,
            wf_namespace,
            wf_ex_id,
            wf_input,
            description,
            **params
        )
Example #3
0
    def start_action(self, rpc_ctx, action_name,
                     action_input, description, params):
        """Receives calls over RPC to start actions on engine.

        :param rpc_ctx: RPC request context.
        :param action_name: name of the Action.
        :param action_input: input dictionary for Action.
        :param description: description of new Action execution.
        :param params: extra parameters to run Action.
        :return: Action execution.
        """
        LOG.info(
            "Received RPC request 'start_action'[name=%s, input=%s, "
            "description=%s, params=%s]",
            action_name,
            utils.cut(action_input),
            description,
            params
        )

        return self.engine.start_action(
            action_name,
            action_input,
            description,
            **params
        )
    def start_workflow(self, rpc_ctx, wf_identifier, wf_namespace,
                       wf_ex_id, wf_input, description, params):
        """Receives calls over RPC to start workflows on engine.

        :param rpc_ctx: RPC request context.
        :param wf_identifier: Workflow definition identifier.
        :param wf_namespace: Workflow namespace.
        :param wf_input: Workflow input.
        :param wf_ex_id: Workflow execution id. If passed, it will be set
            in the new execution object.
        :param description: Workflow execution description.
        :param params: Additional workflow type specific parameters.
        :return: Workflow execution.
        """

        LOG.info(
            "Received RPC request 'start_workflow'[workflow_identifier=%s, "
            "workflow_input=%s, description=%s, params=%s]",
            wf_identifier,
            utils.cut(wf_input),
            description,
            params
        )

        return self.engine.start_workflow(
            wf_identifier,
            wf_namespace,
            wf_ex_id,
            wf_input,
            description,
            **params
        )
    def start_action(self, rpc_ctx, action_name,
                     action_input, description, params):
        """Receives calls over RPC to start actions on engine.

        :param rpc_ctx: RPC request context.
        :param action_name: name of the Action.
        :param action_input: input dictionary for Action.
        :param description: description of new Action execution.
        :param params: extra parameters to run Action.
        :return: Action execution.
        """
        LOG.info(
            "Received RPC request 'start_action'[name=%s, input=%s, "
            "description=%s, params=%s]",
            action_name,
            utils.cut(action_input),
            description,
            params
        )

        return self.engine.start_action(
            action_name,
            action_input,
            description,
            **params
        )
Example #6
0
    def start(self, wf_def, input_dict, desc='', params=None):
        """Start workflow.

        :param wf_def: Workflow definition.
        :param input_dict: Workflow input.
        :param desc: Workflow execution description.
        :param params: Workflow type specific parameters.
        """

        assert not self.wf_ex

        # New workflow execution.
        self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
            wf_def.id, wf_def.updated_at)

        wf_trace.info(
            self.wf_ex, "Starting workflow [name=%s, input=%s]" %
            (wf_def.name, utils.cut(input_dict)))

        # TODO(rakhmerov): This call implicitly changes input_dict! Fix it!
        # After fix we need to move validation after adding risky fields.
        eng_utils.validate_input(wf_def, input_dict, self.wf_spec)

        self._create_execution(wf_def, input_dict, desc, params)

        self.set_state(states.RUNNING)

        wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)

        cmds = wf_ctrl.continue_workflow()

        dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
Example #7
0
    def start(self, wf_def, wf_ex_id, input_dict, desc='', params=None):
        """Start workflow.

        :param wf_def: Workflow definition.
        :param wf_ex_id: Workflow execution id.
        :param input_dict: Workflow input.
        :param desc: Workflow execution description.
        :param params: Workflow type specific parameters.

        :raises
        """

        assert not self.wf_ex

        # New workflow execution.
        self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
            wf_def.id, wf_def.updated_at)

        wf_trace.info(
            self.wf_ex, 'Starting workflow [name=%s, input=%s]' %
            (wf_def.name, utils.cut(input_dict)))

        self.validate_input(input_dict)

        self._create_execution(wf_def, wf_ex_id,
                               self.prepare_input(input_dict), desc, params)

        self.set_state(states.RUNNING)

        wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)

        dispatcher.dispatch_workflow_commands(self.wf_ex,
                                              wf_ctrl.continue_workflow())
Example #8
0
    def put(self, env):
        """Update an environment.

        :param env: Required. Environment structure to update
        """
        acl.enforce('environments:update', context.ctx())

        if not env.name:
            raise exceptions.InputException(
                'Name of the environment is not provided.'
            )

        LOG.debug("Update environment [name=%s, env=%s]", env.name, cut(env))

        definition = json.loads(wsme_pecan.pecan.request.body.decode())
        definition.pop('name')

        self._validate_environment(
            definition,
            ['description', 'variables', 'scope']
        )

        db_model = rest_utils.rest_retry_on_db_error(
            db_api.update_environment
        )(env.name, env.to_dict())

        return resources.Environment.from_db_model(db_model)
Example #9
0
    def run_action(self, rpc_ctx, action_ex_id, action_cls_str,
                   action_cls_attrs, params, safe_rerun, execution_context,
                   timeout):
        """Receives calls over RPC to run action on executor.

        :param timeout: a period of time in seconds after which execution of
            action will be interrupted
        :param execution_context: A dict of values providing information about
            the current execution.
        :param rpc_ctx: RPC request context dictionary.
        :param action_ex_id: Action execution id.
        :param action_cls_str: Action class name.
        :param action_cls_attrs: Action class attributes.
        :param params: Action input parameters.
        :param safe_rerun: Tells if given action can be safely rerun.
        :return: Action result.
        """

        LOG.debug(
            "Received RPC request 'run_action'[action_ex_id=%s, "
            "action_cls_str=%s, action_cls_attrs=%s, params=%s, "
            "timeout=%s]",
            action_ex_id,
            action_cls_str,
            action_cls_attrs,
            utils.cut(params),
            timeout
        )

        redelivered = rpc_ctx.redelivered or False

        try:
            self._aer.add_action_ex_id(action_ex_id)

            res = self.executor.run_action(
                action_ex_id,
                action_cls_str,
                action_cls_attrs,
                params,
                safe_rerun,
                execution_context,
                redelivered,
                timeout=timeout
            )

            LOG.debug(
                "Sending action result to engine"
                " [action_ex_id=%s, action_cls=%s]",
                action_ex_id,
                action_cls_str
            )

            return res
        finally:
            self._aer.remove_action_ex_id(action_ex_id)
    def post(self, env):
        """Create a new environment.

        :param env: Required. Environment structure to create
        """
        acl.enforce('environments:create', context.ctx())

        LOG.debug("Create environment [env=%s]", cut(env))

        self._validate_environment(
            json.loads(wsme_pecan.pecan.request.body.decode()),
            ['name', 'description', 'variables'])

        db_model = db_api.create_environment(env.to_dict())

        return resources.Environment.from_db_model(db_model)
Example #11
0
    def start(self, wf_def, wf_ex_id, input_dict, desc='', params=None):
        """Start workflow.

        :param wf_def: Workflow definition.
        :param wf_ex_id: Workflow execution id.
        :param input_dict: Workflow input.
        :param desc: Workflow execution description.
        :param params: Workflow type specific parameters.

        :raises
        """

        assert not self.wf_ex

        # New workflow execution.
        self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
            wf_def.id,
            wf_def.updated_at
        )

        wf_trace.info(
            self.wf_ex,
            'Starting workflow [name=%s, input=%s]' %
            (wf_def.name, utils.cut(input_dict))
        )

        self.validate_input(input_dict)

        self._create_execution(
            wf_def,
            wf_ex_id,
            self.prepare_input(input_dict),
            desc,
            params
        )

        self.set_state(states.RUNNING)

        # Publish event as soon as state is set to running.
        self.notify(events.WORKFLOW_LAUNCHED)

        wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)

        dispatcher.dispatch_workflow_commands(
            self.wf_ex,
            wf_ctrl.continue_workflow()
        )
    def notify(self, rpc_ctx, ex_id, data, event, timestamp, publishers):
        """Receives calls over RPC to notify on notification server.

        :param rpc_ctx: RPC request context dictionary.
        :param ex_id: Workflow, task, or action execution id.
        :param data: Dictionary to include in the notification message.
        :param event: Event being notified on.
        :param timestamp: Datetime when this event occurred.
        :param publishers: The list of publishers to send the notification.
        """

        LOG.info(
            "Received RPC request 'notify'[ex_id=%s, event=%s, "
            "timestamp=%s, data=%s, publishers=%s]", ex_id, event, timestamp,
            data, utils.cut(publishers))

        self.notifier.notify(ex_id, data, event, timestamp, publishers)
    def run_action(self, rpc_ctx, action_ex_id, action_cls_str,
                   action_cls_attrs, params, safe_rerun, execution_context,
                   timeout):
        """Receives calls over RPC to run action on executor.

        :param timeout: a period of time in seconds after which execution of
            action will be interrupted
        :param execution_context: A dict of values providing information about
            the current execution.
        :param rpc_ctx: RPC request context dictionary.
        :param action_ex_id: Action execution id.
        :param action_cls_str: Action class name.
        :param action_cls_attrs: Action class attributes.
        :param params: Action input parameters.
        :param safe_rerun: Tells if given action can be safely rerun.
        :return: Action result.
        """

        LOG.debug(
            "Received RPC request 'run_action'[action_ex_id=%s, "
            "action_cls_str=%s, action_cls_attrs=%s, params=%s, "
            "timeout=%s]", action_ex_id, action_cls_str, action_cls_attrs,
            utils.cut(params), timeout)

        redelivered = rpc_ctx.redelivered or False

        try:
            self._aer.add_action_ex_id(action_ex_id)

            res = self.executor.run_action(action_ex_id,
                                           action_cls_str,
                                           action_cls_attrs,
                                           params,
                                           safe_rerun,
                                           execution_context,
                                           redelivered,
                                           timeout=timeout)

            LOG.debug(
                "Sending action result to engine"
                " [action_ex_id=%s, action_cls=%s]", action_ex_id,
                action_cls_str)

            return res
        finally:
            self._aer.remove_action_ex_id(action_ex_id)
Example #14
0
    def post(self, env):
        """Create a new environment.

        :param env: Required. Environment structure to create
        """
        acl.enforce('environments:create', context.ctx())

        LOG.debug("Create environment [env=%s]", cut(env))

        self._validate_environment(
            json.loads(wsme_pecan.pecan.request.body.decode()),
            ['name', 'description', 'variables']
        )

        db_model = rest_utils.rest_retry_on_db_error(
            db_api.create_environment
        )(env.to_dict())

        return resources.Environment.from_db_model(db_model)
Example #15
0
    def start_workflow(self, rpc_ctx, workflow_identifier, workflow_input,
                       description, params):
        """Receives calls over RPC to start workflows on engine.

        :param rpc_ctx: RPC request context.
        :param workflow_identifier: Workflow definition identifier.
        :param workflow_input: Workflow input.
        :param description: Workflow execution description.
        :param params: Additional workflow type specific parameters.
        :return: Workflow execution.
        """

        LOG.info(
            "Received RPC request 'start_workflow'[rpc_ctx=%s,"
            " workflow_identifier=%s, workflow_input=%s, description=%s, "
            "params=%s]" % (rpc_ctx, workflow_identifier,
                            utils.cut(workflow_input), description, params))

        return self.engine.start_workflow(workflow_identifier, workflow_input,
                                          description, **params)
    def put(self, env):
        """Update an environment.

        :param env: Required. Environment structure to update
        """
        acl.enforce('environments:update', context.ctx())

        if not env.name:
            raise exceptions.InputException(
                'Name of the environment is not provided.')

        LOG.debug("Update environment [name=%s, env=%s]", env.name, cut(env))

        definition = json.loads(wsme_pecan.pecan.request.body.decode())
        definition.pop('name')

        self._validate_environment(definition,
                                   ['description', 'variables', 'scope'])

        db_model = db_api.update_environment(env.name, env.to_dict())

        return resources.Environment.from_db_model(db_model)
Example #17
0
    def run_action(self, rpc_ctx, action_ex_id, action_cls_str,
                   action_cls_attrs, params, safe_rerun):
        """Receives calls over RPC to run action on executor.

        :param rpc_ctx: RPC request context dictionary.
        :param action_ex_id: Action execution id.
        :param action_cls_str: Action class name.
        :param action_cls_attrs: Action class attributes.
        :param params: Action input parameters.
        :param safe_rerun: Tells if given action can be safely rerun.
        :return: Action result.
        """

        LOG.info("Received RPC request 'run_action'[action_ex_id=%s, "
                 "action_cls_str=%s, action_cls_attrs=%s, params=%s]" %
                 (action_ex_id, action_cls_str, action_cls_attrs,
                  utils.cut(params)))

        redelivered = rpc_ctx.redelivered or False

        return self.executor.run_action(action_ex_id, action_cls_str,
                                        action_cls_attrs, params, safe_rerun,
                                        redelivered)
Example #18
0
 def test_cut_dict_for_state_info(self):
     d = {}
     for i in range(2000):
         d[i] = {'value': 'This is a string that exceeds 35 characters'}
     s = utils.cut(d, 65500)
     assert len(s) <= 65500, str(len(s)) + ' : ' + s[:-30]
Example #19
0
 def test_cut_dict_with_large_dict_of_int(self):
     d = {}
     for i in range(65535):
         d[i] = i
     s = utils.cut(d, 65535)
     assert len(s) <= 65535, len(s)
Example #20
0
        )


for cls in utils.iter_subclasses(Execution):
    event.listen(
        # Catch and trim Execution.state_info to always fit allocated size.
        # Note that the limit is 65500 which is less than 65535 (2^16 -1).
        # The reason is that utils.cut() is not exactly accurate in case if
        # the value is not a string, but, for example, a dictionary. If we
        # limit it exactly to 65535 then once in a while it may go slightly
        # beyond the allowed maximum size. It may depend on the order of
        # keys in a string representation and other things that are hidden
        # inside utils.cut_dict() method.
        cls.state_info,
        'set',
        lambda t, v, o, i: utils.cut(v, 65500),
        retval=True
    )


# Many-to-one for 'ActionExecution' and 'TaskExecution'.

ActionExecution.task_execution_id = sa.Column(
    sa.String(36),
    sa.ForeignKey(TaskExecution.id, ondelete='CASCADE'),
    nullable=True
)

TaskExecution.action_executions = relationship(
    ActionExecution,
    backref=backref('task_execution', remote_side=[TaskExecution.id]),
Example #21
0
    @property
    def executions(self):
        return (
            self.action_executions
            if not self.spec.get('workflow')
            else self.workflow_executions
        )


for cls in utils.iter_subclasses(Execution):
    event.listen(
        # Catch and trim Execution.state_info to always fit allocated size.
        cls.state_info,
        'set',
        lambda t, v, o, i: utils.cut(v, 65532),
        retval=True
    )


# Many-to-one for 'ActionExecution' and 'TaskExecution'.

ActionExecution.task_execution_id = sa.Column(
    sa.String(36),
    sa.ForeignKey(TaskExecution.id, ondelete='CASCADE'),
    nullable=True
)

TaskExecution.action_executions = relationship(
    ActionExecution,
    backref=backref('task_execution', remote_side=[TaskExecution.id]),
Example #22
0
        def _result_msg():
            if state == states.ERROR:
                return "error = %s" % utils.cut(result.error)

            return "result = %s" % utils.cut(result.data)
Example #23
0
 def test_cut_dict_with_large_dict_of_dict(self):
     d = {}
     for i in range(65535):
         d[i] = {'value': str(i)}
     s = utils.cut(d, 65535)
     assert len(s) <= 65535, len(s)
Example #24
0
        def _result_msg():
            if state == states.ERROR:
                return "error = %s" % utils.cut(result.error)

            return "result = %s" % utils.cut(result.data)
Example #25
0
    # Whether the task is fully processed (publishing and calculating commands
    # after it). It allows to simplify workflow controller implementations
    # significantly.
    processed = sa.Column(sa.BOOLEAN, default=False)

    # Data Flow properties.
    in_context = sa.Column(st.JsonLongDictType())
    published = sa.Column(st.JsonLongDictType())


for cls in utils.iter_subclasses(Execution):
    event.listen(
        # Catch and trim Execution.state_info to always fit allocated size.
        cls.state_info,
        'set',
        lambda t, v, o, i: utils.cut(v, 65532),
        retval=True)


def validate_long_type_length(cls, field_name, value):
    """Makes sure the value does not exceeds the maximum size."""
    if value:
        # Get the configured limit.
        size_limit_kb = cfg.CONF.engine.execution_field_size_limit_kb

        # If the size is unlimited.
        if size_limit_kb < 0:
            return

        size_kb = int(sys.getsizeof(str(value)) / 1024)
Example #26
0
 def cut_repr(self):
     return 'Result [data=%s, error=%s, cancel=%s]' % (utils.cut(
         self.data), utils.cut(self.error), str(self.cancel))
Example #27
0
    # Whether the task is fully processed (publishing and calculating commands
    # after it). It allows to simplify workflow controller implementations
    # significantly.
    processed = sa.Column(sa.BOOLEAN, default=False)

    # Data Flow properties.
    in_context = sa.Column(st.JsonLongDictType())
    published = sa.Column(st.JsonLongDictType())


for cls in utils.iter_subclasses(Execution):
    event.listen(
        # Catch and trim Execution.state_info to always fit allocated size.
        cls.state_info,
        'set',
        lambda t, v, o, i: utils.cut(v, 1020),
        retval=True)


def validate_long_type_length(cls, field_name, value):
    """Makes sure the value does not exceeds the maximum size."""
    if value:
        # Get the configured limit.
        size_limit_kb = cfg.CONF.engine.execution_field_size_limit_kb

        # If the size is unlimited.
        if (size_limit_kb < 0):
            return

        size_kb = sys.getsizeof(str(value)) / 1024
        if (size_kb > size_limit_kb):
Example #28
0
 def test_cut_list_with_large_dict_of_int(self):
     d = [i for i in range(65535)]
     s = utils.cut(d, 65535)
     self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
Example #29
0
 def test_cut_dict_with_large_dict_of_str(self):
     d = {}
     for i in range(65535):
         d[str(i)] = str(i)
     s = utils.cut(d, 65535)
     self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
Example #30
0
 def test_cut_dict_with_large_dict_of_dict(self):
     d = {}
     for i in range(65535):
         d[i] = {'value': str(i)}
     s = utils.cut(d, 65535)
     self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65535)))
Example #31
0
 def test_cut_dict_with_large_dict_of_str(self):
     d = {}
     for i in range(65535):
         d[str(i)] = str(i)
     s = utils.cut(d, 65535)
     assert len(s) <= 65535, len(s)
Example #32
0
 def test_cut_dict_for_state_info(self):
     d = {}
     for i in range(2000):
         d[i] = {'value': 'This is a string that exceeds 35 characters'}
     s = utils.cut(d, 65500)
     self.assertThat(len(s), ttm.Not(ttm.GreaterThan(65500)))
Example #33
0
    # Whether the task is fully processed (publishing and calculating commands
    # after it). It allows to simplify workflow controller implementations
    # significantly.
    processed = sa.Column(sa.BOOLEAN, default=False)

    # Data Flow properties.
    in_context = sa.Column(st.JsonLongDictType())
    published = sa.Column(st.JsonDictType())


for cls in utils.iter_subclasses(Execution):
    event.listen(
        # Catch and trim Execution.state_info to always fit allocated size.
        cls.state_info,
        'set',
        lambda t, v, o, i: utils.cut(v, 1020),
        retval=True
    )

# Many-to-one for 'Execution' and 'TaskExecution'.

Execution.task_execution_id = sa.Column(
    sa.String(36),
    sa.ForeignKey(TaskExecution.id),
    nullable=True
)

TaskExecution.executions = relationship(
    Execution,
    backref=backref('task_execution', remote_side=[TaskExecution.id]),
    cascade='all, delete-orphan',
Example #34
0
        return d


for cls in utils.iter_subclasses(Execution):
    event.listen(
        # Catch and trim Execution.state_info to always fit allocated size.
        # Note that the limit is 65500 which is less than 65535 (2^16 -1).
        # The reason is that utils.cut() is not exactly accurate in case if
        # the value is not a string, but, for example, a dictionary. If we
        # limit it exactly to 65535 then once in a while it may go slightly
        # beyond the allowed maximum size. It may depend on the order of
        # keys in a string representation and other things that are hidden
        # inside utils.cut_dict() method.
        cls.state_info,
        'set',
        lambda t, v, o, i: utils.cut(v, 65500),
        retval=True
    )


# Many-to-one for 'ActionExecution' and 'TaskExecution'.

ActionExecution.task_execution_id = sa.Column(
    sa.String(36),
    sa.ForeignKey(TaskExecution.id, ondelete='CASCADE'),
    nullable=True
)

TaskExecution.action_executions = relationship(
    ActionExecution,
    backref=backref('task_execution', remote_side=[TaskExecution.id]),