예제 #1
0
 def _send_call(self, my_task):
     """Sends Celery asynchronous call and stores async call information for
     retrieval laster"""
     args, kwargs, queue = [], {}, None
     if self.args:
         args = _eval_args(self.args, my_task)
     if self.kwargs:
         kwargs = _eval_kwargs(self.kwargs, my_task)
     if self.call_server_id:
         queue = 'server.{0}'.format(valueof(my_task, self.call_server_id))
     elif self.call_queue:
         queue = valueof(my_task, self.call_queue)
     LOG.debug("%s (task id %s) calling %s",
               self.name,
               my_task.id,
               self.call,
               extra=dict(data=dict(args=args, kwargs=kwargs)))
     # Add current workflow information
     kwargs['workflow'] = {'data': my_task.workflow.data}
     async_call = current_app().send_task(self.call,
                                          args=args,
                                          kwargs=kwargs,
                                          queue=queue)
     my_task.internal_data['task_id'] = async_call.task_id
     my_task.internal_data['async_call'] = async_call
     LOG.debug("'%s' called: %s", self.call, async_call.task_id)
예제 #2
0
    def _try_fire_unstructured(self, my_task, force=False):
        # The default threshold is the number of inputs.
        threshold = valueof(my_task, self.threshold)
        if threshold is None:
            threshold = len(self.inputs)

        # Look at the tree to find all places where this task is used.
        tasks = []
        for input in self.inputs:
            for task in my_task.workflow.task_tree:
                if task.thread_id != my_task.thread_id:
                    continue
                if task.task_spec != input:
                    continue
                tasks.append(task)

        # Look up which tasks have already completed.
        waiting_tasks = []
        completed = 0
        for task in tasks:
            if task.parent is None or task._has_state(Task.COMPLETED):
                completed += 1
            else:
                waiting_tasks.append(task)

        # If the threshold was reached, get ready to fire.
        return force or completed >= threshold, waiting_tasks
예제 #3
0
    def _try_fire_structured(self, my_task, force=False):
        # Retrieve a list of all activated tasks from the associated
        # task that did the conditional parallel split.
        split_task = my_task._find_ancestor_from_name(self.split_task)
        if split_task is None:
            msg = "Join with %s, which was not reached" % self.split_task
            raise WorkflowException(self, msg)
        tasks = split_task.task_spec._get_activated_tasks(split_task, my_task)

        # The default threshold is the number of branches that were started.
        threshold = valueof(my_task, self.threshold)
        if threshold is None:
            threshold = len(tasks)

        # Look up which tasks have already completed.
        waiting_tasks = []
        completed = 0
        for task in tasks:
            # Refresh path prediction.
            task.task_spec._predict(task)

            if not self._branch_may_merge_at(task):
                completed += 1
            elif self._branch_is_complete(task):
                completed += 1
            else:
                waiting_tasks.append(task)

        # If the threshold was reached, get ready to fire.
        return force or completed >= threshold, waiting_tasks
예제 #4
0
    def _try_fire_unstructured(self, my_task, force=False):
        # The default threshold is the number of inputs.
        threshold = valueof(my_task, self.threshold)
        if threshold is None:
            threshold = len(self.inputs)

        # Look at the tree to find all places where this task is used.
        tasks = []
        for input in self.inputs:
            for task in my_task.workflow.task_tree:
                if task.thread_id != my_task.thread_id:
                    continue
                if task.task_spec != input:
                    continue
                tasks.append(task)

        # Look up which tasks have already completed.
        waiting_tasks = []
        completed = 0
        for task in tasks:
            if task.parent is None or task._has_state(Task.COMPLETED):
                completed += 1
            else:
                waiting_tasks.append(task)

        # If the threshold was reached, get ready to fire.
        return force or completed >= threshold, waiting_tasks
예제 #5
0
    def _try_fire_structured(self, my_task, force=False):
        # Retrieve a list of all activated tasks from the associated
        # task that did the conditional parallel split.
        split_task = my_task._find_ancestor_from_name(self.split_task)
        if split_task is None:
            msg = 'Join with %s, which was not reached' % self.split_task
            raise WorkflowException(self, msg)
        tasks = split_task.task_spec._get_activated_tasks(split_task, my_task)

        # The default threshold is the number of branches that were started.
        threshold = valueof(my_task, self.threshold)
        if threshold is None:
            threshold = len(tasks)

        # Look up which tasks have already completed.
        waiting_tasks = []
        completed = 0
        for task in tasks:
            # Refresh path prediction.
            task.task_spec._predict(task)

            if not self._branch_may_merge_at(task):
                completed += 1
            elif self._branch_is_complete(task):
                completed += 1
            else:
                waiting_tasks.append(task)

        # If the threshold was reached, get ready to fire.
        return force or completed >= threshold, waiting_tasks
예제 #6
0
 def _create_subworkflow(self, my_task):
     from SpiffWorkflow.specs import WorkflowSpec
     file           = valueof(my_task, self.file)
     serializer     = self.serializer_cls()
     s_state        = open(file).read()
     wf_spec        = WorkflowSpec.deserialize(serializer, s_state, filename = file)
     outer_workflow = my_task.workflow.outer_workflow
     return SpiffWorkflow.Workflow(wf_spec, parent = outer_workflow)
예제 #7
0
 def _create_subworkflow(self, my_task):
     from SpiffWorkflow.specs import WorkflowSpec
     file = valueof(my_task, self.file)
     serializer = self.serializer_cls()
     s_state = open(file).read()
     wf_spec = WorkflowSpec.deserialize(serializer, s_state, filename=file)
     outer_workflow = my_task.workflow.outer_workflow
     return SpiffWorkflow.Workflow(wf_spec, parent=outer_workflow)
예제 #8
0
 def _create_subworkflow(self, my_task):
     from SpiffWorkflow.storage import XmlSerializer
     from SpiffWorkflow.specs import WorkflowSpec
     file           = valueof(my_task, self.file)
     serializer     = XmlSerializer()
     xml            = open(file).read()
     wf_spec        = WorkflowSpec.deserialize(serializer, xml, filename = file)
     outer_workflow = my_task.workflow.outer_workflow
     return SpiffWorkflow.Workflow(wf_spec, parent = outer_workflow)
예제 #9
0
def _eval_args(args, my_task):
    """Parses args and evaluates any Attrib entries"""
    results = []
    for arg in args:
        if isinstance(arg, Attrib) or isinstance(arg, PathAttrib):
            results.append(valueof(my_task, arg))
        else:
            results.append(arg)
    return results
예제 #10
0
def _eval_kwargs(kwargs, my_task):
    """Parses kwargs and evaluates any Attrib entries"""
    results = {}
    for kwarg, value in kwargs.iteritems():
        if isinstance(value, Attrib) or isinstance(value, PathAttrib):
            results[kwarg] = valueof(my_task, value)
        else:
            results[kwarg] = value
    return results
예제 #11
0
 def _create_subworkflow(self, my_task):
     from SpiffWorkflow.storage import XmlSerializer
     from SpiffWorkflow.specs import WorkflowSpec
     file = valueof(my_task, self.file)
     serializer = XmlSerializer()
     xml = open(file).read()
     wf_spec = WorkflowSpec.deserialize(serializer, xml, filename=file)
     outer_workflow = my_task.workflow.outer_workflow
     return SpiffWorkflow.Workflow(wf_spec, parent=outer_workflow)
예제 #12
0
 def _send_call(self, my_task):
     """Sends Celery asynchronous call and stores async call information for
     retrieval laster"""
     args, kwargs, queue = [], {}, None
     if self.args:
         args = _eval_args(self.args, my_task)
     if self.kwargs:
         kwargs = _eval_kwargs(self.kwargs, my_task)
     if self.call_server_id:
         queue = 'server.{0}'.format(valueof(my_task, self.call_server_id))
     elif self.call_queue:
         queue = valueof(my_task, self.call_queue)
     LOG.debug("%s (task id %s) calling %s", self.name, my_task.id,
             self.call, extra=dict(data=dict(args=args, kwargs=kwargs)))
     # Add current workflow information
     kwargs['workflow'] = {
         'data': my_task.workflow.data
     }
     async_call = current_app().send_task(self.call, args=args, kwargs=kwargs, queue=queue)
     my_task.internal_data['task_id'] = async_call.task_id
     my_task.internal_data['async_call'] = async_call
     LOG.debug("'%s' called: %s", self.call, async_call.task_id)
예제 #13
0
    def _predict_hook(self, my_task):
        split_n = valueof(my_task, self.times)
        if split_n is None:
            return
        my_task._set_internal_attribute(splits = split_n)

        # Create the outgoing tasks.
        outputs = []
        for i in range(split_n):
            outputs += self.outputs
        if my_task._is_definite():
            my_task._sync_children(outputs, Task.FUTURE)
        else:
            my_task._sync_children(outputs, Task.LIKELY)
예제 #14
0
    def _predict_hook(self, my_task):
        split_n = valueof(my_task, self.times)
        if split_n is None:
            return
        my_task._set_internal_data(splits = split_n)

        # Create the outgoing tasks.
        outputs = []
        for i in range(split_n):
            outputs += self.outputs
        if my_task._is_definite():
            my_task._sync_children(outputs, Task.FUTURE)
        else:
            my_task._sync_children(outputs, Task.LIKELY)
예제 #15
0
    def _predict_hook(self, my_task):
        split_n = valueof(my_task, self.times)
        if split_n is None:
            return
        my_task._set_internal_attribute(splits=split_n)

        # Create the outgoing tasks.
        outputs = []
        for i in range(split_n):
            outputs += self.outputs

        if my_task._has_state(Task.LIKELY):
            child_state = Task.LIKELY
        else:
            child_state = Task.FUTURE
        my_task._update_children(outputs, child_state)
예제 #16
0
    def _predict_hook(self, my_task):
        split_n = valueof(my_task, self.times)
        if split_n is None:
            return
        my_task._set_internal_attribute(splits = split_n)

        # Create the outgoing tasks.
        outputs = []
        for i in range(split_n):
            outputs += self.outputs

        if my_task._has_state(Task.LIKELY):
            child_state = Task.LIKELY
        else:
            child_state = Task.FUTURE
        my_task._update_children(outputs, child_state)
예제 #17
0
    def _try_fire(self, my_task):
        # If the threshold was already reached, there is nothing else to do.
        if my_task._has_state(Task.COMPLETED):
            return False
        if my_task._has_state(Task.READY):
            return True

        # Retrieve a list of all activated tasks from the associated
        # task that did the conditional parallel split.
        split_task = my_task._find_ancestor_from_name(self.split_task)
        if split_task is None:
            msg = 'Join with %s, which was not reached' % self.split_task
            raise WorkflowException(self, msg)
        tasks = split_task.task_spec._get_activated_threads(split_task)

        # The default threshold is the number of threads that were started.
        threshold = valueof(my_task, self.threshold)
        if threshold is None:
            threshold = len(tasks)

        # Look up which tasks have already completed.
        waiting_tasks = []
        completed = 0
        for task in tasks:
            # Refresh path prediction.
            task.task_spec._predict(task)

            if self._branch_is_complete(task):
                completed += 1
            else:
                waiting_tasks.append(task)

        # If the threshold was reached, get ready to fire.
        if completed >= threshold:
            # If this is a cancelling join, cancel all incoming branches,
            # except for the one that just completed.
            if self.cancel_remaining:
                for task in waiting_tasks:
                    task.cancel()
            return True

        # We do NOT set the task state to COMPLETED, because in
        # case all other incoming tasks get cancelled (or never reach
        # the ThreadMerge for other reasons, such as reaching a stub branch),
        # we need to revisit it.
        return False
예제 #18
0
    def _try_fire(self, my_task):
        # If the threshold was already reached, there is nothing else to do.
        if my_task._has_state(Task.COMPLETED):
            return False
        if my_task._has_state(Task.READY):
            return True

        # Retrieve a list of all activated tasks from the associated
        # task that did the conditional parallel split.
        split_task = my_task._find_ancestor_from_name(self.split_task)
        if split_task is None:
            msg = 'Join with %s, which was not reached' % self.split_task
            raise WorkflowException(self, msg)
        tasks = split_task.task_spec._get_activated_threads(split_task)

        # The default threshold is the number of threads that were started.
        threshold = valueof(my_task, self.threshold)
        if threshold is None:
            threshold = len(tasks)

        # Look up which tasks have already completed.
        waiting_tasks = []
        completed     = 0
        for task in tasks:
            # Refresh path prediction.
            task.task_spec._predict(task)

            if self._branch_is_complete(task):
                completed += 1
            else:
                waiting_tasks.append(task)

        # If the threshold was reached, get ready to fire.
        if completed >= threshold:
            # If this is a cancelling join, cancel all incoming branches,
            # except for the one that just completed.
            if self.cancel_remaining:
                for task in waiting_tasks:
                    task.cancel()
            return True

        # We do NOT set the task state to COMPLETED, because in
        # case all other incoming tasks get cancelled (or never reach
        # the ThreadMerge for other reasons, such as reaching a stub branch),
        # we need to revisit it.
        return False
예제 #19
0
    def _on_ready_before_hook(self, my_task):
        file           = valueof(my_task, self.file)
        xml_reader     = XmlReader()
        workflow_list  = xml_reader.parse_file(file)
        workflow       = workflow_list[0]
        outer_workflow = my_task.workflow.outer_workflow
        subworkflow    = SpiffWorkflow.Workflow(workflow, parent = outer_workflow)
        subworkflow.completed_event.connect(self._on_subworkflow_completed, my_task)

        # Integrate the tree of the subworkflow into the tree of this workflow.
        my_task._update_children(self.outputs, Task.FUTURE)
        for child in my_task.children:
            child._inherit_attributes()
        for child in subworkflow.task_tree.children:
            my_task.children.insert(0, child)
            child.parent = my_task

        my_task._set_internal_attribute(subworkflow = subworkflow)
        return True
예제 #20
0
    def _on_ready_before_hook(self, my_task):
        file = valueof(my_task, self.file)
        xml_reader = XmlReader()
        workflow_list = xml_reader.parse_file(file)
        workflow = workflow_list[0]
        outer_workflow = my_task.workflow.outer_workflow
        subworkflow = SpiffWorkflow.Workflow(workflow, parent=outer_workflow)
        subworkflow.completed_event.connect(self._on_subworkflow_completed,
                                            my_task)

        # Integrate the tree of the subworkflow into the tree of this workflow.
        my_task._update_children(self.outputs, Task.FUTURE)
        for child in my_task.children:
            child._inherit_attributes()
        for child in subworkflow.task_tree.children:
            my_task.children.insert(0, child)
            child.parent = my_task

        my_task._set_internal_attribute(subworkflow=subworkflow)
        return True
예제 #21
0
    def _try_fire_unstructured(self, my_task, force = False):
        # If the threshold was already reached, there is nothing else to do.
        if my_task._has_state(Task.COMPLETED):
            return False
        if my_task._has_state(Task.READY):
            return True

        # The default threshold is the number of inputs.
        threshold = valueof(my_task, self.threshold)
        if threshold is None:
            threshold = len(self.inputs)

        # Look at the tree to find all places where this task is used.
        tasks = []
        for input in self.inputs:
            for task in my_task.workflow.task_tree:
                if task.thread_id != my_task.thread_id:
                    continue
                if task.task_spec != input:
                    continue
                tasks.append(task)

        # Look up which tasks have already completed.
        waiting_tasks = []
        completed     = 0
        for task in tasks:
            if task.parent is None or task._has_state(Task.COMPLETED):
                completed += 1
            else:
                waiting_tasks.append(task)

        # If the threshold was reached, get ready to fire.
        if force or completed >= threshold:
            self._fire(my_task, waiting_tasks)
            return True

        # We do NOT set the task state to COMPLETED, because in
        # case all other incoming tasks get cancelled (or never reach
        # the Join for other reasons, such as reaching a stub branch), we
        # we need to revisit it.
        return False
예제 #22
0
    def _try_fire_unstructured(self, my_task, force=False):
        # If the threshold was already reached, there is nothing else to do.
        if my_task._has_state(Task.COMPLETED):
            return False
        if my_task._has_state(Task.READY):
            return True

        # The default threshold is the number of inputs.
        threshold = valueof(my_task, self.threshold)
        if threshold is None:
            threshold = len(self.inputs)

        # Look at the tree to find all places where this task is used.
        tasks = []
        for input in self.inputs:
            for task in my_task.workflow.task_tree:
                if task.thread_id != my_task.thread_id:
                    continue
                if task.task_spec != input:
                    continue
                tasks.append(task)

        # Look up which tasks have already completed.
        waiting_tasks = []
        completed = 0
        for task in tasks:
            if task.parent is None or task._has_state(Task.COMPLETED):
                completed += 1
            else:
                waiting_tasks.append(task)

        # If the threshold was reached, get ready to fire.
        if force or completed >= threshold:
            self._fire(my_task, waiting_tasks)
            return True

        # We do NOT set the task state to COMPLETED, because in
        # case all other incoming tasks get cancelled (or never reach
        # the Join for other reasons, such as reaching a stub branch), we
        # we need to revisit it.
        return False
예제 #23
0
    def _on_ready_before_hook(self, my_task):
        from SpiffWorkflow.storage import XmlSerializer
        from SpiffWorkflow.specs import WorkflowSpec
        file           = valueof(my_task, self.file)
        serializer     = XmlSerializer()
        xml            = open(file).read()
        wf_spec        = WorkflowSpec.deserialize(serializer, xml, filename = file)
        outer_workflow = my_task.workflow.outer_workflow
        subworkflow    = SpiffWorkflow.Workflow(wf_spec, parent = outer_workflow)
        subworkflow.completed_event.connect(self._on_subworkflow_completed, my_task)

        # Integrate the tree of the subworkflow into the tree of this workflow.
        my_task._sync_children(self.outputs, Task.FUTURE)
        for child in my_task.children:
            child.task_spec._update_state(child)
            child._inherit_attributes()
        for child in subworkflow.task_tree.children:
            my_task.children.insert(0, child)
            child.parent = my_task

        my_task._set_internal_attribute(subworkflow = subworkflow)
예제 #24
0
    def _on_ready_before_hook(self, my_task):
        from SpiffWorkflow.storage import XmlSerializer
        from SpiffWorkflow.specs import WorkflowSpec
        file = valueof(my_task, self.file)
        serializer = XmlSerializer()
        xml = open(file).read()
        wf_spec = WorkflowSpec.deserialize(serializer, xml, filename=file)
        outer_workflow = my_task.workflow.outer_workflow
        subworkflow = SpiffWorkflow.Workflow(wf_spec, parent=outer_workflow)
        subworkflow.completed_event.connect(self._on_subworkflow_completed,
                                            my_task)

        # Integrate the tree of the subworkflow into the tree of this workflow.
        my_task._sync_children(self.outputs, Task.FUTURE)
        for child in my_task.children:
            child.task_spec._update_state(child)
            child._inherit_attributes()
        for child in subworkflow.task_tree.children:
            my_task.children.insert(0, child)
            child.parent = my_task

        my_task._set_internal_attribute(subworkflow=subworkflow)
예제 #25
0
def _eval_assign(assign, scope):
    if isinstance(assign, FuncAttrib):
        return valueof(scope, assign)
    else:
        return dict((left, valueof(scope, right)) 
                for left, right in assign.items())
예제 #26
0
def _eval_assign(assign, scope):
    if isinstance(assign, FuncAttrib):
        return valueof(scope, assign)
    else:
        return dict(
            (left, valueof(scope, right)) for left, right in assign.items())
예제 #27
0
 def get_name_for(self, task):
     return str(valueof(task, self.name))
예제 #28
0
 def get_name_for(self, task):
     return str(valueof(task, self.name))