Exemple #1
0
    def run_pipeline_by_id(cls, id):
        """Runs a pipeline by id.

        :param str id:
        """
        log.info('Running pipeline: {}'.format(id))

        PipelineScheduleService.pre_run_schedule(id)

        # get graph
        graph_data = TaskConnectionService.build_graph_for_pipeline(id)

        # process graph
        queue = deque()

        for source_id in graph_data['source_ids']:
            queue.appendleft((source_id, None))

        while len(queue):
            task_id, data = queue.pop()

            try:
                task_response = TaskService.process_task_with_data(task_id, data)

                try:
                    for next_id in graph_data['graph'][task_id]:
                        queue.appendleft((next_id, task_response))
                except KeyError:
                    # end of list
                    pass
            except StopProcessingException:
                pass

        PipelineScheduleService.post_run_schedule(id)
Exemple #2
0
    def test_process_task_with_data(self, mock_process, mock_fetch):
        """Test that the right task class is created and called."""
        mock_fetch.return_value = self.url_task

        TaskService.process_task_with_data(self.url_task.id, 'fake_data')
        mock_process.assert_called_once_with('fake_data')