def run_pipeline_by_id(cls, id):
"""Runs a pipeline by id.
:param str id:
"""
log.info('Running pipeline: {}'.format(id))
PipelineScheduleService.pre_run_schedule(id)
# get graph
graph_data = TaskConnectionService.build_graph_for_pipeline(id)
# process graph
queue = deque()
for source_id in graph_data['source_ids']:
queue.appendleft((source_id, None))
while len(queue):
task_id, data = queue.pop()
try:
task_response = TaskService.process_task_with_data(task_id, data)
try:
for next_id in graph_data['graph'][task_id]:
queue.appendleft((next_id, task_response))
except KeyError:
# end of list
pass
except StopProcessingException:
pass
PipelineScheduleService.post_run_schedule(id)
def test_build_graph_for_pipeline(self, mock_fetch):
"""Test that a graph is returned when given a pipeline id."""
mock_fetch.return_value = [
self.url_to_log_connection,
self.raw_input_to_log_connection,
]
graph_data = TaskConnectionService.build_graph_for_pipeline(self.pipeline.id)
self.assertEquals(graph_data['graph'], {
self.url_to_log_connection.from_task_id: set([
self.url_to_log_connection.to_task_id,
]),
self.raw_input_to_log_connection.from_task_id: set([
self.raw_input_to_log_connection.to_task_id,
])
})
self.assertItemsEqual(graph_data['source_ids'], [
self.raw_input_to_log_connection.from_task_id,
self.url_to_log_connection.from_task_id,
])
