def test_write_task(self): """Test that the provided entity gets written to the repository.""" entity = TaskEntity.get_mock_object() with self.assertRaises(ResourceNotFoundException): TaskService.fetch_task_by_id(entity.id) TaskService.write_task(entity) self.assertEquals( TaskService.fetch_task_by_id(entity.id), entity, )
def run_pipeline_by_id(cls, id): """Runs a pipeline by id. :param str id: """ log.info('Running pipeline: {}'.format(id)) PipelineScheduleService.pre_run_schedule(id) # get graph graph_data = TaskConnectionService.build_graph_for_pipeline(id) # process graph queue = deque() for source_id in graph_data['source_ids']: queue.appendleft((source_id, None)) while len(queue): task_id, data = queue.pop() try: task_response = TaskService.process_task_with_data(task_id, data) try: for next_id in graph_data['graph'][task_id]: queue.appendleft((next_id, task_response)) except KeyError: # end of list pass except StopProcessingException: pass PipelineScheduleService.post_run_schedule(id)
def test_fetch_task_by_id(self, mock_fetch): """Test that the correct TaskEntity is returned.""" mock_fetch.return_value = self.url_task entity = TaskService.fetch_task_by_id(self.url_task.id) self.assertEquals(entity.id, self.url_task.id)
def test_process_task_with_data(self, mock_process, mock_fetch): """Test that the right task class is created and called.""" mock_fetch.return_value = self.url_task TaskService.process_task_with_data(self.url_task.id, 'fake_data') mock_process.assert_called_once_with('fake_data')
def import_pipeline(cls, pipeline_data): if not pipeline_data.get('pipeline'): raise Exception('pipeline data is required') if not pipeline_data.get('pipeline_schedule'): raise Exception('pipeline schedule data is required') if not pipeline_data.get('tasks'): raise Exception('task data is required') if not pipeline_data.get('task_connections'): raise Exception('task connections are required') # Create Pipeline pipeline_entity = PipelineEntity(pipeline_data['pipeline']) pipeline_entity.validate() # Create PipelineSchedule pipeline_schedule_entity = PipelineScheduleEntity(pipeline_data['pipeline_schedule']) pipeline_schedule_entity.pipeline_id = pipeline_entity.id pipeline_schedule_entity.next_run_at = ( PipelineScheduleService.calculate_next_run_at_for_schedule( pipeline_schedule_entity ) ) pipeline_schedule_entity.validate() # Create Tasks task_entities = [] task_alias_to_id = {} for task_alias, task_data in pipeline_data['tasks'].items(): task_entity = TaskEntity(task_data) task_entity.validate() task_entities.append(task_entity) task_alias_to_id[task_alias] = task_entity.id # Create TaskConnections task_connection_entities = [] for from_alias, to_aliases in pipeline_data['task_connections'].items(): from_task_id = task_alias_to_id[from_alias] for to_alias in to_aliases: task_connection_entity = TaskConnectionEntity({ 'from_task_id': from_task_id, 'to_task_id': task_alias_to_id[to_alias], 'pipeline_id': pipeline_entity.id, }) task_connection_entity.validate() task_connection_entities.append(task_connection_entity) # Save Pipeline PipelineService.write_pipeline(pipeline_entity) # Save PipelineSchedule (enabled = False) PipelineScheduleService.write_pipeline_schedule(pipeline_schedule_entity) # Save Tasks for task_entity in task_entities: TaskService.write_task(task_entity) # Save TaskConnections for task_connection_entity in task_connection_entities: TaskConnectionService.write_task_connection(task_connection_entity) return pipeline_entity