def test_get_dependent_job_ids(self): """Tests calling RecipeHandler.get_dependent_job_ids()""" handler = RecipeHandler(self.recipe, self.recipe_jobs) dependent_job_ids = handler.get_dependent_job_ids(self.job_completed.id) self.assertSetEqual(dependent_job_ids, {self.job_fa_co_a.id, self.job_fa_co_b.id, self.job_co_ru_qu_a.id, self.job_co_ru_qu_b.id})
def test_get_pending_jobs(self): """Tests calling RecipeHandler.get_pending_jobs()""" handler = RecipeHandler(self.recipe, self.recipe_jobs) pending_jobs = handler.get_pending_jobs() pending_job_ids = set() for pending_job in pending_jobs: pending_job_ids.add(pending_job.id) self.assertSetEqual(pending_job_ids, {self.job_co_ru_qu_a.id, self.job_co_ru_qu_b.id})
def test_get_blocked_jobs(self): """Tests calling RecipeHandler.get_blocked_jobs()""" handler = RecipeHandler(self.recipe, self.recipe_jobs) blocked_jobs = handler.get_blocked_jobs() blocked_job_ids = set() for blocked_job in blocked_jobs: blocked_job_ids.add(blocked_job.id) self.assertSetEqual(blocked_job_ids, {self.job_fa_co_b.id, self.job_qu_ca_a.id, self.job_qu_ca_b.id})
def test_get_dependent_job_ids(self): """Tests calling RecipeHandler.get_dependent_job_ids()""" handler = RecipeHandler(self.recipe, self.recipe_jobs) dependent_job_ids = handler.get_dependent_job_ids( self.job_completed.id) self.assertSetEqual( dependent_job_ids, { self.job_fa_co_a.id, self.job_fa_co_b.id, self.job_co_ru_qu_a.id, self.job_co_ru_qu_b.id })
def test_get_blocked_jobs(self): """Tests calling RecipeHandler.get_blocked_jobs()""" handler = RecipeHandler(self.recipe, self.recipe_jobs) blocked_jobs = handler.get_blocked_jobs() blocked_job_ids = set() for blocked_job in blocked_jobs: blocked_job_ids.add(blocked_job.id) self.assertSetEqual( blocked_job_ids, {self.job_fa_co_b.id, self.job_qu_ca_a.id, self.job_qu_ca_b.id})
def _get_recipe_handlers(self, recipe_ids): """Returns the handlers for the given recipe IDs. If a given recipe ID is not valid it will not be included in the results. :param recipe_ids: The recipe IDs :type recipe_ids: [int] :returns: The recipe handlers by recipe ID :rtype: {int: :class:`recipe.handler.RecipeHandler`} """ handlers = {} # {Recipe ID: Recipe handler} recipe_jobs_dict = RecipeJob.objects.get_recipe_jobs(recipe_ids) for recipe_id in recipe_ids: if recipe_id in recipe_jobs_dict: recipe_jobs = recipe_jobs_dict[recipe_id] if recipe_jobs: recipe = recipe_jobs[0].recipe handler = RecipeHandler(recipe, recipe_jobs) handlers[recipe.id] = handler return handlers
def create_recipe(self, recipe_type, data, event, superseded_recipe=None, delta=None, superseded_jobs=None): """Creates a new recipe for the given type and returns a recipe handler for it. All jobs for the recipe will also be created. If the new recipe is superseding an old recipe, superseded_recipe, delta, and superseded_jobs must be provided and the caller must have obtained a model lock on all job models in superseded_jobs and on the superseded_recipe model. All database changes occur in an atomic transaction. :param recipe_type: The type of the recipe to create :type recipe_type: :class:`recipe.models.RecipeType` :param data: The recipe data to run on, should be None if superseded_recipe is provided :type data: :class:`recipe.data.recipe_data.RecipeData` :param event: The event that triggered the creation of this recipe :type event: :class:`trigger.models.TriggerEvent` :param superseded_recipe: The recipe that the created recipe is superseding, possibly None :type superseded_recipe: :class:`recipe.models.Recipe` :param delta: If not None, represents the changes between the old recipe to supersede and the new recipe :type delta: :class:`recipe.handlers.graph_delta.RecipeGraphDelta` :param superseded_jobs: If not None, represents the job models (stored by job name) of the old recipe to supersede. This mapping must include all jobs created by the previous recipe, not just the ones that will actually be replaced by the new recipe definition. :type superseded_jobs: {string: :class:`job.models.Job`} :returns: A handler for the new recipe :rtype: :class:`recipe.handlers.handler.RecipeHandler` :raises :class:`recipe.exceptions.CreateRecipeError`: If general recipe parameters are invalid :raises :class:`recipe.exceptions.SupersedeError`: If the superseded parameters are invalid :raises :class:`recipe.exceptions.ReprocessError`: If recipe cannot be reprocessed :raises :class:`recipe.configuration.data.exceptions.InvalidRecipeData`: If the recipe data is invalid """ if not recipe_type.is_active: raise CreateRecipeError('Recipe type is no longer active') if event is None: raise CreateRecipeError('Event that triggered recipe creation is required') recipe = Recipe() recipe.recipe_type = recipe_type recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe_type.id, recipe_type.revision_num) recipe.event = event recipe_definition = recipe.get_recipe_definition() when = timezone.now() if superseded_recipe: # Mark superseded recipe superseded_recipe.is_superseded = True superseded_recipe.superseded = when superseded_recipe.save() # Use data from superseded recipe data = superseded_recipe.get_recipe_data() if not delta: raise SupersedeError('Cannot supersede a recipe without delta') # New recipe references superseded recipe root_id = superseded_recipe.root_superseded_recipe_id if not root_id: root_id = superseded_recipe.id recipe.root_superseded_recipe_id = root_id recipe.superseded_recipe = superseded_recipe else: if delta: raise SupersedeError('delta must be provided with a superseded recipe') # Validate recipe data and save recipe recipe_definition.validate_data(data) recipe.data = data.get_dict() recipe.save() # Create recipe jobs and link them to the recipe recipe_jobs = self._create_recipe_jobs(recipe, event, when, delta, superseded_jobs) return RecipeHandler(recipe, recipe_jobs)
def test_get_existing_jobs_to_queue(self): """Tests calling RecipeHandler.get_existing_jobs_to_queue()""" input_name_1 = 'Test Input 1' output_name_1 = 'Test Output 1' interface_1 = { 'version': '1.0', 'command': 'my_cmd', 'command_arguments': 'args', 'input_data': [{ 'name': input_name_1, 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': output_name_1, 'type': 'files', 'media_type': 'image/png', }], } job_type_1 = job_test_utils.create_job_type(interface=interface_1) job_1 = job_test_utils.create_job(job_type=job_type_1) input_name_2 = 'Test Input 2' output_name_2 = 'Test Output 2' interface_2 = { 'version': '1.0', 'command': 'my_cmd', 'command_arguments': 'args', 'input_data': [{ 'name': input_name_2, 'type': 'files', 'media_types': ['image/png', 'image/tiff'], }], 'output_data': [{ 'name': output_name_2, 'type': 'file', }], } job_type_2 = job_test_utils.create_job_type(interface=interface_2) job_2 = job_test_utils.create_job(job_type=job_type_2) workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, media_type='text/plain') definition = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'type': 'file', 'media_types': ['text/plain'], }], 'jobs': [{ 'name': 'Job 1', 'job_type': { 'name': job_type_1.name, 'version': job_type_1.version, }, 'recipe_inputs': [{ 'recipe_input': 'Recipe Input', 'job_input': input_name_1, }] }, { 'name': 'Job 2', 'job_type': { 'name': job_type_2.name, 'version': job_type_2.version, }, 'dependencies': [{ 'name': 'Job 1', 'connections': [{ 'output': output_name_1, 'input': input_name_2, }], }], }], } data = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': file_1.id, }], 'workspace_id': workspace.id, } recipe_type = recipe_test_utils.create_recipe_type( definition=definition) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, data=data) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='Job 1', job=job_1) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='Job 2', job=job_2) recipe_jobs = list(RecipeJob.objects.filter(recipe_id=recipe.id)) handler = RecipeHandler(recipe, recipe_jobs) jobs_to_queue = handler.get_existing_jobs_to_queue() # Make sure only Job 1 is returned and that its job data is correct self.assertEqual(len(jobs_to_queue), 1) self.assertEqual(jobs_to_queue[0][0].id, job_1.id) self.assertDictEqual( jobs_to_queue[0][1].get_dict(), { 'version': '1.0', 'input_data': [{ 'name': input_name_1, 'file_id': file_1.id, }], 'output_data': [{ 'name': output_name_1, 'workspace_id': workspace.id, }], })
def test_get_existing_jobs_to_queue(self): """Tests calling RecipeHandler.get_existing_jobs_to_queue()""" input_name_1 = 'Test Input 1' output_name_1 = 'Test Output 1' interface_1 = { 'version': '1.0', 'command': 'my_cmd', 'command_arguments': 'args', 'input_data': [{ 'name': input_name_1, 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': output_name_1, 'type': 'files', 'media_type': 'image/png', }], } job_type_1 = job_test_utils.create_job_type(interface=interface_1) job_1 = job_test_utils.create_job(job_type=job_type_1) input_name_2 = 'Test Input 2' output_name_2 = 'Test Output 2' interface_2 = { 'version': '1.0', 'command': 'my_cmd', 'command_arguments': 'args', 'input_data': [{ 'name': input_name_2, 'type': 'files', 'media_types': ['image/png', 'image/tiff'], }], 'output_data': [{ 'name': output_name_2, 'type': 'file', }], } job_type_2 = job_test_utils.create_job_type(interface=interface_2) job_2 = job_test_utils.create_job(job_type=job_type_2) workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, media_type='text/plain') definition = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'type': 'file', 'media_types': ['text/plain'], }], 'jobs': [{ 'name': 'Job 1', 'job_type': { 'name': job_type_1.name, 'version': job_type_1.version, }, 'recipe_inputs': [{ 'recipe_input': 'Recipe Input', 'job_input': input_name_1, }] }, { 'name': 'Job 2', 'job_type': { 'name': job_type_2.name, 'version': job_type_2.version, }, 'dependencies': [{ 'name': 'Job 1', 'connections': [{ 'output': output_name_1, 'input': input_name_2, }], }], }], } data = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': file_1.id, }], 'workspace_id': workspace.id, } recipe_type = recipe_test_utils.create_recipe_type(definition=definition) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, data=data) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='Job 1', job=job_1) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='Job 2', job=job_2) recipe_jobs = list(RecipeJob.objects.filter(recipe_id=recipe.id)) handler = RecipeHandler(recipe, recipe_jobs) jobs_to_queue = handler.get_existing_jobs_to_queue() # Make sure only Job 1 is returned and that its job data is correct self.assertEqual(len(jobs_to_queue), 1) self.assertEqual(jobs_to_queue[0][0].id, job_1.id) self.assertDictEqual(jobs_to_queue[0][1].get_dict(), { 'version': '1.0', 'input_data': [{ 'name': input_name_1, 'file_id': file_1.id, }], 'output_data': [{ 'name': output_name_1, 'workspace_id': workspace.id, }], })