def test_get_dependent_job_ids(self):
"""Tests calling RecipeHandler.get_dependent_job_ids()"""
handler = RecipeHandler(self.recipe, self.recipe_jobs)
dependent_job_ids = handler.get_dependent_job_ids(self.job_completed.id)
self.assertSetEqual(dependent_job_ids, {self.job_fa_co_a.id, self.job_fa_co_b.id, self.job_co_ru_qu_a.id,
self.job_co_ru_qu_b.id})
def test_get_pending_jobs(self):
"""Tests calling RecipeHandler.get_pending_jobs()"""
handler = RecipeHandler(self.recipe, self.recipe_jobs)
pending_jobs = handler.get_pending_jobs()
pending_job_ids = set()
for pending_job in pending_jobs:
pending_job_ids.add(pending_job.id)
self.assertSetEqual(pending_job_ids, {self.job_co_ru_qu_a.id, self.job_co_ru_qu_b.id})
def test_get_blocked_jobs(self):
"""Tests calling RecipeHandler.get_blocked_jobs()"""
handler = RecipeHandler(self.recipe, self.recipe_jobs)
blocked_jobs = handler.get_blocked_jobs()
blocked_job_ids = set()
for blocked_job in blocked_jobs:
blocked_job_ids.add(blocked_job.id)
self.assertSetEqual(blocked_job_ids, {self.job_fa_co_b.id, self.job_qu_ca_a.id, self.job_qu_ca_b.id})
def test_get_pending_jobs(self):
"""Tests calling RecipeHandler.get_pending_jobs()"""
handler = RecipeHandler(self.recipe, self.recipe_jobs)
pending_jobs = handler.get_pending_jobs()
pending_job_ids = set()
for pending_job in pending_jobs:
pending_job_ids.add(pending_job.id)
self.assertSetEqual(pending_job_ids,
{self.job_co_ru_qu_a.id, self.job_co_ru_qu_b.id})
def test_get_dependent_job_ids(self):
"""Tests calling RecipeHandler.get_dependent_job_ids()"""
handler = RecipeHandler(self.recipe, self.recipe_jobs)
dependent_job_ids = handler.get_dependent_job_ids(
self.job_completed.id)
self.assertSetEqual(
dependent_job_ids, {
self.job_fa_co_a.id, self.job_fa_co_b.id,
self.job_co_ru_qu_a.id, self.job_co_ru_qu_b.id
})
def test_get_blocked_jobs(self):
"""Tests calling RecipeHandler.get_blocked_jobs()"""
handler = RecipeHandler(self.recipe, self.recipe_jobs)
blocked_jobs = handler.get_blocked_jobs()
blocked_job_ids = set()
for blocked_job in blocked_jobs:
blocked_job_ids.add(blocked_job.id)
self.assertSetEqual(
blocked_job_ids,
{self.job_fa_co_b.id, self.job_qu_ca_a.id, self.job_qu_ca_b.id})
def _get_recipe_handlers(self, recipe_ids):
"""Returns the handlers for the given recipe IDs. If a given recipe ID is not valid it will not be included in
the results.
:param recipe_ids: The recipe IDs
:type recipe_ids: [int]
:returns: The recipe handlers by recipe ID
:rtype: {int: :class:`recipe.handler.RecipeHandler`}
"""
handlers = {} # {Recipe ID: Recipe handler}
recipe_jobs_dict = RecipeJob.objects.get_recipe_jobs(recipe_ids)
for recipe_id in recipe_ids:
if recipe_id in recipe_jobs_dict:
recipe_jobs = recipe_jobs_dict[recipe_id]
if recipe_jobs:
recipe = recipe_jobs[0].recipe
handler = RecipeHandler(recipe, recipe_jobs)
handlers[recipe.id] = handler
return handlers
def create_recipe(self, recipe_type, data, event, superseded_recipe=None, delta=None, superseded_jobs=None):
"""Creates a new recipe for the given type and returns a recipe handler for it. All jobs for the recipe will
also be created. If the new recipe is superseding an old recipe, superseded_recipe, delta, and superseded_jobs
must be provided and the caller must have obtained a model lock on all job models in superseded_jobs and on the
superseded_recipe model. All database changes occur in an atomic transaction.
:param recipe_type: The type of the recipe to create
:type recipe_type: :class:`recipe.models.RecipeType`
:param data: The recipe data to run on, should be None if superseded_recipe is provided
:type data: :class:`recipe.data.recipe_data.RecipeData`
:param event: The event that triggered the creation of this recipe
:type event: :class:`trigger.models.TriggerEvent`
:param superseded_recipe: The recipe that the created recipe is superseding, possibly None
:type superseded_recipe: :class:`recipe.models.Recipe`
:param delta: If not None, represents the changes between the old recipe to supersede and the new recipe
:type delta: :class:`recipe.handlers.graph_delta.RecipeGraphDelta`
:param superseded_jobs: If not None, represents the job models (stored by job name) of the old recipe to
supersede. This mapping must include all jobs created by the previous recipe, not just the ones that will
actually be replaced by the new recipe definition.
:type superseded_jobs: {string: :class:`job.models.Job`}
:returns: A handler for the new recipe
:rtype: :class:`recipe.handlers.handler.RecipeHandler`
:raises :class:`recipe.exceptions.CreateRecipeError`: If general recipe parameters are invalid
:raises :class:`recipe.exceptions.SupersedeError`: If the superseded parameters are invalid
:raises :class:`recipe.exceptions.ReprocessError`: If recipe cannot be reprocessed
:raises :class:`recipe.configuration.data.exceptions.InvalidRecipeData`: If the recipe data is invalid
"""
if not recipe_type.is_active:
raise CreateRecipeError('Recipe type is no longer active')
if event is None:
raise CreateRecipeError('Event that triggered recipe creation is required')
recipe = Recipe()
recipe.recipe_type = recipe_type
recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe_type.id, recipe_type.revision_num)
recipe.event = event
recipe_definition = recipe.get_recipe_definition()
when = timezone.now()
if superseded_recipe:
# Mark superseded recipe
superseded_recipe.is_superseded = True
superseded_recipe.superseded = when
superseded_recipe.save()
# Use data from superseded recipe
data = superseded_recipe.get_recipe_data()
if not delta:
raise SupersedeError('Cannot supersede a recipe without delta')
# New recipe references superseded recipe
root_id = superseded_recipe.root_superseded_recipe_id
if not root_id:
root_id = superseded_recipe.id
recipe.root_superseded_recipe_id = root_id
recipe.superseded_recipe = superseded_recipe
else:
if delta:
raise SupersedeError('delta must be provided with a superseded recipe')
# Validate recipe data and save recipe
recipe_definition.validate_data(data)
recipe.data = data.get_dict()
recipe.save()
# Create recipe jobs and link them to the recipe
recipe_jobs = self._create_recipe_jobs(recipe, event, when, delta, superseded_jobs)
return RecipeHandler(recipe, recipe_jobs)
def test_get_existing_jobs_to_queue(self):
"""Tests calling RecipeHandler.get_existing_jobs_to_queue()"""
input_name_1 = 'Test Input 1'
output_name_1 = 'Test Output 1'
interface_1 = {
'version':
'1.0',
'command':
'my_cmd',
'command_arguments':
'args',
'input_data': [{
'name': input_name_1,
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': output_name_1,
'type': 'files',
'media_type': 'image/png',
}],
}
job_type_1 = job_test_utils.create_job_type(interface=interface_1)
job_1 = job_test_utils.create_job(job_type=job_type_1)
input_name_2 = 'Test Input 2'
output_name_2 = 'Test Output 2'
interface_2 = {
'version':
'1.0',
'command':
'my_cmd',
'command_arguments':
'args',
'input_data': [{
'name': input_name_2,
'type': 'files',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': output_name_2,
'type': 'file',
}],
}
job_type_2 = job_test_utils.create_job_type(interface=interface_2)
job_2 = job_test_utils.create_job(job_type=job_type_2)
workspace = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace,
media_type='text/plain')
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': job_type_1.name,
'version': job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': input_name_1,
}]
}, {
'name':
'Job 2',
'job_type': {
'name': job_type_2.name,
'version': job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': output_name_1,
'input': input_name_2,
}],
}],
}],
}
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': file_1.id,
}],
'workspace_id': workspace.id,
}
recipe_type = recipe_test_utils.create_recipe_type(
definition=definition)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
data=data)
recipe_test_utils.create_recipe_job(recipe=recipe,
job_name='Job 1',
job=job_1)
recipe_test_utils.create_recipe_job(recipe=recipe,
job_name='Job 2',
job=job_2)
recipe_jobs = list(RecipeJob.objects.filter(recipe_id=recipe.id))
handler = RecipeHandler(recipe, recipe_jobs)
jobs_to_queue = handler.get_existing_jobs_to_queue()
# Make sure only Job 1 is returned and that its job data is correct
self.assertEqual(len(jobs_to_queue), 1)
self.assertEqual(jobs_to_queue[0][0].id, job_1.id)
self.assertDictEqual(
jobs_to_queue[0][1].get_dict(), {
'version':
'1.0',
'input_data': [{
'name': input_name_1,
'file_id': file_1.id,
}],
'output_data': [{
'name': output_name_1,
'workspace_id': workspace.id,
}],
})
def test_get_existing_jobs_to_queue(self):
"""Tests calling RecipeHandler.get_existing_jobs_to_queue()"""
input_name_1 = 'Test Input 1'
output_name_1 = 'Test Output 1'
interface_1 = {
'version': '1.0',
'command': 'my_cmd',
'command_arguments': 'args',
'input_data': [{
'name': input_name_1,
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': output_name_1,
'type': 'files',
'media_type': 'image/png',
}],
}
job_type_1 = job_test_utils.create_job_type(interface=interface_1)
job_1 = job_test_utils.create_job(job_type=job_type_1)
input_name_2 = 'Test Input 2'
output_name_2 = 'Test Output 2'
interface_2 = {
'version': '1.0',
'command': 'my_cmd',
'command_arguments': 'args',
'input_data': [{
'name': input_name_2,
'type': 'files',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': output_name_2,
'type': 'file',
}],
}
job_type_2 = job_test_utils.create_job_type(interface=interface_2)
job_2 = job_test_utils.create_job(job_type=job_type_2)
workspace = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace, media_type='text/plain')
definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': job_type_1.name,
'version': job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': input_name_1,
}]
}, {
'name': 'Job 2',
'job_type': {
'name': job_type_2.name,
'version': job_type_2.version,
},
'dependencies': [{
'name': 'Job 1',
'connections': [{
'output': output_name_1,
'input': input_name_2,
}],
}],
}],
}
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': file_1.id,
}],
'workspace_id': workspace.id,
}
recipe_type = recipe_test_utils.create_recipe_type(definition=definition)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, data=data)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='Job 1', job=job_1)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='Job 2', job=job_2)
recipe_jobs = list(RecipeJob.objects.filter(recipe_id=recipe.id))
handler = RecipeHandler(recipe, recipe_jobs)
jobs_to_queue = handler.get_existing_jobs_to_queue()
# Make sure only Job 1 is returned and that its job data is correct
self.assertEqual(len(jobs_to_queue), 1)
self.assertEqual(jobs_to_queue[0][0].id, job_1.id)
self.assertDictEqual(jobs_to_queue[0][1].get_dict(), {
'version': '1.0',
'input_data': [{
'name': input_name_1,
'file_id': file_1.id,
}],
'output_data': [{
'name': output_name_1,
'workspace_id': workspace.id,
}],
})
