def test_successful_supersede_same_recipe_type(self):
"""Tests calling RecipeManager.create_recipe() to supersede a recipe with the same recipe type."""
event = trigger_test_utils.create_trigger_event()
handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
event=event,
data=RecipeData(self.data))
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 2')
superseded_jobs = {
'Job 1': recipe_job_1.job,
'Job 2': recipe_job_2.job
}
# Create a new recipe of the same type where we want to reprocess Job 2
graph = self.recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph, graph)
delta.reprocess_identical_node('Job 2') # We want to reprocess Job 2
new_handler = Recipe.objects.create_recipe(
recipe_type=self.recipe_type,
event=event,
data=None,
superseded_recipe=recipe,
delta=delta,
superseded_jobs=superseded_jobs)
# Check that old recipe and job 2 are superseded, job 1 should be copied (not superseded)
recipe = Recipe.objects.get(id=recipe.id)
job_1 = Job.objects.get(id=recipe_job_1.job_id)
job_2 = Job.objects.get(id=recipe_job_2.job_id)
self.assertTrue(recipe.is_superseded)
self.assertFalse(job_1.is_superseded)
self.assertTrue(job_2.is_superseded)
# Check that new recipe supersedes the old one, job 1 is copied from old recipe, and job 2 supersedes old job 2
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, job_name='Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, job_name='Job 2')
self.assertEqual(new_recipe.superseded_recipe_id, recipe.id)
self.assertEqual(new_recipe.root_superseded_recipe_id, recipe.id)
self.assertDictEqual(new_recipe.data, recipe.data)
self.assertEqual(new_recipe_job_1.job.id, job_1.id)
self.assertFalse(new_recipe_job_1.is_original)
self.assertIsNone(new_recipe_job_1.job.superseded_job)
self.assertIsNone(new_recipe_job_1.job.root_superseded_job)
self.assertNotEqual(new_recipe_job_2.job.id, job_2.id)
self.assertTrue(new_recipe_job_2.is_original)
self.assertEqual(new_recipe_job_2.job.superseded_job_id, job_2.id)
self.assertEqual(new_recipe_job_2.job.root_superseded_job_id, job_2.id)
def validate(self, batch):
"""Validates the given batch to make sure it is valid with respect to this batch definition. The given batch
must have all of its related fields populated, though id and root_batch_id may be None. The derived definition
attributes, such as estimated recipe total and previous batch diff, will be populated by this method.
:param batch: The batch model
:type batch: :class:`batch.models.Batch`
:returns: A list of warnings discovered during validation
:rtype: list
:raises :class:`batch.definition.exceptions.InvalidDefinition`: If the definition is invalid
"""
if self.root_batch_id:
if batch.recipe_type_id != batch.superseded_batch.recipe_type_id:
raise InvalidDefinition(
'MISMATCHED_RECIPE_TYPE',
'New batch and previous batch must have the same recipe type'
)
if not batch.superseded_batch.is_creation_done:
raise InvalidDefinition(
'PREV_BATCH_STILL_CREATING',
'Previous batch must have completed creating all of its recipes'
)
# Generate recipe diff against the previous batch
recipe_graph = batch.recipe_type_rev.get_recipe_definition(
).get_graph()
prev_recipe_graph = batch.superseded_batch.recipe_type_rev.get_recipe_definition(
).get_graph()
self.prev_batch_diff = RecipeGraphDelta(prev_recipe_graph,
recipe_graph)
if self.all_jobs:
self.job_names = recipe_graph.get_topological_order()
if self.job_names:
for job_name in self.job_names:
self.prev_batch_diff.reprocess_identical_node(job_name)
if not self.prev_batch_diff.can_be_reprocessed:
raise InvalidDefinition(
'PREV_BATCH_NO_REPROCESS',
'Previous batch cannot be reprocessed')
self._estimate_recipe_total(batch)
if not self.estimated_recipes:
raise InvalidDefinition(
'NO_RECIPES',
'Batch definition must result in creating at least one recipe')
return []
def create_recipe_handler(recipe_type=None,
data=None,
event=None,
superseded_recipe=None,
delta=None,
superseded_jobs=None):
"""Creates a recipe along with its declared jobs for unit testing
:returns: The recipe handler with created recipe and jobs
:rtype: :class:`recipe.handlers.handler.RecipeHandler`
"""
if not recipe_type:
recipe_type = create_recipe_type()
if not data:
data = {}
if not isinstance(data, LegacyRecipeData):
data = LegacyRecipeData(data)
if not event:
event = trigger_test_utils.create_trigger_event()
if superseded_recipe and not delta:
delta = RecipeGraphDelta(RecipeGraph(), RecipeGraph())
return Recipe.objects.create_recipe_old(
recipe_type,
data,
event,
superseded_recipe=superseded_recipe,
delta=delta,
superseded_jobs=superseded_jobs)
def test_convert_diff_to_v6_empty(self):
"""Tests calling convert_diff_to_v6() with an empty diff"""
# Try diff with empty graphs
graph_a = RecipeGraph()
graph_b = RecipeGraph()
diff = RecipeGraphDelta(graph_a, graph_b)
json = convert_diff_to_v6(diff)
RecipeDiffV6(diff=json.get_dict(), do_validate=True) # Revalidate
def test_convert_diff_to_v6_new_required_input(self):
"""Tests calling convert_diff_to_v6() with a diff containing a new required input that blocks reprocessing"""
job_a = job_test_utils.create_job()
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': job_a.job_type.name,
'version': job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}]
}
graph_a = OldRecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'New Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': job_a.job_type.name,
'version': job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'New Recipe Input 1',
'job_input': 'Job Input 1',
}]
}]
}
graph_b = OldRecipeDefinition(definition_b).get_graph()
diff = RecipeGraphDelta(graph_a, graph_b)
json = convert_diff_to_v6(diff)
RecipeDiffV6(diff=json.get_dict(), do_validate=True) # Revalidate
self.assertFalse(json.get_dict()['can_be_reprocessed'])
def test_successful_supersede_same_recipe_type(self):
"""Tests calling RecipeManager.create_recipe() to supersede a recipe with the same recipe type."""
event = trigger_test_utils.create_trigger_event()
handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type, event=event, data=RecipeData(self.data))
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=handler.recipe.id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=handler.recipe.id, job_name='Job 2')
superseded_jobs = {'Job 1': recipe_job_1.job, 'Job 2': recipe_job_2.job}
# Create a new recipe of the same type where we want to reprocess Job 2
graph = self.recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph, graph)
delta.reprocess_identical_node('Job 2') # We want to reprocess Job 2
new_handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type, event=event, data=None,
superseded_recipe=recipe, delta=delta,
superseded_jobs=superseded_jobs)
# Check that old recipe and job 2 are superseded, job 1 should be copied (not superseded)
recipe = Recipe.objects.get(id=recipe.id)
job_1 = Job.objects.get(id=recipe_job_1.job_id)
job_2 = Job.objects.get(id=recipe_job_2.job_id)
self.assertTrue(recipe.is_superseded)
self.assertFalse(job_1.is_superseded)
self.assertTrue(job_2.is_superseded)
# Check that new recipe supersedes the old one, job 1 is copied from old recipe, and job 2 supersedes old job 2
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=new_handler.recipe.id,
job_name='Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=new_handler.recipe.id,
job_name='Job 2')
self.assertEqual(new_recipe.superseded_recipe_id, recipe.id)
self.assertEqual(new_recipe.root_superseded_recipe_id, recipe.id)
self.assertDictEqual(new_recipe.data, recipe.data)
self.assertEqual(new_recipe_job_1.job.id, job_1.id)
self.assertFalse(new_recipe_job_1.is_original)
self.assertIsNone(new_recipe_job_1.job.superseded_job)
self.assertIsNone(new_recipe_job_1.job.root_superseded_job)
self.assertNotEqual(new_recipe_job_2.job.id, job_2.id)
self.assertTrue(new_recipe_job_2.is_original)
self.assertEqual(new_recipe_job_2.job.superseded_job_id, job_2.id)
self.assertEqual(new_recipe_job_2.job.root_superseded_job_id, job_2.id)
def test_superseded(self):
"""Tests successfully calling the recipe details view for superseded recipes."""
graph1 = RecipeGraph()
graph1.add_job('kml', self.job_type1.name, self.job_type1.version)
graph2 = RecipeGraph()
graph2.add_job('kml', self.job_type1.name, self.job_type1.version)
delta = RecipeGraphDelta(graph1, graph2)
superseded_jobs = {
recipe_job.job_name: recipe_job.job
for recipe_job in self.recipe1_jobs
}
new_recipe = recipe_test_utils.create_recipe_handler(
recipe_type=self.recipe_type,
superseded_recipe=self.recipe1,
delta=delta,
superseded_jobs=superseded_jobs).recipe
# Make sure the original recipe was updated
url = rest_util.get_url('/recipes/%i/' % self.recipe1.id)
response = self.client.generic('GET', url)
self.assertEqual(response.status_code, status.HTTP_200_OK,
response.content)
result = json.loads(response.content)
self.assertTrue(result['is_superseded'])
self.assertIsNone(result['root_superseded_recipe'])
self.assertIsNotNone(result['superseded_by_recipe'])
self.assertEqual(result['superseded_by_recipe']['id'], new_recipe.id)
self.assertIsNotNone(result['superseded'])
self.assertEqual(len(result['jobs']), 1)
for recipe_job in result['jobs']:
self.assertTrue(recipe_job['is_original'])
# Make sure the new recipe has the expected relations
url = rest_util.get_url('/recipes/%i/' % new_recipe.id)
response = self.client.generic('GET', url)
self.assertEqual(response.status_code, status.HTTP_200_OK,
response.content)
result = json.loads(response.content)
self.assertFalse(result['is_superseded'])
self.assertIsNotNone(result['root_superseded_recipe'])
self.assertEqual(result['root_superseded_recipe']['id'],
self.recipe1.id)
self.assertIsNotNone(result['superseded_recipe'])
self.assertEqual(result['superseded_recipe']['id'], self.recipe1.id)
self.assertIsNone(result['superseded'])
self.assertEqual(len(result['jobs']), 1)
for recipe_job in result['jobs']:
self.assertFalse(recipe_job['is_original'])
def test_init_new_required_input(self):
"""Tests creating a RecipeGraphDelta between two graphs where the new graph has a new required input that blocks
reprocessing
"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'New Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'New Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
self.assertFalse(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {'Job A': 'Job A'})
self.assertSetEqual(delta.get_deleted_nodes(), set())
self.assertDictEqual(delta.get_identical_nodes(), {'Job 2': 'Job B', 'Job 3': 'Job C'})
self.assertSetEqual(delta.get_new_nodes(), set())
def test_init_identical(self):
"""Tests creating a RecipeGraphDelta between two identical graphs"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_e.job_type.name,
'version': self.job_e.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job F',
'job_type': {
'name': self.job_f.job_type.name,
'version': self.job_f.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job G',
'job_type': {
'name': self.job_g.job_type.name,
'version': self.job_g.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job E',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job H',
'job_type': {
'name': self.job_h.job_type.name,
'version': self.job_h.job_type.version,
},
'dependencies': [{
'name': 'Job C',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 4',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 1',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job 5',
'job_type': {
'name': self.job_e.job_type.name,
'version': self.job_e.job_type.version,
},
'dependencies': [{
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 6',
'job_type': {
'name': self.job_f.job_type.name,
'version': self.job_f.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 7',
'job_type': {
'name': self.job_g.job_type.name,
'version': self.job_g.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 5',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job 8',
'job_type': {
'name': self.job_h.job_type.name,
'version': self.job_h.job_type.version,
},
'dependencies': [{
'name': 'Job 3',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
expected_results = {'Job 1': 'Job A', 'Job 2': 'Job B', 'Job 3': 'Job C', 'Job 4': 'Job D', 'Job 5': 'Job E',
'Job 6': 'Job F', 'Job 7': 'Job G', 'Job 8': 'Job H'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), set())
self.assertDictEqual(delta.get_identical_nodes(), expected_results)
self.assertSetEqual(delta.get_new_nodes(), set())
def reprocess_recipe(self, recipe_id, job_names=None, all_jobs=False):
"""Schedules an existing recipe for re-processing. All requested jobs, jobs that have changed in the latest
revision, and any of their dependent jobs will be re-processed. All database changes occur in an atomic
transaction. A recipe instance that is already superseded cannot be re-processed again.
:param recipe_id: The identifier of the recipe to re-process.
:type recipe_id: int
:param job_names: A list of job names from the recipe that should be forced to re-process even if the latest
recipe revision left them unchanged. If none are passed, then only jobs that changed are scheduled.
:type job_names: [string]
:param all_jobs: Indicates all jobs should be forced to re-process even if the latest recipe revision left them
unchanged. This is a convenience for passing all the individual names in the job_names parameter and this
parameter will override any values passed there.
:type all_jobs: bool
:returns: A handler for the new recipe
:rtype: :class:`recipe.handlers.handler.RecipeHandler`
"""
# Determine the old recipe graph
prev_recipe = Recipe.objects.select_related('recipe_type', 'recipe_type_rev').get(pk=recipe_id)
prev_graph = prev_recipe.get_recipe_definition().get_graph()
# Superseded recipes cannot be reprocessed
if prev_recipe.is_superseded:
raise ReprocessError('Unable to re-process a recipe that is already superseded')
# Populate the list of all job names in the recipe as a shortcut
if all_jobs:
job_names = prev_graph.get_topological_order()
# Determine the current recipe graph
current_type = prev_recipe.recipe_type
current_graph = current_type.get_recipe_definition().get_graph()
# Make sure that something is different to reprocess
if current_type.revision_num == prev_recipe.recipe_type_rev.revision_num and not job_names:
raise ReprocessError('Job names must be provided when the recipe type has not changed')
# Compute the job differences between recipe revisions including forced ones
graph_delta = RecipeGraphDelta(prev_graph, current_graph)
if job_names:
for job_name in job_names:
graph_delta.reprocess_identical_node(job_name)
# Get the old recipe jobs that will be superseded
prev_recipe_jobs = RecipeJob.objects.filter(recipe=prev_recipe)
# Acquire model locks
superseded_recipe = Recipe.objects.select_for_update().get(pk=recipe_id)
prev_jobs = Job.objects.select_for_update().filter(pk__in=[rj.job_id for rj in prev_recipe_jobs])
prev_jobs_dict = {j.id: j for j in prev_jobs}
superseded_jobs = {rj.job_name: prev_jobs_dict[rj.job_id] for rj in prev_recipe_jobs}
# Create an event to represent this request
description = {'user': '******'}
event = TriggerEvent.objects.create_trigger_event('USER', None, description, timezone.now())
# Create the new recipe while superseding the old one and queuing the associated jobs
try:
from queue.models import Queue
return Queue.objects.queue_new_recipe(current_type, None, event, superseded_recipe, graph_delta,
superseded_jobs)
except ImportError:
raise ReprocessError('Unable to import from queue application')
def test_successful_supersede(self):
"""Tests calling QueueManager.queue_new_recipe() successfully when superseding a recipe."""
# Queue initial recipe and complete its first job
node = node_test_utils.create_node()
recipe_id = Queue.objects.queue_new_recipe(self.recipe_type, self.data, self.event)
recipe = Recipe.objects.get(id=recipe_id)
recipe_job_1 = RecipeJob.objects.select_related('job__job_exe').get(recipe_id=recipe_id, job_name='Job 1')
job_exe_1 = JobExecution.objects.get(job_id=recipe_job_1.job_id)
queued_job_exe = QueuedJobExecution(Queue.objects.get(job_exe_id=job_exe_1.id))
queued_job_exe.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000, disk_total=2000))
Queue.objects.schedule_job_executions('123', [queued_job_exe], {})
results = JobResults()
results.add_file_list_parameter('Test Output 1', [product_test_utils.create_product().file_id])
JobExecution.objects.filter(id=job_exe_1.id).update(results=results.get_dict())
Queue.objects.handle_job_completion(job_exe_1.id, now())
# Create a new recipe type that has a new version of job 2 (job 1 is identical)
new_job_type_2 = job_test_utils.create_job_type(name=self.job_type_2.name, version='New Version',
interface=self.job_type_2.interface)
new_definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'New Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name': 'New Job 2',
'job_type': {
'name': new_job_type_2.name,
'version': new_job_type_2.version,
},
'dependencies': [{
'name': 'New Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 2',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(name=self.recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=recipe_id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=recipe_id, job_name='Job 2')
superseded_jobs = {'Job 1': recipe_job_1.job, 'Job 2': recipe_job_2.job}
graph_a = self.recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
# Queue new recipe that supersedes the old recipe
new_recipe_id = Queue.objects.queue_new_recipe(new_recipe_type, None, event, recipe, delta, superseded_jobs)
# Ensure old recipe is superseded
recipe = Recipe.objects.get(id=recipe_id)
self.assertTrue(recipe.is_superseded)
# Ensure new recipe supersedes old recipe
new_recipe = Recipe.objects.get(id=new_recipe_id)
self.assertEqual(new_recipe.superseded_recipe_id, recipe_id)
# Ensure that job 1 is already completed (it was copied from original recipe) and that job 2 is queued
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=new_recipe_id, job_name='New Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=new_recipe_id, job_name='New Job 2')
self.assertEqual(new_recipe_job_1.job.status, 'COMPLETED')
self.assertFalse(new_recipe_job_1.is_original)
self.assertEqual(new_recipe_job_2.job.status, 'QUEUED')
self.assertTrue(new_recipe_job_2.is_original)
# Complete both the old and new job 2 and check that only the new recipe completes
job_exe_2 = JobExecution.objects.get(job_id=recipe_job_2.job_id)
queued_job_exe_2 = QueuedJobExecution(Queue.objects.get(job_exe_id=job_exe_2.id))
queued_job_exe_2.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000, disk_total=2000))
Queue.objects.schedule_job_executions('123', [queued_job_exe_2], {})
Queue.objects.handle_job_completion(job_exe_2.id, now())
new_job_exe_2 = JobExecution.objects.get(job_id=new_recipe_job_2.job_id)
new_queued_job_exe_2 = QueuedJobExecution(Queue.objects.get(job_exe_id=new_job_exe_2.id))
new_queued_job_exe_2.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000,
disk_total=2000))
Queue.objects.schedule_job_executions('123', [new_queued_job_exe_2], {})
Queue.objects.handle_job_completion(new_job_exe_2.id, now())
recipe = Recipe.objects.get(id=recipe.id)
new_recipe = Recipe.objects.get(id=new_recipe.id)
self.assertIsNone(recipe.completed)
self.assertIsNotNone(new_recipe.completed)
def test_init_identical(self):
"""Tests creating a RecipeGraphDelta between two identical graphs"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_e.job_type.name,
'version': self.job_e.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job F',
'job_type': {
'name': self.job_f.job_type.name,
'version': self.job_f.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job G',
'job_type': {
'name': self.job_g.job_type.name,
'version': self.job_g.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job E',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job H',
'job_type': {
'name': self.job_h.job_type.name,
'version': self.job_h.job_type.version,
},
'dependencies': [{
'name': 'Job C',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 4',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 1',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job 5',
'job_type': {
'name': self.job_e.job_type.name,
'version': self.job_e.job_type.version,
},
'dependencies': [{
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 6',
'job_type': {
'name': self.job_f.job_type.name,
'version': self.job_f.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 7',
'job_type': {
'name': self.job_g.job_type.name,
'version': self.job_g.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 5',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job 8',
'job_type': {
'name': self.job_h.job_type.name,
'version': self.job_h.job_type.version,
},
'dependencies': [{
'name': 'Job 3',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
expected_results = {'Job 1': 'Job A', 'Job 2': 'Job B', 'Job 3': 'Job C', 'Job 4': 'Job D', 'Job 5': 'Job E',
'Job 6': 'Job F', 'Job 7': 'Job G', 'Job 8': 'Job H'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), set())
self.assertDictEqual(delta.get_identical_nodes(), expected_results)
self.assertSetEqual(delta.get_new_nodes(), set())
def _handle_recipe_jobs(self, msg_already_run, recipes, new_revision_id,
revisions, recipe_job_ids, job_names, all_jobs,
when):
"""Handles the reprocessing of the recipe jobs
:param msg_already_run: Whether the database transaction has already occurred
:type msg_already_run: bool
:param recipes: The new recipe models
:type recipes: list
:param new_revision_id: The ID of the new recipe type revision to use for reprocessing
:type new_revision_id: int
:param revisions: Recipe type revisions stored by revision ID
:type revisions: dict
:param recipe_job_ids: Dict where recipe ID maps to a dict where job_name maps to a list of job IDs
:type recipe_job_ids: dict
:param job_names: The job names within the recipes to force reprocess
:type job_names: list
:param all_jobs: If True then all jobs within the recipe should be reprocessed, False otherwise
:type all_jobs: bool
:param when: The time that the jobs were superseded
:type when: :class:`datetime.datetime`
:return: A list of messages that should be sent regarding the superseded jobs
:rtype: list
"""
superseded_job_ids = []
unpublish_job_ids = []
recipe_job_models = []
recipe_job_count = 0
new_graph = revisions[new_revision_id].get_recipe_definition(
).get_graph()
for recipe in recipes:
job_ids = recipe_job_ids[
recipe.
superseded_recipe_id] # Get job IDs for superseded recipe
old_graph = revisions[
recipe.recipe_type_rev_id].get_recipe_definition().get_graph()
names = old_graph.get_topological_order(
) if all_jobs else job_names
# Compute the job differences between recipe revisions (force reprocess for jobs in job_names)
graph_delta = RecipeGraphDelta(old_graph, new_graph)
for job_name in names:
graph_delta.reprocess_identical_node(job_name)
# Jobs that are identical from old recipe to new recipe are just copied to new recipe
if not msg_already_run:
for identical_job_name in graph_delta.get_identical_nodes():
if identical_job_name in job_ids:
for job_id in job_ids[identical_job_name]:
recipe_job = RecipeNode()
recipe_job.job_id = job_id
recipe_job.node_name = identical_job_name
recipe_job.recipe_id = recipe.id
recipe_job.is_original = False
recipe_job_count += 1
recipe_job_models.append(recipe_job)
if len(recipe_job_models) >= MODEL_BATCH_SIZE:
RecipeNode.objects.bulk_create(
recipe_job_models)
recipe_job_models = []
# Jobs that changed from old recipe to new recipe should be superseded
for changed_job_name in graph_delta.get_changed_nodes():
if changed_job_name in job_ids:
superseded_job_ids.extend(job_ids[changed_job_name])
# Jobs that were deleted from old recipe to new recipe should be superseded and unpublished
for deleted_job_name in graph_delta.get_deleted_nodes():
if deleted_job_name in job_ids:
superseded_job_ids.extend(job_ids[deleted_job_name])
unpublish_job_ids.extend(job_ids[deleted_job_name])
# Finish creating any remaining RecipeNode models
if recipe_job_models and not msg_already_run:
RecipeNode.objects.bulk_create(recipe_job_models)
logger.info('Copied %d job(s) to the new recipe(s)', recipe_job_count)
# Supersede recipe jobs that were not copied over to a new recipe
if not msg_already_run:
Job.objects.supersede_jobs(superseded_job_ids, when)
logger.info('Superseded %d job(s)', len(superseded_job_ids))
logger.info('Found %d job(s) that should be unpublished',
len(unpublish_job_ids))
# Create messages to unpublish and cancel jobs
messages = create_cancel_jobs_messages(superseded_job_ids, when)
messages.extend(create_unpublish_jobs_messages(unpublish_job_ids,
when))
return messages
def test_init_changed(self):
"""Tests creating a RecipeGraphDelta between two graphs where some nodes were changed (and 1 deleted)"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': 'new_version',
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
expected_identical = {'Job A': 'Job A', 'Job B': 'Job B'}
expected_changed = {'Job D': 'Job D', 'Job E': 'Job E'}
expected_deleted = {'Job C'}
expected_new = set()
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), expected_changed)
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
def test_init_new_required_input(self):
"""Tests creating a RecipeGraphDelta between two graphs where the new graph has a new required input that blocks
reprocessing
"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'New Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'New Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
self.assertFalse(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), set())
self.assertDictEqual(delta.get_identical_nodes(), {})
self.assertSetEqual(delta.get_new_nodes(), set())
def test_successful_supersede_mixed(self):
"""Tests calling QueueManager.queue_new_recipe() successfully when superseding a recipe where the results of a
Seed job get passed to the input of a legacy job
"""
workspace = storage_test_utils.create_workspace()
source_file = source_test_utils.create_source(workspace=workspace)
event = trigger_test_utils.create_trigger_event()
interface_1 = {
'seedVersion': '1.0.0',
'job': {
'name': 'job-type-a',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': 'Job Type 1',
'description': 'This is a description',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': '',
'inputs': {
'files': [{
'name': 'test-input-a'
}]
},
'outputs': {
'files': [{
'name': 'test-output-a',
'pattern': '*.png'
}]
}
}
}
}
job_type_1 = job_test_utils.create_seed_job_type(manifest=interface_1)
interface_2 = {
'version':
'1.0',
'command':
'test_command',
'command_arguments':
'test_arg',
'input_data': [{
'name': 'Test Input 2',
'type': 'file',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}]
}
job_type_2 = job_test_utils.create_job_type(interface=interface_2)
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': job_type_1.name,
'version': job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'test-input-a',
}]
}, {
'name':
'Job 2',
'job_type': {
'name': job_type_2.name,
'version': job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': 'test-output-a',
'input': 'Test Input 2',
}]
}]
}]
}
recipe_definition = RecipeDefinition(definition)
recipe_definition.validate_job_interfaces()
recipe_type = recipe_test_utils.create_recipe_type(
definition=definition)
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': source_file.id,
}],
'workspace_id': workspace.id,
}
data = LegacyRecipeData(data)
# Queue initial recipe and complete its first job
handler = Queue.objects.queue_new_recipe(recipe_type, data, event)
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeNode.objects.select_related('job')
recipe_job_1 = recipe_job_1.get(recipe_id=handler.recipe.id,
node_name='Job 1')
Job.objects.update_jobs_to_running([recipe_job_1.job], now())
results = JobResults()
results.add_file_parameter('test-output-a',
product_test_utils.create_product().id)
job_test_utils.create_job_exe(job=recipe_job_1.job,
status='COMPLETED',
output=results)
Job.objects.update_jobs_to_completed([recipe_job_1.job], now())
Job.objects.process_job_output([recipe_job_1.job_id], now())
# Create a new recipe type that has a new version of job 2 (job 1 is identical)
new_job_type_2 = job_test_utils.create_job_type(
name=job_type_2.name,
version='New Version',
interface=job_type_2.manifest)
new_definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'New Job 1',
'job_type': {
'name': job_type_1.name,
'version': job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'test-input-a',
}]
}, {
'name':
'New Job 2',
'job_type': {
'name': new_job_type_2.name,
'version': new_job_type_2.version,
},
'dependencies': [{
'name':
'New Job 1',
'connections': [{
'output': 'test-output-a',
'input': 'Test Input 2',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(
name=recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
recipe_job_1 = RecipeNode.objects.select_related('job').get(
recipe_id=handler.recipe.id, node_name='Job 1')
recipe_job_2 = RecipeNode.objects.select_related('job').get(
recipe_id=handler.recipe.id, node_name='Job 2')
superseded_jobs = {
'Job 1': recipe_job_1.job,
'Job 2': recipe_job_2.job
}
graph_a = recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
# Queue new recipe that supersedes the old recipe
new_handler = Queue.objects.queue_new_recipe(
new_recipe_type,
None,
event,
superseded_recipe=recipe,
delta=delta,
superseded_jobs=superseded_jobs)
# Ensure old recipe is superseded
recipe = Recipe.objects.get(id=handler.recipe.id)
self.assertTrue(recipe.is_superseded)
# Ensure new recipe supersedes old recipe
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
self.assertEqual(new_recipe.superseded_recipe_id, handler.recipe.id)
# Ensure that job 1 is already completed (it was copied from original recipe) and that job 2 is queued
new_recipe_job_1 = RecipeNode.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, node_name='New Job 1')
new_recipe_job_2 = RecipeNode.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, node_name='New Job 2')
self.assertEqual(new_recipe_job_1.job.status, 'COMPLETED')
self.assertFalse(new_recipe_job_1.is_original)
self.assertEqual(new_recipe_job_2.job.status, 'QUEUED')
self.assertTrue(new_recipe_job_2.is_original)
def test_successful_supersede(self):
"""Tests calling QueueManager.queue_new_recipe() successfully when superseding a recipe."""
# Queue initial recipe and complete its first job
handler = Queue.objects.queue_new_recipe(self.recipe_type, self.data,
self.event)
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeNode.objects.select_related('job')
recipe_job_1 = recipe_job_1.get(recipe_id=handler.recipe.id,
node_name='Job 1')
Job.objects.update_jobs_to_running([recipe_job_1.job], now())
results = JobResults()
results.add_file_list_parameter(
'Test Output 1', [product_test_utils.create_product().id])
job_test_utils.create_job_exe(job=recipe_job_1.job,
status='COMPLETED',
output=results)
Job.objects.update_jobs_to_completed([recipe_job_1.job], now())
Job.objects.process_job_output([recipe_job_1.job_id], now())
# Create a new recipe type that has a new version of job 2 (job 1 is identical)
new_job_type_2 = job_test_utils.create_job_type(
name=self.job_type_2.name,
version='New Version',
interface=self.job_type_2.manifest)
new_definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'New Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name':
'New Job 2',
'job_type': {
'name': new_job_type_2.name,
'version': new_job_type_2.version,
},
'dependencies': [{
'name':
'New Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 2',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(
name=self.recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
recipe_job_1 = RecipeNode.objects.select_related('job').get(
recipe_id=handler.recipe.id, node_name='Job 1')
recipe_job_2 = RecipeNode.objects.select_related('job').get(
recipe_id=handler.recipe.id, node_name='Job 2')
superseded_jobs = {
'Job 1': recipe_job_1.job,
'Job 2': recipe_job_2.job
}
graph_a = self.recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
# Queue new recipe that supersedes the old recipe
new_handler = Queue.objects.queue_new_recipe(
new_recipe_type,
None,
event,
superseded_recipe=recipe,
delta=delta,
superseded_jobs=superseded_jobs)
# Ensure old recipe is superseded
recipe = Recipe.objects.get(id=handler.recipe.id)
self.assertTrue(recipe.is_superseded)
# Ensure new recipe supersedes old recipe
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
self.assertEqual(new_recipe.superseded_recipe_id, handler.recipe.id)
# Ensure that job 1 is already completed (it was copied from original recipe) and that job 2 is queued
new_recipe_job_1 = RecipeNode.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, node_name='New Job 1')
new_recipe_job_2 = RecipeNode.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, node_name='New Job 2')
self.assertEqual(new_recipe_job_1.job.status, 'COMPLETED')
self.assertFalse(new_recipe_job_1.is_original)
self.assertEqual(new_recipe_job_2.job.status, 'QUEUED')
self.assertTrue(new_recipe_job_2.is_original)
def test_successful_supersede_different_recipe_type(self):
"""Tests calling RecipeManager.create_recipe() to supersede a recipe with a different recipe type version that
has one identical node, and deletes another node to replace it with a new one.
"""
interface_3 = {
'version':
'1.0',
'command':
'my_command',
'command_arguments':
'args',
'input_data': [{
'name': 'Test Input 3',
'type': 'files',
'media_types': ['image/tiff'],
}],
'output_data': [{
'name': 'Test Output 3',
'type': 'file',
}]
}
job_type_3 = job_test_utils.create_job_type(interface=interface_3)
new_definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name':
'Job 3',
'job_type': {
'name': job_type_3.name,
'version': job_type_3.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 3',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(
name=self.recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
event=event,
data=RecipeData(self.data))
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 2')
job_exe_2 = job_test_utils.create_job_exe(job=recipe_job_2.job)
try:
from product.models import ProductFile
from product.test import utils as product_test_utils
product = product_test_utils.create_product(
job_exe=job_exe_2, has_been_published=True, is_published=True)
except ImportError:
product = None
superseded_jobs = {
'Job 1': recipe_job_1.job,
'Job 2': recipe_job_2.job
}
# Create a new recipe with a different version
graph_a = self.recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
new_handler = Recipe.objects.create_recipe(
recipe_type=new_recipe_type,
event=event,
data=None,
superseded_recipe=recipe,
delta=delta,
superseded_jobs=superseded_jobs)
# Check that old recipe and job 2 are superseded, job 1 should be copied (not superseded)
recipe = Recipe.objects.get(id=recipe.id)
job_1 = Job.objects.get(id=recipe_job_1.job_id)
job_2 = Job.objects.get(id=recipe_job_2.job_id)
self.assertTrue(recipe.is_superseded)
self.assertFalse(job_1.is_superseded)
self.assertTrue(job_2.is_superseded)
# Check that product of job 2 (which was superseded with no new job) was unpublished
if product:
product = ProductFile.objects.get(id=product.id)
self.assertFalse(product.is_published)
self.assertIsNotNone(product.unpublished)
# Check that new recipe supersedes the old one, job 1 is copied from old recipe, and job 2 is new and does not
# supersede anything
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, job_name='Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, job_name='Job 3')
self.assertEqual(new_recipe.superseded_recipe_id, recipe.id)
self.assertEqual(new_recipe.root_superseded_recipe_id, recipe.id)
self.assertDictEqual(new_recipe.data, recipe.data)
self.assertEqual(new_recipe_job_1.job.id, job_1.id)
self.assertFalse(new_recipe_job_1.is_original)
self.assertIsNone(new_recipe_job_1.job.superseded_job)
self.assertIsNone(new_recipe_job_1.job.root_superseded_job)
self.assertNotEqual(new_recipe_job_2.job.id, job_2.id)
self.assertTrue(new_recipe_job_2.is_original)
self.assertIsNone(new_recipe_job_2.job.superseded_job_id)
self.assertIsNone(new_recipe_job_2.job.root_superseded_job_id)
def test_init_deleted_and_new(self):
"""Tests creating a RecipeGraphDelta between two graphs where some nodes were deleted and new nodes added"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 4',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 3',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job 5',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 3',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
expected_identical = {'Job 1': 'Job A', 'Job 2': 'Job B', 'Job 3': 'Job C'}
expected_deleted = {'Job D', 'Job E'}
expected_new = {'Job 4', 'Job 5'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
def test_reprocess_identical_node(self):
"""Tests calling RecipeGraphDelta.reprocess_identical_node() to indicate identical nodes that should be marked
as changed"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 4',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 5',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
# Initial delta
delta = RecipeGraphDelta(graph_a, graph_b)
expected_identical = {'Job 1': 'Job A', 'Job 2': 'Job B', 'Job 4': 'Job D'}
expected_deleted = {'Job C'}
expected_new = {'Job 5'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
# Mark Job 2 (and its child Job 4) as changed so it will be reprocessed
delta.reprocess_identical_node('Job 2')
expected_changed = {'Job 2': 'Job B', 'Job 4': 'Job D'}
expected_identical = {'Job 1': 'Job A'}
expected_deleted = {'Job C'}
expected_new = {'Job 5'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), expected_changed)
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
def test_reprocess_identical_node(self):
"""Tests calling RecipeGraphDelta.reprocess_identical_node() to indicate identical nodes that should be marked
as changed"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 4',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 5',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
# Initial delta
delta = RecipeGraphDelta(graph_a, graph_b)
expected_identical = {'Job 1': 'Job A', 'Job 2': 'Job B', 'Job 4': 'Job D'}
expected_deleted = {'Job C'}
expected_new = {'Job 5'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
# Mark Job 2 (and its child Job 4) as changed so it will be reprocessed
delta.reprocess_identical_node('Job 2')
expected_changed = {'Job 2': 'Job B', 'Job 4': 'Job D'}
expected_identical = {'Job 1': 'Job A'}
expected_deleted = {'Job C'}
expected_new = {'Job 5'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), expected_changed)
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
def test_convert_diff_to_v6_full_diff(self):
"""Tests calling convert_diff_to_v6() with a full diff with all types (deleted, new, changed, etc) of nodes"""
job_a = job_test_utils.create_job()
job_b = job_test_utils.create_job()
job_c = job_test_utils.create_job()
job_d = job_test_utils.create_job()
job_e = job_test_utils.create_job()
job_f = job_test_utils.create_job()
new_job_type_d = JobType.objects.get(id=job_d.job_type_id)
new_job_type_d.version = 'new_version'
new_job_type_d.save()
definition_a = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name':
'Job A',
'job_type': {
'name': job_a.job_type.name,
'version': job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name':
'Job B',
'job_type': {
'name': job_b.job_type.name,
'version': job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name':
'Job C',
'job_type': {
'name': job_c.job_type.name,
'version': job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name':
'Job D',
'job_type': {
'name': job_d.job_type.name,
'version': job_d.job_type.version,
},
'dependencies': [{
'name':
'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name':
'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name':
'Job E',
'job_type': {
'name': job_e.job_type.name,
'version': job_e.job_type.version,
},
'dependencies': [{
'name':
'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_a = OldRecipeDefinition(definition_a).get_graph()
definition_b = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name':
'Job A',
'job_type': {
'name': job_a.job_type.name,
'version': job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name':
'Job B',
'job_type': {
'name': job_b.job_type.name,
'version': job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name':
'Job D',
'job_type': {
'name': new_job_type_d.name,
'version': new_job_type_d.version,
},
'dependencies': [{
'name':
'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name':
'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name':
'Job E',
'job_type': {
'name': job_e.job_type.name,
'version': job_e.job_type.version,
},
'dependencies': [{
'name':
'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name':
'Job F',
'job_type': {
'name': job_f.job_type.name,
'version': job_f.job_type.version,
},
'dependencies': [{
'name':
'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_b = OldRecipeDefinition(definition_b).get_graph()
diff = RecipeGraphDelta(graph_a, graph_b)
json = convert_diff_to_v6(diff)
RecipeDiffV6(diff=json.get_dict(), do_validate=True) # Revalidate
self.assertTrue(json.get_dict()['can_be_reprocessed'])