def test_init_changed(self):
"""Tests creating a RecipeGraphDelta between two graphs where some nodes were changed (and 1 deleted)"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': 'new_version',
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
expected_identical = {'Job A': 'Job A', 'Job B': 'Job B'}
expected_changed = {'Job D': 'Job D', 'Job E': 'Job E'}
expected_deleted = {'Job C'}
expected_new = set()
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), expected_changed)
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
def test_reprocess_identical_node(self):
"""Tests calling RecipeGraphDelta.reprocess_identical_node() to indicate identical nodes that should be marked
as changed"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 4',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 5',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
# Initial delta
delta = RecipeGraphDelta(graph_a, graph_b)
expected_identical = {'Job 1': 'Job A', 'Job 2': 'Job B', 'Job 4': 'Job D'}
expected_deleted = {'Job C'}
expected_new = {'Job 5'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
# Mark Job 2 (and its child Job 4) as changed so it will be reprocessed
delta.reprocess_identical_node('Job 2')
expected_changed = {'Job 2': 'Job B', 'Job 4': 'Job D'}
expected_identical = {'Job 1': 'Job A'}
expected_deleted = {'Job C'}
expected_new = {'Job 5'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), expected_changed)
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
def test_init_identical(self):
"""Tests creating a RecipeGraphDelta between two identical graphs"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_e.job_type.name,
'version': self.job_e.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job F',
'job_type': {
'name': self.job_f.job_type.name,
'version': self.job_f.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job G',
'job_type': {
'name': self.job_g.job_type.name,
'version': self.job_g.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job E',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job H',
'job_type': {
'name': self.job_h.job_type.name,
'version': self.job_h.job_type.version,
},
'dependencies': [{
'name': 'Job C',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 4',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 1',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job 5',
'job_type': {
'name': self.job_e.job_type.name,
'version': self.job_e.job_type.version,
},
'dependencies': [{
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 6',
'job_type': {
'name': self.job_f.job_type.name,
'version': self.job_f.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 7',
'job_type': {
'name': self.job_g.job_type.name,
'version': self.job_g.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 5',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job 8',
'job_type': {
'name': self.job_h.job_type.name,
'version': self.job_h.job_type.version,
},
'dependencies': [{
'name': 'Job 3',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
expected_results = {'Job 1': 'Job A', 'Job 2': 'Job B', 'Job 3': 'Job C', 'Job 4': 'Job D', 'Job 5': 'Job E',
'Job 6': 'Job F', 'Job 7': 'Job G', 'Job 8': 'Job H'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), set())
self.assertDictEqual(delta.get_identical_nodes(), expected_results)
self.assertSetEqual(delta.get_new_nodes(), set())
def test_init_new_required_input(self):
"""Tests creating a RecipeGraphDelta between two graphs where the new graph has a new required input that blocks
reprocessing
"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'New Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'New Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
self.assertFalse(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {'Job A': 'Job A'})
self.assertSetEqual(delta.get_deleted_nodes(), set())
self.assertDictEqual(delta.get_identical_nodes(), {'Job 2': 'Job B', 'Job 3': 'Job C'})
self.assertSetEqual(delta.get_new_nodes(), set())
def _handle_recipe_jobs(self, msg_already_run, recipes, new_revision_id,
revisions, recipe_job_ids, job_names, all_jobs,
when):
"""Handles the reprocessing of the recipe jobs
:param msg_already_run: Whether the database transaction has already occurred
:type msg_already_run: bool
:param recipes: The new recipe models
:type recipes: list
:param new_revision_id: The ID of the new recipe type revision to use for reprocessing
:type new_revision_id: int
:param revisions: Recipe type revisions stored by revision ID
:type revisions: dict
:param recipe_job_ids: Dict where recipe ID maps to a dict where job_name maps to a list of job IDs
:type recipe_job_ids: dict
:param job_names: The job names within the recipes to force reprocess
:type job_names: list
:param all_jobs: If True then all jobs within the recipe should be reprocessed, False otherwise
:type all_jobs: bool
:param when: The time that the jobs were superseded
:type when: :class:`datetime.datetime`
:return: A list of messages that should be sent regarding the superseded jobs
:rtype: list
"""
superseded_job_ids = []
unpublish_job_ids = []
recipe_job_models = []
recipe_job_count = 0
new_graph = revisions[new_revision_id].get_recipe_definition(
).get_graph()
for recipe in recipes:
job_ids = recipe_job_ids[
recipe.
superseded_recipe_id] # Get job IDs for superseded recipe
old_graph = revisions[
recipe.recipe_type_rev_id].get_recipe_definition().get_graph()
names = old_graph.get_topological_order(
) if all_jobs else job_names
# Compute the job differences between recipe revisions (force reprocess for jobs in job_names)
graph_delta = RecipeGraphDelta(old_graph, new_graph)
for job_name in names:
graph_delta.reprocess_identical_node(job_name)
# Jobs that are identical from old recipe to new recipe are just copied to new recipe
if not msg_already_run:
for identical_job_name in graph_delta.get_identical_nodes():
if identical_job_name in job_ids:
for job_id in job_ids[identical_job_name]:
recipe_job = RecipeNode()
recipe_job.job_id = job_id
recipe_job.node_name = identical_job_name
recipe_job.recipe_id = recipe.id
recipe_job.is_original = False
recipe_job_count += 1
recipe_job_models.append(recipe_job)
if len(recipe_job_models) >= MODEL_BATCH_SIZE:
RecipeNode.objects.bulk_create(
recipe_job_models)
recipe_job_models = []
# Jobs that changed from old recipe to new recipe should be superseded
for changed_job_name in graph_delta.get_changed_nodes():
if changed_job_name in job_ids:
superseded_job_ids.extend(job_ids[changed_job_name])
# Jobs that were deleted from old recipe to new recipe should be superseded and unpublished
for deleted_job_name in graph_delta.get_deleted_nodes():
if deleted_job_name in job_ids:
superseded_job_ids.extend(job_ids[deleted_job_name])
unpublish_job_ids.extend(job_ids[deleted_job_name])
# Finish creating any remaining RecipeNode models
if recipe_job_models and not msg_already_run:
RecipeNode.objects.bulk_create(recipe_job_models)
logger.info('Copied %d job(s) to the new recipe(s)', recipe_job_count)
# Supersede recipe jobs that were not copied over to a new recipe
if not msg_already_run:
Job.objects.supersede_jobs(superseded_job_ids, when)
logger.info('Superseded %d job(s)', len(superseded_job_ids))
logger.info('Found %d job(s) that should be unpublished',
len(unpublish_job_ids))
# Create messages to unpublish and cancel jobs
messages = create_cancel_jobs_messages(superseded_job_ids, when)
messages.extend(create_unpublish_jobs_messages(unpublish_job_ids,
when))
return messages
def test_reprocess_identical_node(self):
"""Tests calling RecipeGraphDelta.reprocess_identical_node() to indicate identical nodes that should be marked
as changed"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 4',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 5',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
# Initial delta
delta = RecipeGraphDelta(graph_a, graph_b)
expected_identical = {'Job 1': 'Job A', 'Job 2': 'Job B', 'Job 4': 'Job D'}
expected_deleted = {'Job C'}
expected_new = {'Job 5'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
# Mark Job 2 (and its child Job 4) as changed so it will be reprocessed
delta.reprocess_identical_node('Job 2')
expected_changed = {'Job 2': 'Job B', 'Job 4': 'Job D'}
expected_identical = {'Job 1': 'Job A'}
expected_deleted = {'Job C'}
expected_new = {'Job 5'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), expected_changed)
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
def test_init_deleted_and_new(self):
"""Tests creating a RecipeGraphDelta between two graphs where some nodes were deleted and new nodes added"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 4',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 3',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job 5',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 3',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
expected_identical = {'Job 1': 'Job A', 'Job 2': 'Job B', 'Job 3': 'Job C'}
expected_deleted = {'Job D', 'Job E'}
expected_new = {'Job 4', 'Job 5'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
def test_init_new_required_input(self):
"""Tests creating a RecipeGraphDelta between two graphs where the new graph has a new required input that blocks
reprocessing
"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'New Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'New Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
self.assertFalse(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), set())
self.assertDictEqual(delta.get_identical_nodes(), {})
self.assertSetEqual(delta.get_new_nodes(), set())
def test_init_identical(self):
"""Tests creating a RecipeGraphDelta between two identical graphs"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_e.job_type.name,
'version': self.job_e.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job F',
'job_type': {
'name': self.job_f.job_type.name,
'version': self.job_f.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job G',
'job_type': {
'name': self.job_g.job_type.name,
'version': self.job_g.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job E',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job H',
'job_type': {
'name': self.job_h.job_type.name,
'version': self.job_h.job_type.version,
},
'dependencies': [{
'name': 'Job C',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 4',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 1',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job 5',
'job_type': {
'name': self.job_e.job_type.name,
'version': self.job_e.job_type.version,
},
'dependencies': [{
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 6',
'job_type': {
'name': self.job_f.job_type.name,
'version': self.job_f.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 7',
'job_type': {
'name': self.job_g.job_type.name,
'version': self.job_g.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 5',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job 8',
'job_type': {
'name': self.job_h.job_type.name,
'version': self.job_h.job_type.version,
},
'dependencies': [{
'name': 'Job 3',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
expected_results = {'Job 1': 'Job A', 'Job 2': 'Job B', 'Job 3': 'Job C', 'Job 4': 'Job D', 'Job 5': 'Job E',
'Job 6': 'Job F', 'Job 7': 'Job G', 'Job 8': 'Job H'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), set())
self.assertDictEqual(delta.get_identical_nodes(), expected_results)
self.assertSetEqual(delta.get_new_nodes(), set())
