def get_definition(self):
"""Returns the batch definition represented by this JSON
:returns: The batch definition
:rtype: :class:`batch.definition.definition.BatchDefinition`:
"""
definition = BatchDefinition()
if 'dataset' in self._definition:
definition.dataset = self._definition['dataset']
if 'forced_nodes' in self._definition:
definition.forced_nodes = ForcedNodesV6(
self._definition['forced_nodes']).get_forced_nodes()
if 'supersedes' in self._definition:
definition.supersedes = self._definition['supersedes']
if 'previous_batch' in self._definition:
prev_batch_dict = self._definition['previous_batch']
definition.root_batch_id = prev_batch_dict['root_batch_id']
if 'forced_nodes' in prev_batch_dict:
definition.forced_nodes = ForcedNodesV6(
prev_batch_dict['forced_nodes']).get_forced_nodes()
return definition
def test_execute_reprocess(self):
"""Tests calling CreateBatchRecipes.execute() successfully when re-processing recipes"""
# Importing module here to patch the max recipe num
import batch.messages.create_batch_recipes
batch.messages.create_batch_recipes.MAX_RECIPE_NUM = 5
jt_1 = job_test_utils.create_seed_job_type()
jt_2 = job_test_utils.create_seed_job_type()
jt_3 = job_test_utils.create_seed_job_type()
jt_4 = job_test_utils.create_seed_job_type()
recipe_def = {
'version': '7',
'input': {
'files': [{
'name': 'INPUT_IMAGE',
'media_types': ['image/png'],
'required': True,
'multiple': False
}],
'json': []
},
'nodes': {
'node_a': {
'dependencies': [],
'input': {
'input_a': {
'type': 'recipe',
'input': 'INPUT_IMAGE'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': jt_1.name,
'job_type_version': jt_1.version,
'job_type_revision': jt_1.revision_num
}
},
'node_b': {
'dependencies': [],
'input': {
'input_a': {
'type': 'recipe',
'input': 'INPUT_IMAGE'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': jt_2.name,
'job_type_version': jt_2.version,
'job_type_revision': jt_2.revision_num
}
}
}
}
sub_recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=recipe_def)
# Recipe with two jobs and one subrecipe (c -> d -> r)
recipe_def = {
'version': '7',
'input': {
'files': [{
'name': 'INPUT_IMAGE',
'media_types': ['image/png'],
'required': True,
'multiple': False
}],
'json': []
},
'nodes': {
'recipe_node': {
'dependencies': [{
'name': 'node_d',
'acceptance': True
}],
'input': {
'input_a': {
'type': 'dependency',
'node': 'node_d',
'output': 'OUTPUT_IMAGE'
}
},
'node_type': {
'node_type': 'recipe',
'recipe_type_name': sub_recipe_type.name,
'recipe_type_revision': sub_recipe_type.revision_num
}
},
'node_c': {
'dependencies': [],
'input': {
'INPUT_IMAGE': {
'type': 'recipe',
'input': 'INPUT_IMAGE'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': jt_3.name,
'job_type_version': jt_3.version,
'job_type_revision': jt_3.revision_num
}
},
'node_d': {
'dependencies': [{
'name': 'node_c',
'acceptance': True
}],
'input': {
'INPUT_IMAGE': {
'type': 'dependency',
'node': 'node_c',
'output': 'OUTPUT_IMAGE'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': jt_4.name,
'job_type_version': jt_4.version,
'job_type_revision': jt_4.revision_num
}
}
}
}
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=recipe_def)
# Create a dataset of 6 files
dataset_def = {
'parameters': {
'files': [{
'media_types': ['image/png'],
'required': True,
'multiple': False,
'name': 'INPUT_IMAGE'
}],
'json': []
}
}
the_dataset = data_test_utils.create_dataset(definition=dataset_def)
workspace = storage_test_utils.create_workspace()
# Create 6 files & recipes to go along
src_file_ids = []
recipe_ids = []
data_list = []
for i in range(0, 6):
file_name = 'input_%d.png' % i
src_file = storage_test_utils.create_file(file_name=file_name,
file_type='SOURCE',
media_type='image/png',
file_size=10,
data_type_tags=['type'],
file_path='the_path',
workspace=workspace)
src_file_ids.append(src_file.id)
data_dict = {
'version': '6',
'files': {
'INPUT_IMAGE': [src_file.id]
},
'json': {}
}
data_list.append(DataV6(data=data_dict).get_dict())
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_dict)
recipe_ids.append(recipe.id)
members = data_test_utils.create_dataset_members(dataset=the_dataset,
data_list=data_list)
recipe_test_utils.process_recipe_inputs(recipe_ids)
batch_definition = BatchDefinition()
batch_definition.dataset = the_dataset.id
batch_definition.supersedes = True
forced_nodes = ForcedNodes()
forced_nodes.all_nodes = True
batch_definition.forced_nodes = forced_nodes
new_batch = batch_test_utils.create_batch(recipe_type=recipe_type,
definition=batch_definition)
# Create message
message = batch.messages.create_batch_recipes.CreateBatchRecipes()
message.batch_id = new_batch.id
# Execute message
result = message.execute()
self.assertTrue(result)
self.assertEqual(len(message.new_messages), 2)
batch_recipes_message = message.new_messages[0]
create_recipes_message = message.new_messages[1]
self.assertEqual(batch_recipes_message.type, 'create_batch_recipes')
self.assertEqual(batch_recipes_message.batch_id, new_batch.id)
self.assertFalse(batch_recipes_message.is_prev_batch_done)
self.assertEqual(batch_recipes_message.current_recipe_id,
recipe_ids[1])
# Test the create_recipes_message
self.assertEqual(create_recipes_message.type, 'create_recipes')
self.assertSetEqual(
set(create_recipes_message.root_recipe_ids), {
recipe_ids[5], recipe_ids[4], recipe_ids[3], recipe_ids[2],
recipe_ids[1]
})
self.assertEqual(create_recipes_message.batch_id, new_batch.id)
self.assertEqual(create_recipes_message.event_id, new_batch.event_id)
self.assertEqual(create_recipes_message.recipe_type_name,
new_batch.recipe_type.name)
self.assertEqual(create_recipes_message.recipe_type_rev_num,
new_batch.recipe_type.revision_num)
# Execute next create_batch_recipes messages
result = batch_recipes_message.execute()
self.assertTrue(result)
# Should only have one last rcreate_recipes message
self.assertEqual(len(batch_recipes_message.new_messages), 1)
create_recipes_message = batch_recipes_message.new_messages[0]
self.assertTrue(batch_recipes_message.is_prev_batch_done)
self.assertEqual(create_recipes_message.type, 'create_recipes')
self.assertSetEqual(set(create_recipes_message.root_recipe_ids),
{recipe_ids[0]})
self.assertEqual(create_recipes_message.batch_id, new_batch.id)
self.assertEqual(create_recipes_message.event_id, new_batch.event_id)
self.assertEqual(create_recipes_message.recipe_type_name,
new_batch.recipe_type.name)
self.assertEqual(create_recipes_message.recipe_type_rev_num,
new_batch.recipe_type.revision_num)
# Test setting supersedes to false and make sure we don't have any reprocess messages
batch_definition_2 = BatchDefinition()
batch_definition_2.dataset = the_dataset.id
batch_definition_2.supersedes = False
forced_nodes = ForcedNodes()
forced_nodes.all_nodes = True
batch_definition_2.forced_nodes = forced_nodes
new_batch_2 = batch_test_utils.create_batch(
recipe_type=recipe_type, definition=batch_definition_2)
# Create message
message_2 = batch.messages.create_batch_recipes.CreateBatchRecipes()
message_2.batch_id = new_batch_2.id
# Execute message
result_2 = message_2.execute()
self.assertTrue(result_2)
self.assertEqual(len(message_2.new_messages), 6)
batch_recipes_message_2 = message_2.new_messages[0]
self.assertEqual(batch_recipes_message_2.type, 'create_batch_recipes')
self.assertEqual(batch_recipes_message_2.batch_id, new_batch_2.id)
self.assertFalse(batch_recipes_message_2.is_prev_batch_done)
# Make sure we've got 5 create-new-recipe messages
for msg in message_2.new_messages[1:]:
self.assertEqual(msg.create_recipes_type, 'new-recipe')
self.assertEqual(msg.batch_id, new_batch_2.id)
self.assertEqual(msg.event_id, new_batch_2.event_id)
self.assertEqual(msg.recipe_type_name,
new_batch_2.recipe_type.name)
self.assertEqual(msg.recipe_type_rev_num,
new_batch_2.recipe_type.revision_num)
# Execute next create_batch_recipes messages
result_3 = batch_recipes_message_2.execute()
self.assertTrue(result_3)
# Should only have one last rcreate_recipes message
self.assertEqual(len(batch_recipes_message_2.new_messages), 1)
create_recipes_message_3 = batch_recipes_message_2.new_messages[0]
self.assertTrue(batch_recipes_message_2.is_prev_batch_done)
self.assertEqual(create_recipes_message_3.type, 'create_recipes')
self.assertEqual(create_recipes_message_3.batch_id, new_batch_2.id)
self.assertEqual(create_recipes_message_3.event_id,
new_batch_2.event_id)
self.assertEqual(create_recipes_message_3.recipe_type_name,
new_batch_2.recipe_type.name)
self.assertEqual(create_recipes_message_3.recipe_type_rev_num,
new_batch_2.recipe_type.revision_num)