def test_json(self): """Tests converting an UpdateRecipe message to and from JSON""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_failed = job_test_utils.create_job(status='FAILED', input=data_dict) job_pending = job_test_utils.create_job(status='PENDING') definition = RecipeDefinition(Interface()) definition.add_job_node('job_failed', job_failed.job_type.name, job_failed.job_type.version, job_failed.job_type_rev.revision_num) definition.add_job_node('job_pending', job_pending.job_type.name, job_pending.job_type.version, job_pending.job_type_rev.revision_num) definition.add_dependency('job_failed', 'job_pending') definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_failed', job=job_failed) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_pending', job=job_pending) # Create message message = create_update_recipe_message(recipe.id) # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = UpdateRecipe.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) # Check for message to set job_pending to BLOCKED self.assertEqual(len(new_message.new_messages), 1) msg = new_message.new_messages[0] self.assertEqual(msg.type, 'blocked_jobs') self.assertListEqual(msg._blocked_job_ids, [job_pending.id])
def test_convert_recipe_definition_to_v6_json_full(self): """Tests calling convert_recipe_definition_to_v6_json() with a full definition""" interface = Interface() interface.add_parameter(FileParameter('file_param_a', ['image/gif'])) interface.add_parameter(JsonParameter('json_param_a', 'object')) interface.add_parameter( JsonParameter('json_param_b', 'object', required=False)) definition = RecipeDefinition(interface) definition.add_job_node('A', 'job_type_1', '1.0', 1) definition.add_job_node('B', 'job_type_2', '2.0', 1) definition.add_job_node('C', 'job_type_3', '1.0', 2) definition.add_recipe_node('D', 'recipe_type_1', 1) definition.add_job_node('E', 'job_type_4', '1.0', 1) definition.add_dependency('A', 'B') definition.add_dependency('A', 'C') definition.add_dependency('B', 'E') definition.add_dependency('C', 'D') definition.add_recipe_input_connection('A', 'input_1', 'file_param_a') definition.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition.add_recipe_input_connection('D', 'd_input_2', 'json_param_a') json = convert_recipe_definition_to_v6_json(definition) RecipeDefinitionV6(definition=json.get_dict(), do_validate=True) # Revalidate self.assertSetEqual(set(json.get_dict()['nodes'].keys()), {'A', 'B', 'C', 'D', 'E'})
def test_add_dependency_missing_child(self): """Tests calling RecipeDefinition.add_dependency() with a missing child node""" input_interface = Interface() definition = RecipeDefinition(input_interface) definition.add_job_node('node_1', 'job_type_1', '1.0', 1) with self.assertRaises(InvalidDefinition) as context: definition.add_dependency('node_1', 'missing_child') self.assertEqual(context.exception.error.name, 'UNKNOWN_NODE')
def get_definition(self): """Returns the recipe definition represented by this JSON :returns: The recipe definition :rtype: :class:`recipe.definition.definition.RecipeDefinition`: """ interface_json = InterfaceV6(self._definition['input'], do_validate=False) interface = interface_json.get_interface() definition = RecipeDefinition(interface) # Add all nodes to definition first for node_name, node_dict in self._definition['nodes'].items(): node_type_dict = node_dict['node_type'] if node_type_dict['node_type'] == 'condition': cond_interface_json = InterfaceV6(node_type_dict['interface'], do_validate=False) data_filter_json = DataFilterV6(node_type_dict['data_filter'], do_validate=False) definition.add_condition_node( node_name, cond_interface_json.get_interface(), data_filter_json.get_filter()) elif node_type_dict['node_type'] == 'job': definition.add_job_node(node_name, node_type_dict['job_type_name'], node_type_dict['job_type_version'], node_type_dict['job_type_revision']) elif node_type_dict['node_type'] == 'recipe': definition.add_recipe_node( node_name, node_type_dict['recipe_type_name'], node_type_dict['recipe_type_revision']) # Now add dependencies and connections for node_name, node_dict in self._definition['nodes'].items(): for dependency_dict in node_dict['dependencies']: acceptance = dependency_dict['acceptance'] if ( 'acceptance' in dependency_dict) else True definition.add_dependency(dependency_dict['name'], node_name, acceptance) for conn_name, conn_dict in node_dict['input'].items(): if conn_dict['type'] == 'recipe': definition.add_recipe_input_connection( node_name, conn_name, conn_dict['input']) elif conn_dict['type'] == 'dependency': definition.add_dependency_input_connection( node_name, conn_name, conn_dict['node'], conn_dict['output']) return definition
def test_topological_order_successful(self): """Tests calling RecipeDefinition.get_topological_order() successfully""" input_interface = Interface() definition = RecipeDefinition(input_interface) definition.add_job_node('A', 'job_type_1', '1.0', 1) definition.add_job_node('B', 'job_type_2', '1.0', 1) definition.add_recipe_node('C', 'recipe_type_1', 1) definition.add_recipe_node('D', 'recipe_type_2', 1) definition.add_job_node('E', 'job_type_3', '1.0', 1) definition.add_job_node('F', 'job_type_4', '1.0', 1) definition.add_dependency('A', 'B') definition.add_dependency('A', 'C') definition.add_dependency('A', 'E') definition.add_dependency('B', 'C') definition.add_dependency('B', 'D') definition.add_dependency('C', 'D') definition.add_dependency('D', 'E') definition.add_dependency('E', 'F') order = definition.get_topological_order() expected_order = [ 'A', 'B', 'C', 'D', 'E', 'F' ] # This is the only valid topological order for this graph self.assertListEqual(order, expected_order)
def test_generate_node_input_data(self): """Tests calling RecipeDefinition.generate_node_input_data()""" input_interface = Interface() input_interface.add_parameter( FileParameter('recipe_input_1', ['image/gif'], multiple=True)) input_interface.add_parameter(JsonParameter('recipe_input_2', 'string')) definition = RecipeDefinition(input_interface) definition.add_job_node('node_a', 'job_type_1', '1.0', 1) definition.add_job_node('node_b', 'job_type_2', '1.0', 1) definition.add_job_node('node_c', 'job_type_3', '1.0', 1) definition.add_dependency('node_c', 'node_b') definition.add_dependency('node_c', 'node_a') definition.add_recipe_input_connection('node_c', 'input_1', 'recipe_input_1') definition.add_recipe_input_connection('node_c', 'input_2', 'recipe_input_2') definition.add_dependency_input_connection('node_c', 'input_3', 'node_a', 'output_a_1') definition.add_dependency_input_connection('node_c', 'input_4', 'node_a', 'output_a_2') definition.add_dependency_input_connection('node_c', 'input_5', 'node_b', 'output_b_1') recipe_data = Data() recipe_data.add_value(FileValue('recipe_input_1', [1, 2, 3, 4, 5])) recipe_data.add_value(JsonValue('recipe_input_2', 'Scale is awesome!')) a_output_data = Data() a_output_data.add_value(FileValue('output_a_1', [1234])) a_output_data.add_value(JsonValue('output_a_2', {'foo': 'bar'})) b_output_data = Data() b_output_data.add_value(JsonValue('output_b_1', 12.34)) node_outputs = { 'node_a': RecipeNodeOutput('node_a', 'job', 1, a_output_data), 'node_b': RecipeNodeOutput('node_b', 'job', 1, b_output_data) } node_data = definition.generate_node_input_data( 'node_c', recipe_data, node_outputs) self.assertSetEqual( set(node_data.values.keys()), {'input_1', 'input_2', 'input_3', 'input_4', 'input_5'}) self.assertListEqual(node_data.values['input_1'].file_ids, [1, 2, 3, 4, 5]) self.assertEqual(node_data.values['input_2'].value, 'Scale is awesome!') self.assertListEqual(node_data.values['input_3'].file_ids, [1234]) self.assertDictEqual(node_data.values['input_4'].value, {'foo': 'bar'}) self.assertEqual(node_data.values['input_5'].value, 12.34)
def test_validate_successful(self): """Tests calling RecipeDefinition.validate() successfully""" input_interface = Interface() definition = RecipeDefinition(input_interface) definition.add_job_node('A', 'job_type_1', '1.0', 1) definition.add_recipe_node('B', 'recipe_type_1', 1) definition.add_dependency('A', 'B') definition.add_dependency_input_connection('B', 'input_1', 'A', 'output_1') mocked_interfaces = {'A': MagicMock(), 'B': MagicMock()} warnings = definition.validate(mocked_interfaces, mocked_interfaces) self.assertListEqual(warnings, [])
def test_get_nodes_to_create(self): """Tests calling Recipe.get_nodes_to_create()""" job_type = job_test_utils.create_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type() # Create recipe definition = RecipeDefinition(Interface()) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) definition.add_condition_node('B', Interface(), DataFilter(True)) definition.add_condition_node('C', Interface(), DataFilter(True)) definition.add_condition_node('D', Interface(), DataFilter(False)) definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_recipe_node('H', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_dependency('A', 'D') definition.add_dependency('A', 'E') definition.add_dependency('B', 'E') definition.add_dependency('B', 'F') definition.add_dependency('C', 'F') definition.add_dependency('D', 'G') definition.add_dependency('E', 'G') definition.add_dependency('E', 'H') definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) # Nodes A, B, and D already exist job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', save=True) condition_b = recipe_test_utils.create_recipe_condition( is_processed=True, is_accepted=True, save=False) condition_d = recipe_test_utils.create_recipe_condition( is_processed=True, is_accepted=False, save=False) RecipeCondition.objects.bulk_create([condition_b, condition_d]) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', condition=condition_b, save=False) recipe_node_d = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='D', condition=condition_d, save=False) RecipeNode.objects.bulk_create( [recipe_node_a, recipe_node_b, recipe_node_d]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) nodes_to_create = recipe_instance.get_nodes_to_create() self.assertSetEqual(set(nodes_to_create.keys()), {'C', 'E', 'H'})
def test_convert_recipe_diff_to_v6_json_with_changes(self): """Tests calling convert_recipe_diff_to_v6_json() with a diff containing a variety of changes""" interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) interface_2.add_parameter( JsonParameter('json_param_2', 'object', required=False)) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_job_node('E', 'job_type_4', '1.0', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('B', 'E') definition_1.add_dependency('C', 'D') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2 = RecipeDefinition(interface_2) # Nodes B and E are deleted definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('C', 'job_type_3', '2.1', 1) # Change to job type version and revision definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_recipe_node('F', 'recipe_type_2', 5) # New node definition_2.add_dependency('A', 'C') definition_2.add_dependency('C', 'D') definition_2.add_dependency('D', 'F') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_2.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2.add_recipe_input_connection('F', 'f_input_1', 'json_param_2') diff = RecipeDiff(definition_1, definition_2) json = convert_recipe_diff_to_v6_json(diff) RecipeDiffV6(diff=json.get_dict(), do_validate=True) # Revalidate self.assertTrue(json.get_dict()['can_be_reprocessed'])
def test_convert_recipe_diff_to_v6_json_new_required_input(self): """Tests calling convert_recipe_diff_to_v6_json() with a diff where there is a breaking recipe interface change """ interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) interface_2.add_parameter( JsonParameter('json_param_2', 'object', required=True)) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('C', 'D') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2 = RecipeDefinition(interface_2) definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('B', 'job_type_2', '2.0', 1) definition_2.add_job_node('C', 'job_type_3', '1.1', 1) # Change to job type version and revision definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_dependency('A', 'B') definition_2.add_dependency('A', 'C') definition_2.add_dependency('C', 'D') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_2.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_2.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') diff = RecipeDiff(definition_1, definition_2) json = convert_recipe_diff_to_v6_json(diff) RecipeDiffV6(diff=json.get_dict(), do_validate=True) # Revalidate self.assertFalse(json.get_dict()['can_be_reprocessed'])
def test_topological_order_circular(self): """Tests calling RecipeDefinition.get_topological_order() with a circular dependency""" input_interface = Interface() definition = RecipeDefinition(input_interface) definition.add_job_node('A', 'job_type_1', '1.0', 1) definition.add_job_node('B', 'job_type_2', '1.0', 1) definition.add_recipe_node('C', 'recipe_type_1', 1) definition.add_recipe_node('D', 'recipe_type_2', 1) definition.add_dependency('A', 'B') definition.add_dependency('B', 'C') definition.add_dependency('C', 'D') definition.add_dependency('D', 'B') with self.assertRaises(InvalidDefinition) as context: definition.get_topological_order() self.assertEqual(context.exception.error.name, 'CIRCULAR_DEPENDENCY')
def test_validate_invalid_connection(self): """Tests calling RecipeDefinition.validate() with an invalid connection to a node's input interface""" input_interface = Interface() definition = RecipeDefinition(input_interface) definition.add_job_node('A', 'job_type_1', '1.0', 1) definition.add_recipe_node('B', 'recipe_type_1', 1) definition.add_dependency('A', 'B') definition.add_dependency_input_connection('B', 'input_1', 'A', 'output_1') mocked_interfaces = {'A': MagicMock(), 'B': MagicMock()} mocked_interfaces[ 'B'].validate_connection.side_effect = InvalidInterfaceConnection( '', '') with self.assertRaises(InvalidDefinition) as context: definition.validate(mocked_interfaces, mocked_interfaces) self.assertEqual(context.exception.error.name, 'NODE_INTERFACE')
def test_validate_successful(self): """Tests calling RecipeDefinition.validate() successfully""" recipe_interface = Interface() recipe_interface.add_parameter( JsonParameter('recipe_input_1', 'integer')) definition = RecipeDefinition(recipe_interface) definition.add_job_node('A', 'job_type_1', '1.0', 1) condition_interface = Interface() condition_interface.add_parameter( JsonParameter('cond_param', 'integer')) definition.add_condition_node('B', condition_interface, DataFilter(True)) definition.add_recipe_node('C', 'recipe_type_1', 1) definition.add_dependency('A', 'B') definition.add_dependency('B', 'C') definition.add_recipe_input_connection('A', 'a_input_1', 'recipe_input_1') definition.add_dependency_input_connection('B', 'cond_param', 'A', 'a_output_1') definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_1') definition.add_dependency_input_connection('C', 'c_input_2', 'B', 'cond_param') job_input_interface = Interface() job_input_interface.add_parameter(JsonParameter( 'a_input_1', 'integer')) job_output_interface = Interface() job_output_interface.add_parameter( JsonParameter('a_output_1', 'integer')) recipe_input_interface = Interface() recipe_input_interface.add_parameter( JsonParameter('c_input_1', 'integer')) recipe_input_interface.add_parameter( JsonParameter('c_input_2', 'integer')) input_interfaces = { 'A': job_input_interface, 'C': recipe_input_interface } output_interfaces = {'A': job_output_interface, 'C': Interface()} warnings = definition.validate(input_interfaces, output_interfaces) self.assertListEqual(warnings, [])
def test_json_forced_nodes(self): """Tests converting an UpdateRecipe message to and from JSON when forced nodes are provided""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_completed = job_test_utils.create_job(status='COMPLETED', input=data_dict, output=data_dict) sub_recipe_type = recipe_test_utils.create_recipe_type_v6() definition = RecipeDefinition(Interface()) definition.add_job_node('job_completed', job_completed.job_type.name, job_completed.job_type.version, job_completed.job_type_rev.revision_num) definition.add_recipe_node('the_sub_recipe', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_dependency('job_completed', 'the_sub_recipe') definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_completed', job=job_completed) forced_nodes = ForcedNodes() sub_forced_nodes = ForcedNodes() sub_forced_nodes.set_all_nodes() forced_nodes.add_subrecipe('the_sub_recipe', sub_forced_nodes) # Create message message = create_update_recipe_message(recipe.id, forced_nodes=forced_nodes) # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = UpdateRecipe.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) # Check for message to create sub-recipe self.assertEqual(len(new_message.new_messages), 1) msg = new_message.new_messages[0] self.assertEqual(msg.type, 'create_recipes') self.assertEqual(msg.event_id, recipe.event_id) msg_forced_nodes_dict = convert_forced_nodes_to_v6(msg.forced_nodes).get_dict() expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict() self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict) self.assertEqual(msg.create_recipes_type, SUB_RECIPE_TYPE) self.assertEqual(msg.recipe_id, recipe.id) self.assertEqual(msg.root_recipe_id, recipe.root_superseded_recipe_id) self.assertIsNone(msg.superseded_recipe_id) sub = SubRecipe(sub_recipe_type.name, sub_recipe_type.revision_num, 'the_sub_recipe', True) self.assertListEqual(msg.sub_recipes, [sub])
def test_init_new_required_input(self): """Tests creating a RecipeDiff when the newer definition has a new required input parameter""" interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) interface_2.add_parameter(JsonParameter('json_param_2', 'object', required=True)) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('C', 'D') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2 = RecipeDefinition(interface_2) definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('B', 'job_type_2', '2.0', 1) definition_2.add_job_node('C', 'job_type_3', '1.1', 1) # Change to job type version and revision definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_dependency('A', 'B') definition_2.add_dependency('A', 'C') definition_2.add_dependency('C', 'D') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_2.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_2.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') diff = RecipeDiff(definition_1, definition_2) self.assertFalse(diff.can_be_reprocessed) self.assertEqual(len(diff.reasons), 1) self.assertEqual(diff.reasons[0].name, 'INPUT_CHANGE') # Cannot be reprocessed, so no nodes to copy, supersede, or unpublish self.assertDictEqual(diff.get_nodes_to_copy(), {}) self.assertDictEqual(diff.get_nodes_to_supersede(), {}) self.assertDictEqual(diff.get_nodes_to_unpublish(), {}) # Ensure no nodes have reprocess_new_node set to true for node_diff in diff.graph.values(): self.assertFalse(node_diff.reprocess_new_node)
def test_validate_missing_dependency(self): """Tests calling RecipeDefinition.validate() with a connection that has a missing dependency""" input_interface = Interface() input_interface.parameters = {'recipe_input_1': MagicMock()} definition = RecipeDefinition(input_interface) definition.add_job_node('A', 'job_type_1', '1.0', 1) definition.add_recipe_node('B', 'recipe_type_1', 1) definition.add_job_node('C', 'job_type_2', '1.0', 1) definition.add_dependency('B', 'C') definition.add_dependency_input_connection('B', 'input_1', 'A', 'output_1') mocked_interfaces = { 'A': MagicMock(), 'B': MagicMock(), 'C': MagicMock() } with self.assertRaises(InvalidDefinition) as context: definition.validate(mocked_interfaces, mocked_interfaces) self.assertEqual(context.exception.error.name, 'NODE_INTERFACE')
def test_convert_recipe_to_v6_json(self): """Tests calling convert_recipe_to_v6_json() successfully""" job_type_1 = job_test_utils.create_seed_job_type() job_type_2 = job_test_utils.create_seed_job_type() job_type_3 = job_test_utils.create_seed_job_type() job_type_4 = job_test_utils.create_seed_job_type() recipe_type_1 = recipe_test_utils.create_recipe_type_v6() interface = Interface() interface.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface.add_parameter(JsonParameter('json_param_1', 'object')) df1 = DataFilter(filter_list=[{'name': 'file_param_1', 'type': 'media-type', 'condition': '==', 'values': ['image/gif']}, {'name': 'json_param_1', 'type': 'object', 'condition': 'superset of', 'values': [{}]}], all=False) definition = RecipeDefinition(interface) definition.add_job_node('A', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('B', job_type_2.name, job_type_2.version, job_type_2.revision_num) definition.add_job_node('C', job_type_3.name, job_type_3.version, job_type_3.revision_num) definition.add_recipe_node('D', recipe_type_1.name, recipe_type_1.revision_num) definition.add_job_node('E', job_type_4.name, job_type_4.version, job_type_4.revision_num) definition.add_condition_node('F', interface, df1) #False definition.add_job_node('G', job_type_4.name, job_type_4.version, job_type_4.revision_num) definition.add_dependency('A', 'B') definition.add_dependency('A', 'C') definition.add_dependency('B', 'E') definition.add_dependency('C', 'D') definition.add_dependency('A', 'F') definition.add_dependency('F', 'G') definition.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') recipe = recipe_test_utils.create_recipe() job_a = job_test_utils.create_job(job_type=job_type_1, status='COMPLETED', save=False) job_b = job_test_utils.create_job(job_type=job_type_2, status='RUNNING', save=False) job_c = job_test_utils.create_job(job_type=job_type_3, status='COMPLETED', save=False) job_e = job_test_utils.create_job(job_type=job_type_4, status='PENDING', num_exes=0, save=False) Job.objects.bulk_create([job_a, job_b, job_c, job_e]) condition_f = recipe_test_utils.create_recipe_condition(is_processed=True, is_accepted=False, save=True) recipe_d = recipe_test_utils.create_recipe(recipe_type=recipe_type_1) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False) recipe_node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='B', job=job_b, save=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False) recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', sub_recipe=recipe_d, save=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', condition=condition_f, save=False) recipe_nodes = [recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f] recipe_instance = RecipeInstance(definition, recipe, recipe_nodes) json = convert_recipe_to_v6_json(recipe_instance) RecipeInstanceV6(json=json.get_dict(), do_validate=True) # Revalidate self.assertSetEqual(set(json.get_dict()['nodes'].keys()), {'A', 'B', 'C', 'D', 'E', 'F'})
def test_has_completed_empty(self): """Tests calling Recipe.has_completed() when a recipe is empty and has not created its nodes yet""" job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() definition = RecipeDefinition(Interface()) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('B', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('C', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('D', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('H', job_type.name, job_type.version, job_type.revision_num) definition.add_dependency('A', 'C') definition.add_dependency('A', 'E') definition.add_dependency('A', 'H') definition.add_dependency('C', 'D') definition.add_dependency('G', 'H') definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) self.assertFalse(recipe_instance.has_completed())
def test_get_original_leaf_nodes(self): """Tests calling Recipe.get_original_leaf_nodes()""" job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() definition = RecipeDefinition(Interface()) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('B', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('C', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('D', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('H', job_type.name, job_type.version, job_type.revision_num) definition.add_dependency('A', 'C') definition.add_dependency('A', 'E') definition.add_dependency('A', 'H') definition.add_dependency('C', 'D') definition.add_dependency('G', 'H') job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', save=False, is_superseded=True) job_c = job_test_utils.create_job(job_type=job_type, status='CANCELED', num_exes=0, save=False) job_d = job_test_utils.create_job(job_type=job_type, status='PENDING', num_exes=0, save=False) job_e = job_test_utils.create_job(job_type=job_type, status='BLOCKED', num_exes=0, save=False) job_f = job_test_utils.create_job(job_type=job_type, status='PENDING', num_exes=0, save=False) job_h = job_test_utils.create_job(job_type=job_type, status='PENDING', num_exes=0, save=False) Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h]) recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_b.jobs_completed = 3 recipe_b.jobs_running = 2 recipe_b.jobs_total = 5 recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_g.jobs_completed = 2 recipe_g.jobs_failed = 1 recipe_g.jobs_total = 3 Recipe.objects.bulk_create([recipe_b, recipe_g]) definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False, is_original=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False, is_original=False) recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', job=job_d, save=False, is_original=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', job=job_f, save=False) recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='H', job=job_h, save=False) recipe_node_g = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='G', sub_recipe=recipe_g, save=False, is_original=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False) RecipeNode.objects.bulk_create([ recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h ]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) results = recipe_instance.get_original_leaf_nodes() self.assertEqual(len(results.values()), 4) leaf_jobs = [ node.job.id for node in results.values() if node.node_type == JobNodeDefinition.NODE_TYPE ] leaf_recipes = [ node.recipe.id for node in results.values() if node.node_type == RecipeNodeDefinition.NODE_TYPE ] self.assertItemsEqual( leaf_jobs, [recipe_node_e.job.id, recipe_node_f.job.id, recipe_node_h.job.id]) self.assertItemsEqual(leaf_recipes, [recipe_node_b.sub_recipe.id])
def test_get_nodes_to_process_input(self): """Tests calling Recipe.get_nodes_to_process_input()""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() # Create recipe definition = RecipeDefinition(Interface()) cond_interface_1 = Interface() cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer')) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) df1 = DataFilter(filter_list=[{ 'name': 'cond_int', 'type': 'integer', 'condition': '==', 'values': [0] }, { 'name': 'cond_int', 'type': 'integer', 'condition': '!=', 'values': [0] }], all=False) #always True df2 = DataFilter(filter_list=[{ 'name': 'cond_int', 'type': 'integer', 'condition': '==', 'values': [0] }, { 'name': 'cond_int', 'type': 'integer', 'condition': '!=', 'values': [0] }], all=True) #always False definition.add_condition_node('B', cond_interface_1, df1) #True definition.add_condition_node('C', cond_interface_1, df1) #True definition.add_condition_node('D', cond_interface_1, df2) #False definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_recipe_node('H', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_dependency('A', 'D') definition.add_dependency('A', 'E') definition.add_dependency('B', 'E') definition.add_dependency('B', 'F') definition.add_dependency('C', 'F') definition.add_dependency('D', 'G') definition.add_dependency('E', 'G') definition.add_dependency('E', 'H') definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict) # Nodes A, B, and D already exist job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', input=data_dict, output=data_dict, save=True) condition_b = recipe_test_utils.create_recipe_condition( is_processed=True, is_accepted=True, save=False) condition_d = recipe_test_utils.create_recipe_condition( is_processed=True, is_accepted=False, save=False) RecipeCondition.objects.bulk_create([condition_b, condition_d]) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', condition=condition_b, save=False) recipe_node_d = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='D', condition=condition_d, save=False) RecipeNode.objects.bulk_create( [recipe_node_a, recipe_node_b, recipe_node_d]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) nodes_to_process = recipe_instance.get_nodes_to_process_input() self.assertSetEqual(set(nodes_to_process.keys()), {'C', 'E'})
def test_get_jobs_to_update(self): """Tests calling Recipe.get_jobs_to_update()""" job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() definition = RecipeDefinition(Interface()) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('B', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('C', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('D', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('H', job_type.name, job_type.version, job_type.revision_num) definition.add_dependency('A', 'C') definition.add_dependency('A', 'E') definition.add_dependency('A', 'H') definition.add_dependency('B', 'E') definition.add_dependency('B', 'G') definition.add_dependency('C', 'D') definition.add_dependency('E', 'F') definition.add_dependency('G', 'H') job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', save=False) job_c = job_test_utils.create_job(job_type=job_type, status='CANCELED', num_exes=0, save=False) job_d = job_test_utils.create_job(job_type=job_type, status='PENDING', num_exes=0, save=False) job_e = job_test_utils.create_job(job_type=job_type, status='BLOCKED', num_exes=0, save=False) job_f = job_test_utils.create_job(job_type=job_type, status='PENDING', num_exes=0, save=False) job_h = job_test_utils.create_job(job_type=job_type, status='PENDING', num_exes=0, save=False) Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h]) recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_b.jobs_completed = 3 recipe_b.jobs_running = 2 recipe_b.jobs_total = 5 recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_g.jobs_completed = 2 recipe_g.jobs_failed = 1 recipe_g.jobs_total = 3 Recipe.objects.bulk_create([recipe_b, recipe_g]) definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False) recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', job=job_d, save=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', job=job_f, save=False) recipe_node_g = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='G', sub_recipe=recipe_g, save=False) recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='H', job=job_h, save=False) RecipeNode.objects.bulk_create([ recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h ]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) results = recipe_instance.get_jobs_to_update() self.assertSetEqual(set(results['BLOCKED']), {job_d.id, job_h.id}) self.assertSetEqual(set(results['PENDING']), {job_e.id})
def test_execute_with_recipe_legacy(self): """Tests calling ProcessRecipeInput.execute() successfully when a legacy sub-recipe has to get its data from its recipe """ workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0) file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0) manifest_a = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-a', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': { 'files': [], 'json': [] }, 'outputs': { 'files': [{ 'name': 'output_a', 'pattern': '*.png' }] } } } } job_type_a = job_test_utils.create_job_type(interface=manifest_a) output_data_a = Data() output_data_a.add_value(FileValue('output_a', [file_1.id])) output_data_a_dict = convert_data_to_v6_json(output_data_a).get_dict() manifest_b = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-b', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': { 'files': [], 'json': [] }, 'outputs': { 'files': [{ 'name': 'output_b', 'pattern': '*.png', 'multiple': True }] } } } } job_type_b = job_test_utils.create_job_type(interface=manifest_b) output_data_b = Data() output_data_b.add_value( FileValue('output_b', [file_2.id, file_3.id, file_4.id])) output_data_b_dict = convert_data_to_v6_json(output_data_b).get_dict() job_a = job_test_utils.create_job(job_type=job_type_a, num_exes=1, status='COMPLETED', output=output_data_a_dict) job_b = job_test_utils.create_job(job_type=job_type_b, num_exes=1, status='COMPLETED', output=output_data_b_dict) sub_recipe_interface_c = Interface() sub_recipe_interface_c.add_parameter( FileParameter('input_a', ['image/png'])) sub_recipe_interface_c.add_parameter( FileParameter('input_b', ['image/png'], multiple=True)) sub_recipe_def_c = RecipeDefinition(sub_recipe_interface_c) sub_recipe_def_dict_c = convert_recipe_definition_to_v1_json( sub_recipe_def_c).get_dict() sub_recipe_type_c = recipe_test_utils.create_recipe_type( definition=sub_recipe_def_dict_c) sub_recipe_c = recipe_test_utils.create_recipe( recipe_type=sub_recipe_type_c) definition = RecipeDefinition(Interface()) definition.add_job_node('node_a', job_type_a.name, job_type_a.version, job_type_a.revision_num) definition.add_job_node('node_b', job_type_b.name, job_type_b.version, job_type_b.revision_num) definition.add_recipe_node('node_c', sub_recipe_type_c.name, sub_recipe_type_c.revision_num) definition.add_dependency('node_c', 'node_a') definition.add_dependency_input_connection('node_c', 'input_a', 'node_a', 'output_a') definition.add_dependency('node_c', 'node_b') definition.add_dependency_input_connection('node_c', 'input_b', 'node_b', 'output_b') def_dict = convert_recipe_definition_to_v6_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type(definition=def_dict) recipe_data_dict = { 'version': '1.0', 'input_data': [], 'workspace_id': workspace.id } recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict) recipe_node_a = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='node_a', job=job_a) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='node_b', job=job_b) recipe_node_c = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='node_c', sub_recipe=sub_recipe_c) RecipeNode.objects.bulk_create( [recipe_node_a, recipe_node_b, recipe_node_c]) job_a.recipe = recipe job_a.save() job_b.recipe = recipe job_b.save() sub_recipe_c.recipe = recipe sub_recipe_c.save() # Create message message = ProcessRecipeInput() message.recipe_id = sub_recipe_c.id # Execute message result = message.execute() self.assertTrue(result) sub_recipe_c = Recipe.objects.get(id=sub_recipe_c.id) # Check for update_recipes message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'update_recipes') # Check sub-recipe for expected input_file_size self.assertEqual(sub_recipe_c.input_file_size, 24469.0) # Check sub-recipe for expected input data self.assertEqual( sub_recipe_c.input['version'], '1.0') # Should be legacy input data with workspace ID self.assertEqual(sub_recipe_c.input['workspace_id'], workspace.id) self.assertSetEqual(set(sub_recipe_c.get_input_data().values.keys()), {'input_a', 'input_b'}) self.assertListEqual( sub_recipe_c.get_input_data().values['input_a'].file_ids, [file_1.id]) self.assertListEqual( sub_recipe_c.get_input_data().values['input_b'].file_ids, [file_2.id, file_3.id, file_4.id]) # Make sure sub-recipe input file models are created input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id) self.assertEqual(len(input_files), 4) file_ids = {input_file.input_file_id for input_file in input_files} self.assertSetEqual(file_ids, {file_1.id, file_2.id, file_3.id, file_4.id}) # Test executing message again message_json_dict = message.to_json() message = ProcessRecipeInput.from_json(message_json_dict) result = message.execute() self.assertTrue(result) # Still should have update_recipes message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'update_recipes') # Make sure recipe input file models are unchanged input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id) self.assertEqual(len(input_files), 4)
def test_convert_recipe_to_v6_json(self): """Tests calling convert_recipe_to_v6_json() successfully""" job_type_1 = job_test_utils.create_job_type() job_type_2 = job_test_utils.create_job_type() job_type_3 = job_test_utils.create_job_type() job_type_4 = job_test_utils.create_job_type() recipe_type_1 = recipe_test_utils.create_recipe_type() interface = Interface() interface.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface.add_parameter(JsonParameter('json_param_1', 'object')) definition = RecipeDefinition(interface) definition.add_job_node('A', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('B', job_type_2.name, job_type_2.version, job_type_2.revision_num) definition.add_job_node('C', job_type_3.name, job_type_3.version, job_type_3.revision_num) definition.add_recipe_node('D', recipe_type_1.name, recipe_type_1.revision_num) definition.add_job_node('E', job_type_4.name, job_type_4.version, job_type_4.revision_num) definition.add_dependency('A', 'B') definition.add_dependency('A', 'C') definition.add_dependency('B', 'E') definition.add_dependency('C', 'D') definition.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') recipe = recipe_test_utils.create_recipe() job_a = job_test_utils.create_job(job_type=job_type_1, status='COMPLETED', save=False) job_b = job_test_utils.create_job(job_type=job_type_2, status='RUNNING', save=False) job_c = job_test_utils.create_job(job_type=job_type_3, status='COMPLETED', save=False) job_e = job_test_utils.create_job(job_type=job_type_4, status='PENDING', num_exes=0, save=False) Job.objects.bulk_create([job_a, job_b, job_c, job_e]) recipe_d = recipe_test_utils.create_recipe(recipe_type=recipe_type_1) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False) recipe_node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='B', job=job_b, save=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False) recipe_node_d = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='D', sub_recipe=recipe_d, save=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_nodes = [ recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e ] recipe_instance = RecipeInstance(definition, recipe_nodes) json = convert_recipe_to_v6_json(recipe_instance) RecipeInstanceV6(json=json.get_dict(), do_validate=True) # Revalidate self.assertSetEqual(set(json.get_dict()['nodes'].keys()), {'A', 'B', 'C', 'D', 'E'})
def test_set_force_reprocess(self): """Tests calling RecipeDiff.set_force_reprocess()""" interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_job_node('E', 'job_type_4', '1.0', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('C', 'D') definition_1.add_dependency('C', 'E') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_1.add_dependency_input_connection('E', 'e_input_1', 'C', 'c_output_1') # No changes in definition 2 definition_2 = RecipeDefinition(interface_2) definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('B', 'job_type_2', '2.0', 1) definition_2.add_job_node('C', 'job_type_3', '1.0', 2) definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_job_node('E', 'job_type_4', '1.0', 1) definition_2.add_dependency('A', 'B') definition_2.add_dependency('A', 'C') definition_2.add_dependency('C', 'D') definition_2.add_dependency('C', 'E') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_2.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_2.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2.add_dependency_input_connection('E', 'e_input_1', 'C', 'c_output_1') recipe_d_forced_nodes = ForcedNodes() recipe_d_forced_nodes.add_node('1') recipe_d_forced_nodes.add_node('2') top_forced_nodes = ForcedNodes() top_forced_nodes.add_node('C') top_forced_nodes.add_subrecipe('D', recipe_d_forced_nodes) diff = RecipeDiff(definition_1, definition_2) diff.set_force_reprocess(top_forced_nodes) # No recipe input changes so recipe can be reprocessed self.assertTrue(diff.can_be_reprocessed) self.assertListEqual(diff.reasons, []) # Check each node for correct fields node_a = diff.graph['A'] self.assertEqual(node_a.status, NodeDiff.UNCHANGED) self.assertFalse(node_a.reprocess_new_node) self.assertListEqual(node_a.changes, []) node_b = diff.graph['B'] self.assertEqual(node_b.status, NodeDiff.UNCHANGED) self.assertFalse(node_b.reprocess_new_node) self.assertListEqual(node_b.changes, []) node_c = diff.graph['C'] self.assertEqual(node_c.status, NodeDiff.UNCHANGED) self.assertTrue(node_c.reprocess_new_node) # Force reprocess self.assertListEqual(node_c.changes, []) node_d = diff.graph['D'] self.assertEqual(node_d.status, NodeDiff.UNCHANGED) self.assertTrue(node_d.reprocess_new_node) # Force reprocess self.assertListEqual(node_d.changes, []) # Check forced nodes object that got passed to recipe node D self.assertEqual(node_d.force_reprocess_nodes, recipe_d_forced_nodes) node_e = diff.graph['E'] self.assertEqual(node_e.status, NodeDiff.UNCHANGED) self.assertTrue( node_e.reprocess_new_node) # Force reprocess due to C being forced self.assertListEqual(node_e.changes, []) # Check nodes to copy, supersede, and unpublish self.assertSetEqual(set(diff.get_nodes_to_copy().keys()), {'A', 'B'}) self.assertSetEqual(set(diff.get_nodes_to_supersede().keys()), {'C', 'D', 'E'}) self.assertSetEqual(set(diff.get_nodes_to_unpublish().keys()), set())
def test_init_changes_in_middle_of_chains(self): """Tests creating a RecipeDiff where nodes are deleted from and inserted into the middle of a chain""" interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('C', 'D') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2 = RecipeDefinition(interface_2) definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('B', 'job_type_2', '2.0', 1) # Node C is deleted definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_job_node('E', 'job_type_4', '5.0', 2) # New node inbetween A and B definition_2.add_dependency('A', 'E') definition_2.add_dependency('E', 'B') definition_2.add_dependency('A', 'D') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('E', 'e_input_1', 'A', 'a_output_1') definition_2.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_2.add_dependency_input_connection('D', 'd_input_1', 'A', 'a_output_2') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') diff = RecipeDiff(definition_1, definition_2) # No recipe input changes so recipe can be reprocessed self.assertTrue(diff.can_be_reprocessed) self.assertListEqual(diff.reasons, []) # Check each node for correct fields node_a = diff.graph['A'] self.assertEqual(node_a.status, NodeDiff.UNCHANGED) self.assertFalse(node_a.reprocess_new_node) self.assertListEqual(node_a.changes, []) node_b = diff.graph['B'] self.assertEqual(node_b.status, NodeDiff.CHANGED) self.assertTrue(node_b.reprocess_new_node) self.assertEqual(len(node_b.changes), 2) self.assertEqual(node_b.changes[0].name, 'PARENT_NEW') self.assertEqual(node_b.changes[1].name, 'PARENT_REMOVED') node_c = diff.graph['C'] self.assertEqual(node_c.status, NodeDiff.DELETED) self.assertFalse(node_c.reprocess_new_node) self.assertListEqual(node_c.changes, []) node_d = diff.graph['D'] self.assertEqual(node_d.status, NodeDiff.CHANGED) self.assertTrue(node_d.reprocess_new_node) self.assertEqual(len(node_d.changes), 3) self.assertEqual(node_d.changes[0].name, 'PARENT_NEW') self.assertEqual(node_d.changes[1].name, 'PARENT_REMOVED') self.assertEqual(node_d.changes[2].name, 'INPUT_CHANGE') node_e = diff.graph['E'] self.assertEqual(node_e.status, NodeDiff.NEW) self.assertTrue(node_e.reprocess_new_node) self.assertListEqual(node_e.changes, []) # Check nodes to copy, supersede, and unpublish self.assertSetEqual(set(diff.get_nodes_to_copy().keys()), {'A'}) self.assertSetEqual(set(diff.get_nodes_to_supersede().keys()), {'B', 'C', 'D'}) self.assertSetEqual(set(diff.get_nodes_to_unpublish().keys()), {'C'})
def test_init_identical(self): """Tests creating a RecipeDiff between two identical recipe definitions""" interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('C', 'D') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2 = RecipeDefinition(interface_2) definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('B', 'job_type_2', '2.0', 1) definition_2.add_job_node('C', 'job_type_3', '1.0', 2) definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_dependency('A', 'B') definition_2.add_dependency('A', 'C') definition_2.add_dependency('C', 'D') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_2.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_2.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') diff = RecipeDiff(definition_1, definition_2) self.assertTrue(diff.can_be_reprocessed) self.assertListEqual(diff.reasons, []) # Every node should be unchanged and all should be copied during a reprocess nodes_to_copy = diff.get_nodes_to_copy() self.assertSetEqual(set(nodes_to_copy.keys()), {'A', 'B', 'C', 'D'}) for node_diff in nodes_to_copy.values(): self.assertEqual(node_diff.status, NodeDiff.UNCHANGED) self.assertFalse(node_diff.reprocess_new_node) self.assertListEqual(node_diff.changes, []) self.assertDictEqual(diff.get_nodes_to_supersede(), {}) self.assertDictEqual(diff.get_nodes_to_unpublish(), {})
def test_init_changes(self): """Tests creating a RecipeDiff when the newer definition has a variety of changes in it""" interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) interface_2.add_parameter( JsonParameter('json_param_2', 'object', required=False)) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_job_node('E', 'job_type_4', '1.0', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('B', 'E') definition_1.add_dependency('C', 'D') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2 = RecipeDefinition(interface_2) # Nodes B and E are deleted definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('C', 'job_type_3', '2.1', 1) # Change to job type version and revision definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_recipe_node('F', 'recipe_type_2', 5) # New node definition_2.add_dependency('A', 'C') definition_2.add_dependency('C', 'D') definition_2.add_dependency('D', 'F') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_2.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2.add_recipe_input_connection('F', 'f_input_1', 'json_param_2') diff = RecipeDiff(definition_1, definition_2) # Non-breaking recipe input changes so recipe can be reprocessed self.assertTrue(diff.can_be_reprocessed) self.assertListEqual(diff.reasons, []) # Check each node for correct fields node_a = diff.graph['A'] self.assertEqual(node_a.status, NodeDiff.UNCHANGED) self.assertFalse(node_a.reprocess_new_node) self.assertListEqual(node_a.changes, []) node_b = diff.graph['B'] self.assertEqual(node_b.status, NodeDiff.DELETED) self.assertFalse(node_b.reprocess_new_node) self.assertListEqual(node_b.changes, []) node_c = diff.graph['C'] self.assertEqual(node_c.status, NodeDiff.CHANGED) self.assertTrue(node_c.reprocess_new_node) self.assertEqual(len(node_c.changes), 2) self.assertEqual(node_c.changes[0].name, 'JOB_TYPE_VERSION_CHANGE') self.assertEqual(node_c.changes[1].name, 'JOB_TYPE_REVISION_CHANGE') node_d = diff.graph['D'] self.assertEqual(node_d.status, NodeDiff.CHANGED) self.assertTrue(node_d.reprocess_new_node) self.assertEqual(len(node_d.changes), 1) self.assertEqual(node_d.changes[0].name, 'PARENT_CHANGED') node_e = diff.graph['E'] self.assertEqual(node_e.status, NodeDiff.DELETED) self.assertFalse(node_e.reprocess_new_node) self.assertListEqual(node_e.changes, []) node_f = diff.graph['F'] self.assertEqual(node_f.status, NodeDiff.NEW) self.assertTrue(node_f.reprocess_new_node) self.assertListEqual(node_f.changes, []) # Check nodes to copy, supersede, and unpublish self.assertSetEqual(set(diff.get_nodes_to_copy().keys()), {'A'}) self.assertSetEqual(set(diff.get_nodes_to_supersede().keys()), {'B', 'C', 'D', 'E'}) self.assertSetEqual(set(diff.get_nodes_to_unpublish().keys()), {'B', 'E'})
def test_has_completed_false(self): """Tests calling Recipe.has_completed() when an entire recipe has not completed""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() definition = RecipeDefinition(Interface()) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('B', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('C', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('D', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('H', job_type.name, job_type.version, job_type.revision_num) definition.add_dependency('A', 'C') definition.add_dependency('A', 'E') definition.add_dependency('A', 'H') definition.add_dependency('C', 'D') definition.add_dependency('G', 'H') job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_c = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_d = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_e = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_f = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_h = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h]) recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_b.is_completed = True recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_g.is_completed = False Recipe.objects.bulk_create([recipe_b, recipe_g]) definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False, is_original=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False, is_original=False) recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', job=job_d, save=False, is_original=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', job=job_f, save=False) recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='H', job=job_h, save=False) recipe_node_g = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='G', sub_recipe=recipe_g, save=False, is_original=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False) RecipeNode.objects.bulk_create([ recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h ]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) self.assertFalse(recipe_instance.has_completed())
def test_has_completed_true(self): """Tests calling Recipe.has_completed() when an entire recipe has completed""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() cond_interface_1 = Interface() cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer')) df2 = DataFilter(filter_list=[{ 'name': 'cond_int', 'type': 'integer', 'condition': '==', 'values': [0] }, { 'name': 'cond_int', 'type': 'integer', 'condition': '!=', 'values': [0] }], all=True) #always False definition = RecipeDefinition(Interface()) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('B', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('C', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('D', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('H', job_type.name, job_type.version, job_type.revision_num) definition.add_condition_node('I', cond_interface_1, df2) #False definition.add_job_node('J', job_type.name, job_type.version, job_type.revision_num) definition.add_dependency('A', 'C') definition.add_dependency('A', 'E') definition.add_dependency('A', 'H') definition.add_dependency('C', 'D') definition.add_dependency('G', 'H') definition.add_dependency('A', 'I') definition.add_dependency('I', 'J') job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_c = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_d = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_e = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_f = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_h = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h]) condition_i = recipe_test_utils.create_recipe_condition( is_processed=True, is_accepted=False, save=True) recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_b.is_completed = True recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_g.is_completed = True Recipe.objects.bulk_create([recipe_b, recipe_g]) definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False, is_original=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False, is_original=False) recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', job=job_d, save=False, is_original=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', job=job_f, save=False) recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='H', job=job_h, save=False) recipe_node_i = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='I', condition=condition_i, save=False) recipe_node_g = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='G', sub_recipe=recipe_g, save=False, is_original=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False) RecipeNode.objects.bulk_create([ recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h, recipe_node_i ]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) self.assertTrue(recipe_instance.has_completed())
def test_execute_with_recipe_missing_output(self): """Tests calling ProcessJobInput.execute() when a job has to get its data from its recipe but the previous job didn't provide it's output""" workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0) file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0) manifest_1 = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-a', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': {'files': [], 'json': []}, 'outputs': { 'files': [{'name': 'OUTPUT_A', 'pattern': '*.png'}] } } } } job_type_1 = job_test_utils.create_seed_job_type(manifest=manifest_1) manifest_2 = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-b', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': {'files': [{'name': 'INPUT_B', 'multiple': True}]}, 'outputs': { 'files': [{'name': 'OUTPUT_B', 'pattern': '*.png'}] } } } } job_type_2 = job_test_utils.create_seed_job_type(manifest=manifest_2) output_dict = { 'version': '1.0', 'output_data': [{ }] } job_1 = job_test_utils.create_job(job_type=job_type_1, num_exes=1, status='COMPLETED', output=output_dict) job_2 = job_test_utils.create_job(job_type=job_type_2, num_exes=0, status='PENDING', input_file_size=None, input=None) from recipe.definition.definition import RecipeDefinition from recipe.definition.json.definition_v1 import convert_recipe_definition_to_v1_json from recipe.test import utils as recipe_test_utils definition = RecipeDefinition(Interface()) definition.add_job_node('node_a', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('node_b', job_type_2.name, job_type_2.version, job_type_2.revision_num) definition.add_dependency('node_a', 'node_b') definition.add_dependency_input_connection('node_b', 'INPUT_B', 'node_a', 'OUTPUT_A') def_dict = convert_recipe_definition_to_v1_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6(definition=def_dict) recipe_data_dict = {'version': '1.0', 'input_data': [], 'workspace_id': workspace.id} recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_a', job=job_1) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_b', job=job_2) job_1.recipe = recipe job_1.save() job_2.recipe = recipe job_2.save() # Create message message = ProcessJobInput() message.job_id = job_2.id # Execute message result = message.execute() self.assertTrue(result) job_2 = Job.objects.get(id=job_2.id) # Check for queued jobs message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'cancel_jobs')