def test_convert_recipe_definition_to_v6_json_full(self): """Tests calling convert_recipe_definition_to_v6_json() with a full definition""" interface = Interface() interface.add_parameter(FileParameter('file_param_a', ['image/gif'])) interface.add_parameter(JsonParameter('json_param_a', 'object')) interface.add_parameter( JsonParameter('json_param_b', 'object', required=False)) definition = RecipeDefinition(interface) definition.add_job_node('A', 'job_type_1', '1.0', 1) definition.add_job_node('B', 'job_type_2', '2.0', 1) definition.add_job_node('C', 'job_type_3', '1.0', 2) definition.add_recipe_node('D', 'recipe_type_1', 1) definition.add_job_node('E', 'job_type_4', '1.0', 1) definition.add_dependency('A', 'B') definition.add_dependency('A', 'C') definition.add_dependency('B', 'E') definition.add_dependency('C', 'D') definition.add_recipe_input_connection('A', 'input_1', 'file_param_a') definition.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition.add_recipe_input_connection('D', 'd_input_2', 'json_param_a') json = convert_recipe_definition_to_v6_json(definition) RecipeDefinitionV6(definition=json.get_dict(), do_validate=True) # Revalidate self.assertSetEqual(set(json.get_dict()['nodes'].keys()), {'A', 'B', 'C', 'D', 'E'})
def test_convert_recipe_diff_to_v6_json_with_changes(self): """Tests calling convert_recipe_diff_to_v6_json() with a diff containing a variety of changes""" interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) interface_2.add_parameter( JsonParameter('json_param_2', 'object', required=False)) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_job_node('E', 'job_type_4', '1.0', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('B', 'E') definition_1.add_dependency('C', 'D') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2 = RecipeDefinition(interface_2) # Nodes B and E are deleted definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('C', 'job_type_3', '2.1', 1) # Change to job type version and revision definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_recipe_node('F', 'recipe_type_2', 5) # New node definition_2.add_dependency('A', 'C') definition_2.add_dependency('C', 'D') definition_2.add_dependency('D', 'F') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_2.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2.add_recipe_input_connection('F', 'f_input_1', 'json_param_2') diff = RecipeDiff(definition_1, definition_2) json = convert_recipe_diff_to_v6_json(diff) RecipeDiffV6(diff=json.get_dict(), do_validate=True) # Revalidate self.assertTrue(json.get_dict()['can_be_reprocessed'])
def test_convert_recipe_to_v6_json(self): """Tests calling convert_recipe_to_v6_json() successfully""" job_type_1 = job_test_utils.create_seed_job_type() job_type_2 = job_test_utils.create_seed_job_type() job_type_3 = job_test_utils.create_seed_job_type() job_type_4 = job_test_utils.create_seed_job_type() recipe_type_1 = recipe_test_utils.create_recipe_type_v6() interface = Interface() interface.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface.add_parameter(JsonParameter('json_param_1', 'object')) df1 = DataFilter(filter_list=[{'name': 'file_param_1', 'type': 'media-type', 'condition': '==', 'values': ['image/gif']}, {'name': 'json_param_1', 'type': 'object', 'condition': 'superset of', 'values': [{}]}], all=False) definition = RecipeDefinition(interface) definition.add_job_node('A', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('B', job_type_2.name, job_type_2.version, job_type_2.revision_num) definition.add_job_node('C', job_type_3.name, job_type_3.version, job_type_3.revision_num) definition.add_recipe_node('D', recipe_type_1.name, recipe_type_1.revision_num) definition.add_job_node('E', job_type_4.name, job_type_4.version, job_type_4.revision_num) definition.add_condition_node('F', interface, df1) #False definition.add_job_node('G', job_type_4.name, job_type_4.version, job_type_4.revision_num) definition.add_dependency('A', 'B') definition.add_dependency('A', 'C') definition.add_dependency('B', 'E') definition.add_dependency('C', 'D') definition.add_dependency('A', 'F') definition.add_dependency('F', 'G') definition.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') recipe = recipe_test_utils.create_recipe() job_a = job_test_utils.create_job(job_type=job_type_1, status='COMPLETED', save=False) job_b = job_test_utils.create_job(job_type=job_type_2, status='RUNNING', save=False) job_c = job_test_utils.create_job(job_type=job_type_3, status='COMPLETED', save=False) job_e = job_test_utils.create_job(job_type=job_type_4, status='PENDING', num_exes=0, save=False) Job.objects.bulk_create([job_a, job_b, job_c, job_e]) condition_f = recipe_test_utils.create_recipe_condition(is_processed=True, is_accepted=False, save=True) recipe_d = recipe_test_utils.create_recipe(recipe_type=recipe_type_1) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False) recipe_node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='B', job=job_b, save=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False) recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', sub_recipe=recipe_d, save=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', condition=condition_f, save=False) recipe_nodes = [recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f] recipe_instance = RecipeInstance(definition, recipe, recipe_nodes) json = convert_recipe_to_v6_json(recipe_instance) RecipeInstanceV6(json=json.get_dict(), do_validate=True) # Revalidate self.assertSetEqual(set(json.get_dict()['nodes'].keys()), {'A', 'B', 'C', 'D', 'E', 'F'})
def test_recipe_input_conn_successful(self): """Tests calling RecipeDefinition.add_recipe_input_connection() successfully""" input_interface = Interface() input_interface.parameters = {'recipe_input_1': MagicMock()} definition = RecipeDefinition(input_interface) definition.add_job_node('node_1', 'job_type_1', '1.0', 1) definition.add_recipe_input_connection('node_1', 'input_1', 'recipe_input_1')
def test_recipe_input_conn_missing_input(self): """Tests calling RecipeDefinition.add_recipe_input_connection() with an unknown recipe input""" input_interface = Interface() definition = RecipeDefinition(input_interface) definition.add_job_node('node_1', 'job_type_1', '1.0', 1) with self.assertRaises(InvalidDefinition) as context: definition.add_recipe_input_connection('node_1', 'input_1', 'missing_recipe_input') self.assertEqual(context.exception.error.name, 'UNKNOWN_INPUT')
def test_recipe_input_conn_missing_input_node(self): """Tests calling RecipeDefinition.add_recipe_input_connection() with an unknown input node""" input_interface = Interface() input_interface.parameters = {'recipe_input_1': MagicMock()} definition = RecipeDefinition(input_interface) definition.add_job_node('node_1', 'job_type_1', '1.0', 1) with self.assertRaises(InvalidDefinition) as context: definition.add_recipe_input_connection('missing_node', 'input_1', 'recipe_input_1') self.assertEqual(context.exception.error.name, 'UNKNOWN_NODE')
def test_json(self): """Tests converting a ProcessCondition message to and from JSON""" definition = RecipeDefinition(Interface()) cond_interface_1 = Interface() cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer')) df1 = DataFilter(filter_list=[{ 'name': 'cond_int', 'type': 'integer', 'condition': '==', 'values': [0] }]) definition = RecipeDefinition(cond_interface_1) definition.add_condition_node('node_a', cond_interface_1, df1) definition.add_recipe_input_connection('node_a', 'cond_int', 'cond_int') definition_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_dict) data_1 = Data() data_1.add_value(JsonValue('cond_int', 0)) data_1_dict = convert_data_to_v6_json(data_1).get_dict() recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_1_dict) condition = recipe_test_utils.create_recipe_condition(recipe=recipe, save=True) recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_a', condition=condition, save=True) # Create message message = create_process_condition_messages([condition.id])[0] # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = ProcessCondition.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) condition = RecipeCondition.objects.get(id=condition.id) self.assertEqual(len(new_message.new_messages), 1) self.assertEqual(new_message.new_messages[0].type, 'update_recipe') self.assertEqual(new_message.new_messages[0].root_recipe_id, recipe.id) self.assertTrue(condition.is_processed) self.assertIsNotNone(condition.processed) self.assertTrue(condition.is_accepted)
def get_definition(self): """Returns the recipe definition represented by this JSON :returns: The recipe definition :rtype: :class:`recipe.definition.definition.RecipeDefinition`: """ interface_json = InterfaceV6(self._definition['input'], do_validate=False) interface = interface_json.get_interface() definition = RecipeDefinition(interface) # Add all nodes to definition first for node_name, node_dict in self._definition['nodes'].items(): node_type_dict = node_dict['node_type'] if node_type_dict['node_type'] == 'condition': cond_interface_json = InterfaceV6(node_type_dict['interface'], do_validate=False) data_filter_json = DataFilterV6(node_type_dict['data_filter'], do_validate=False) definition.add_condition_node( node_name, cond_interface_json.get_interface(), data_filter_json.get_filter()) elif node_type_dict['node_type'] == 'job': definition.add_job_node(node_name, node_type_dict['job_type_name'], node_type_dict['job_type_version'], node_type_dict['job_type_revision']) elif node_type_dict['node_type'] == 'recipe': definition.add_recipe_node( node_name, node_type_dict['recipe_type_name'], node_type_dict['recipe_type_revision']) # Now add dependencies and connections for node_name, node_dict in self._definition['nodes'].items(): for dependency_dict in node_dict['dependencies']: acceptance = dependency_dict['acceptance'] if ( 'acceptance' in dependency_dict) else True definition.add_dependency(dependency_dict['name'], node_name, acceptance) for conn_name, conn_dict in node_dict['input'].items(): if conn_dict['type'] == 'recipe': definition.add_recipe_input_connection( node_name, conn_name, conn_dict['input']) elif conn_dict['type'] == 'dependency': definition.add_dependency_input_connection( node_name, conn_name, conn_dict['node'], conn_dict['output']) return definition
def test_generate_node_input_data(self): """Tests calling RecipeDefinition.generate_node_input_data()""" input_interface = Interface() input_interface.add_parameter( FileParameter('recipe_input_1', ['image/gif'], multiple=True)) input_interface.add_parameter(JsonParameter('recipe_input_2', 'string')) definition = RecipeDefinition(input_interface) definition.add_job_node('node_a', 'job_type_1', '1.0', 1) definition.add_job_node('node_b', 'job_type_2', '1.0', 1) definition.add_job_node('node_c', 'job_type_3', '1.0', 1) definition.add_dependency('node_c', 'node_b') definition.add_dependency('node_c', 'node_a') definition.add_recipe_input_connection('node_c', 'input_1', 'recipe_input_1') definition.add_recipe_input_connection('node_c', 'input_2', 'recipe_input_2') definition.add_dependency_input_connection('node_c', 'input_3', 'node_a', 'output_a_1') definition.add_dependency_input_connection('node_c', 'input_4', 'node_a', 'output_a_2') definition.add_dependency_input_connection('node_c', 'input_5', 'node_b', 'output_b_1') recipe_data = Data() recipe_data.add_value(FileValue('recipe_input_1', [1, 2, 3, 4, 5])) recipe_data.add_value(JsonValue('recipe_input_2', 'Scale is awesome!')) a_output_data = Data() a_output_data.add_value(FileValue('output_a_1', [1234])) a_output_data.add_value(JsonValue('output_a_2', {'foo': 'bar'})) b_output_data = Data() b_output_data.add_value(JsonValue('output_b_1', 12.34)) node_outputs = { 'node_a': RecipeNodeOutput('node_a', 'job', 1, a_output_data), 'node_b': RecipeNodeOutput('node_b', 'job', 1, b_output_data) } node_data = definition.generate_node_input_data( 'node_c', recipe_data, node_outputs) self.assertSetEqual( set(node_data.values.keys()), {'input_1', 'input_2', 'input_3', 'input_4', 'input_5'}) self.assertListEqual(node_data.values['input_1'].file_ids, [1, 2, 3, 4, 5]) self.assertEqual(node_data.values['input_2'].value, 'Scale is awesome!') self.assertListEqual(node_data.values['input_3'].file_ids, [1234]) self.assertDictEqual(node_data.values['input_4'].value, {'foo': 'bar'}) self.assertEqual(node_data.values['input_5'].value, 12.34)
def test_recipe_input_conn_duplicate_input(self): """Tests calling RecipeDefinition.add_recipe_input_connection() to connect to a duplicate input""" input_interface = Interface() input_interface.parameters = {'recipe_input_1': MagicMock()} definition = RecipeDefinition(input_interface) definition.add_job_node('node_1', 'job_type_1', '1.0', 1) definition.add_job_node('node_2', 'job_type_2', '1.0', 1) definition.add_recipe_input_connection('node_1', 'input_1', 'recipe_input_1') with self.assertRaises(InvalidDefinition) as context: definition.add_recipe_input_connection('node_1', 'input_1', 'recipe_input_1') self.assertEqual(context.exception.error.name, 'NODE_INTERFACE')
def test_convert_recipe_diff_to_v6_json_new_required_input(self): """Tests calling convert_recipe_diff_to_v6_json() with a diff where there is a breaking recipe interface change """ interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) interface_2.add_parameter( JsonParameter('json_param_2', 'object', required=True)) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('C', 'D') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2 = RecipeDefinition(interface_2) definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('B', 'job_type_2', '2.0', 1) definition_2.add_job_node('C', 'job_type_3', '1.1', 1) # Change to job type version and revision definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_dependency('A', 'B') definition_2.add_dependency('A', 'C') definition_2.add_dependency('C', 'D') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_2.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_2.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') diff = RecipeDiff(definition_1, definition_2) json = convert_recipe_diff_to_v6_json(diff) RecipeDiffV6(diff=json.get_dict(), do_validate=True) # Revalidate self.assertFalse(json.get_dict()['can_be_reprocessed'])
def test_validate_successful(self): """Tests calling RecipeDefinition.validate() successfully""" recipe_interface = Interface() recipe_interface.add_parameter( JsonParameter('recipe_input_1', 'integer')) definition = RecipeDefinition(recipe_interface) definition.add_job_node('A', 'job_type_1', '1.0', 1) condition_interface = Interface() condition_interface.add_parameter( JsonParameter('cond_param', 'integer')) definition.add_condition_node('B', condition_interface, DataFilter(True)) definition.add_recipe_node('C', 'recipe_type_1', 1) definition.add_dependency('A', 'B') definition.add_dependency('B', 'C') definition.add_recipe_input_connection('A', 'a_input_1', 'recipe_input_1') definition.add_dependency_input_connection('B', 'cond_param', 'A', 'a_output_1') definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_1') definition.add_dependency_input_connection('C', 'c_input_2', 'B', 'cond_param') job_input_interface = Interface() job_input_interface.add_parameter(JsonParameter( 'a_input_1', 'integer')) job_output_interface = Interface() job_output_interface.add_parameter( JsonParameter('a_output_1', 'integer')) recipe_input_interface = Interface() recipe_input_interface.add_parameter( JsonParameter('c_input_1', 'integer')) recipe_input_interface.add_parameter( JsonParameter('c_input_2', 'integer')) input_interfaces = { 'A': job_input_interface, 'C': recipe_input_interface } output_interfaces = {'A': job_output_interface, 'C': Interface()} warnings = definition.validate(input_interfaces, output_interfaces) self.assertListEqual(warnings, [])
def test_init_new_required_input(self): """Tests creating a RecipeDiff when the newer definition has a new required input parameter""" interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) interface_2.add_parameter(JsonParameter('json_param_2', 'object', required=True)) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('C', 'D') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2 = RecipeDefinition(interface_2) definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('B', 'job_type_2', '2.0', 1) definition_2.add_job_node('C', 'job_type_3', '1.1', 1) # Change to job type version and revision definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_dependency('A', 'B') definition_2.add_dependency('A', 'C') definition_2.add_dependency('C', 'D') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_2.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_2.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') diff = RecipeDiff(definition_1, definition_2) self.assertFalse(diff.can_be_reprocessed) self.assertEqual(len(diff.reasons), 1) self.assertEqual(diff.reasons[0].name, 'INPUT_CHANGE') # Cannot be reprocessed, so no nodes to copy, supersede, or unpublish self.assertDictEqual(diff.get_nodes_to_copy(), {}) self.assertDictEqual(diff.get_nodes_to_supersede(), {}) self.assertDictEqual(diff.get_nodes_to_unpublish(), {}) # Ensure no nodes have reprocess_new_node set to true for node_diff in diff.graph.values(): self.assertFalse(node_diff.reprocess_new_node)
def test_condition_hit(self): """Tests calling Recipe.has_completed() when an entire recipe has completed""" """ Job -> Condition -> Recipe parse-job -> condition-node -> recipe-node """ manifest_1 = { 'seedVersion': '1.0.0', 'job': { 'name': 'parse-job', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': 'Test Parse Job', 'description': 'Test Parse job', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': { 'files': [{ 'name': 'INPUT_FILE', 'mediaTypes': ['image/x-hdf5-image'], 'required': True }], 'json': [] }, 'outputs': { 'files': [{ 'name': 'OUTPUT_A', 'pattern': '*.png', 'multiple': True }] } } } } job_type_1 = job_test_utils.create_seed_job_type(manifest=manifest_1) input_interface = Interface() input_interface.add_parameter( FileParameter('INPUT_FILE', ['image/x-hdf5-image'], multiple=False)) definition = RecipeDefinition(input_interface) definition.add_job_node('parse-job', job_type_1.name, job_type_1.version, job_type_1.revision_num) cond_interface = Interface() cond_interface.add_parameter(FileParameter('INPUT_FILE', ['image/png'])) df = DataFilter(filter_list=[{ 'name': 'cond', 'type': 'media-type', 'condition': '==', 'value': ['image/png'] }]) definition.add_condition_node('condition-node', cond_interface, df) sub_job_manifest = { 'seedVersion': '1.0.0', 'job': { 'name': 'recipe-job', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': 'Test Recipe Job', 'description': 'Test Recipe job', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': { 'files': [{ 'name': 'INPUT_FILE', 'mediaTypes': ['image/png'], 'required': True }], 'json': [] }, 'outputs': { 'files': [{ 'name': 'OUTPUT_A', 'pattern': '*.png', 'multiple': True }] } } } } sub_job = job_test_utils.create_seed_job_type( manifest=sub_job_manifest) sub_interface = Interface() sub_interface.add_parameter(FileParameter('INPUT_FILE', ['image/png'])) definition_b = RecipeDefinition(sub_interface) definition_b.add_job_node('job_b', sub_job.name, sub_job.version, sub_job.revision_num) definition_b.add_recipe_input_connection('job_b', 'INPUT_FILE', 'INPUT_FILE') definition_b_dict = convert_recipe_definition_to_v6_json( definition_b).get_dict() sub_recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_b_dict) definition.add_recipe_node('recipe-node', sub_recipe_type.name, sub_recipe_type.revision_num) # Connect the recipe input to the parse job definition.add_recipe_input_connection('parse-job', 'INPUT_FILE', 'INPUT_FILE') # Connect the condition node to the parse job output definition.add_dependency_input_connection('condition-node', 'cond', 'parse-job', 'OUTPUT_A') # Connect the sub recipe to the condition output definition.add_dependency_input_connection('recipe-node', 'INPUT_FILE', 'condition-node', 'cond')
def test_convert_recipe_to_v6_json(self): """Tests calling convert_recipe_to_v6_json() successfully""" job_type_1 = job_test_utils.create_job_type() job_type_2 = job_test_utils.create_job_type() job_type_3 = job_test_utils.create_job_type() job_type_4 = job_test_utils.create_job_type() recipe_type_1 = recipe_test_utils.create_recipe_type() interface = Interface() interface.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface.add_parameter(JsonParameter('json_param_1', 'object')) definition = RecipeDefinition(interface) definition.add_job_node('A', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('B', job_type_2.name, job_type_2.version, job_type_2.revision_num) definition.add_job_node('C', job_type_3.name, job_type_3.version, job_type_3.revision_num) definition.add_recipe_node('D', recipe_type_1.name, recipe_type_1.revision_num) definition.add_job_node('E', job_type_4.name, job_type_4.version, job_type_4.revision_num) definition.add_dependency('A', 'B') definition.add_dependency('A', 'C') definition.add_dependency('B', 'E') definition.add_dependency('C', 'D') definition.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') recipe = recipe_test_utils.create_recipe() job_a = job_test_utils.create_job(job_type=job_type_1, status='COMPLETED', save=False) job_b = job_test_utils.create_job(job_type=job_type_2, status='RUNNING', save=False) job_c = job_test_utils.create_job(job_type=job_type_3, status='COMPLETED', save=False) job_e = job_test_utils.create_job(job_type=job_type_4, status='PENDING', num_exes=0, save=False) Job.objects.bulk_create([job_a, job_b, job_c, job_e]) recipe_d = recipe_test_utils.create_recipe(recipe_type=recipe_type_1) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False) recipe_node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='B', job=job_b, save=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False) recipe_node_d = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='D', sub_recipe=recipe_d, save=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_nodes = [ recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e ] recipe_instance = RecipeInstance(definition, recipe_nodes) json = convert_recipe_to_v6_json(recipe_instance) RecipeInstanceV6(json=json.get_dict(), do_validate=True) # Revalidate self.assertSetEqual(set(json.get_dict()['nodes'].keys()), {'A', 'B', 'C', 'D', 'E'})
def test_execute_with_recipe(self): """Tests calling ProcessRecipeInput.execute() successfully when a sub-recipe has to get its data from its recipe """ workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0) file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0) manifest_a = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-a', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': { 'files': [], 'json': [] }, 'outputs': { 'files': [{ 'name': 'output_a', 'pattern': '*.png' }] } } } } job_type_a = job_test_utils.create_job_type(interface=manifest_a) output_data_a = Data() output_data_a.add_value(FileValue('output_a', [file_1.id])) output_data_a_dict = convert_data_to_v6_json(output_data_a).get_dict() manifest_b = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-b', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': { 'files': [], 'json': [] }, 'outputs': { 'files': [{ 'name': 'output_b', 'pattern': '*.png', 'multiple': True }] } } } } job_type_b = job_test_utils.create_job_type(interface=manifest_b) output_data_b = Data() output_data_b.add_value( FileValue('output_b', [file_2.id, file_3.id, file_4.id])) output_data_b_dict = convert_data_to_v6_json(output_data_b).get_dict() job_a = job_test_utils.create_job(job_type=job_type_a, num_exes=1, status='COMPLETED', output=output_data_a_dict) job_b = job_test_utils.create_job(job_type=job_type_b, num_exes=1, status='COMPLETED', output=output_data_b_dict) sub_recipe_interface_c = Interface() sub_recipe_interface_c.add_parameter( FileParameter('input_a', ['image/png'])) sub_recipe_interface_c.add_parameter( FileParameter('input_b', ['image/png'], multiple=True)) sub_recipe_interface_c.add_parameter(JsonParameter( 'input_c', 'string')) sub_recipe_def_c = RecipeDefinition(sub_recipe_interface_c) sub_recipe_def_dict_c = convert_recipe_definition_to_v6_json( sub_recipe_def_c).get_dict() sub_recipe_type_c = recipe_test_utils.create_recipe_type( definition=sub_recipe_def_dict_c) sub_recipe_c = recipe_test_utils.create_recipe( recipe_type=sub_recipe_type_c) recipe_interface = Interface() recipe_interface.add_parameter(JsonParameter('recipe_input', 'string')) definition = RecipeDefinition(recipe_interface) definition.add_job_node('node_a', job_type_a.name, job_type_a.version, job_type_a.revision_num) definition.add_job_node('node_b', job_type_b.name, job_type_b.version, job_type_b.revision_num) definition.add_recipe_node('node_c', sub_recipe_type_c.name, sub_recipe_type_c.revision_num) definition.add_recipe_input_connection('node_c', 'input_c', 'recipe_input') definition.add_dependency('node_c', 'node_a') definition.add_dependency_input_connection('node_c', 'input_a', 'node_a', 'output_a') definition.add_dependency('node_c', 'node_b') definition.add_dependency_input_connection('node_c', 'input_b', 'node_b', 'output_b') def_dict = convert_recipe_definition_to_v6_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type(definition=def_dict) recipe_data = Data() recipe_data.add_value(JsonValue('recipe_input', 'hello')) recipe_data_dict = convert_data_to_v6_json(recipe_data).get_dict() recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict) recipe_node_a = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='node_a', job=job_a) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='node_b', job=job_b) recipe_node_c = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='node_c', sub_recipe=sub_recipe_c) RecipeNode.objects.bulk_create( [recipe_node_a, recipe_node_b, recipe_node_c]) job_a.recipe = recipe job_a.save() job_b.recipe = recipe job_b.save() sub_recipe_c.recipe = recipe sub_recipe_c.save() # Create message message = ProcessRecipeInput() message.recipe_id = sub_recipe_c.id # Execute message result = message.execute() self.assertTrue(result) sub_recipe_c = Recipe.objects.get(id=sub_recipe_c.id) # Check for update_recipes message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'update_recipes') # Check sub-recipe for expected input_file_size self.assertEqual(sub_recipe_c.input_file_size, 24469.0) # Check sub-recipe for expected input data self.assertNotEqual(sub_recipe_c.input['version'], '1.0') # Should not be legacy self.assertSetEqual(set(sub_recipe_c.get_input_data().values.keys()), {'input_a', 'input_b', 'input_c'}) self.assertListEqual( sub_recipe_c.get_input_data().values['input_a'].file_ids, [file_1.id]) self.assertListEqual( sub_recipe_c.get_input_data().values['input_b'].file_ids, [file_2.id, file_3.id, file_4.id]) self.assertEqual(sub_recipe_c.get_input_data().values['input_c'].value, 'hello') # Make sure sub-recipe input file models are created input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id) self.assertEqual(len(input_files), 4) file_ids = {input_file.input_file_id for input_file in input_files} self.assertSetEqual(file_ids, {file_1.id, file_2.id, file_3.id, file_4.id}) # Test executing message again message_json_dict = message.to_json() message = ProcessRecipeInput.from_json(message_json_dict) result = message.execute() self.assertTrue(result) # Still should have update_recipes message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'update_recipes') # Make sure recipe input file models are unchanged input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id) self.assertEqual(len(input_files), 4)
def test_set_force_reprocess(self): """Tests calling RecipeDiff.set_force_reprocess()""" interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_job_node('E', 'job_type_4', '1.0', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('C', 'D') definition_1.add_dependency('C', 'E') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_1.add_dependency_input_connection('E', 'e_input_1', 'C', 'c_output_1') # No changes in definition 2 definition_2 = RecipeDefinition(interface_2) definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('B', 'job_type_2', '2.0', 1) definition_2.add_job_node('C', 'job_type_3', '1.0', 2) definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_job_node('E', 'job_type_4', '1.0', 1) definition_2.add_dependency('A', 'B') definition_2.add_dependency('A', 'C') definition_2.add_dependency('C', 'D') definition_2.add_dependency('C', 'E') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_2.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_2.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2.add_dependency_input_connection('E', 'e_input_1', 'C', 'c_output_1') recipe_d_forced_nodes = ForcedNodes() recipe_d_forced_nodes.add_node('1') recipe_d_forced_nodes.add_node('2') top_forced_nodes = ForcedNodes() top_forced_nodes.add_node('C') top_forced_nodes.add_subrecipe('D', recipe_d_forced_nodes) diff = RecipeDiff(definition_1, definition_2) diff.set_force_reprocess(top_forced_nodes) # No recipe input changes so recipe can be reprocessed self.assertTrue(diff.can_be_reprocessed) self.assertListEqual(diff.reasons, []) # Check each node for correct fields node_a = diff.graph['A'] self.assertEqual(node_a.status, NodeDiff.UNCHANGED) self.assertFalse(node_a.reprocess_new_node) self.assertListEqual(node_a.changes, []) node_b = diff.graph['B'] self.assertEqual(node_b.status, NodeDiff.UNCHANGED) self.assertFalse(node_b.reprocess_new_node) self.assertListEqual(node_b.changes, []) node_c = diff.graph['C'] self.assertEqual(node_c.status, NodeDiff.UNCHANGED) self.assertTrue(node_c.reprocess_new_node) # Force reprocess self.assertListEqual(node_c.changes, []) node_d = diff.graph['D'] self.assertEqual(node_d.status, NodeDiff.UNCHANGED) self.assertTrue(node_d.reprocess_new_node) # Force reprocess self.assertListEqual(node_d.changes, []) # Check forced nodes object that got passed to recipe node D self.assertEqual(node_d.force_reprocess_nodes, recipe_d_forced_nodes) node_e = diff.graph['E'] self.assertEqual(node_e.status, NodeDiff.UNCHANGED) self.assertTrue( node_e.reprocess_new_node) # Force reprocess due to C being forced self.assertListEqual(node_e.changes, []) # Check nodes to copy, supersede, and unpublish self.assertSetEqual(set(diff.get_nodes_to_copy().keys()), {'A', 'B'}) self.assertSetEqual(set(diff.get_nodes_to_supersede().keys()), {'C', 'D', 'E'}) self.assertSetEqual(set(diff.get_nodes_to_unpublish().keys()), set())
def test_init_changes_in_middle_of_chains(self): """Tests creating a RecipeDiff where nodes are deleted from and inserted into the middle of a chain""" interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('C', 'D') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2 = RecipeDefinition(interface_2) definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('B', 'job_type_2', '2.0', 1) # Node C is deleted definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_job_node('E', 'job_type_4', '5.0', 2) # New node inbetween A and B definition_2.add_dependency('A', 'E') definition_2.add_dependency('E', 'B') definition_2.add_dependency('A', 'D') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('E', 'e_input_1', 'A', 'a_output_1') definition_2.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_2.add_dependency_input_connection('D', 'd_input_1', 'A', 'a_output_2') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') diff = RecipeDiff(definition_1, definition_2) # No recipe input changes so recipe can be reprocessed self.assertTrue(diff.can_be_reprocessed) self.assertListEqual(diff.reasons, []) # Check each node for correct fields node_a = diff.graph['A'] self.assertEqual(node_a.status, NodeDiff.UNCHANGED) self.assertFalse(node_a.reprocess_new_node) self.assertListEqual(node_a.changes, []) node_b = diff.graph['B'] self.assertEqual(node_b.status, NodeDiff.CHANGED) self.assertTrue(node_b.reprocess_new_node) self.assertEqual(len(node_b.changes), 2) self.assertEqual(node_b.changes[0].name, 'PARENT_NEW') self.assertEqual(node_b.changes[1].name, 'PARENT_REMOVED') node_c = diff.graph['C'] self.assertEqual(node_c.status, NodeDiff.DELETED) self.assertFalse(node_c.reprocess_new_node) self.assertListEqual(node_c.changes, []) node_d = diff.graph['D'] self.assertEqual(node_d.status, NodeDiff.CHANGED) self.assertTrue(node_d.reprocess_new_node) self.assertEqual(len(node_d.changes), 3) self.assertEqual(node_d.changes[0].name, 'PARENT_NEW') self.assertEqual(node_d.changes[1].name, 'PARENT_REMOVED') self.assertEqual(node_d.changes[2].name, 'INPUT_CHANGE') node_e = diff.graph['E'] self.assertEqual(node_e.status, NodeDiff.NEW) self.assertTrue(node_e.reprocess_new_node) self.assertListEqual(node_e.changes, []) # Check nodes to copy, supersede, and unpublish self.assertSetEqual(set(diff.get_nodes_to_copy().keys()), {'A'}) self.assertSetEqual(set(diff.get_nodes_to_supersede().keys()), {'B', 'C', 'D'}) self.assertSetEqual(set(diff.get_nodes_to_unpublish().keys()), {'C'})
def test_init_identical(self): """Tests creating a RecipeDiff between two identical recipe definitions""" interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('C', 'D') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2 = RecipeDefinition(interface_2) definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('B', 'job_type_2', '2.0', 1) definition_2.add_job_node('C', 'job_type_3', '1.0', 2) definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_dependency('A', 'B') definition_2.add_dependency('A', 'C') definition_2.add_dependency('C', 'D') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_2.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_2.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') diff = RecipeDiff(definition_1, definition_2) self.assertTrue(diff.can_be_reprocessed) self.assertListEqual(diff.reasons, []) # Every node should be unchanged and all should be copied during a reprocess nodes_to_copy = diff.get_nodes_to_copy() self.assertSetEqual(set(nodes_to_copy.keys()), {'A', 'B', 'C', 'D'}) for node_diff in nodes_to_copy.values(): self.assertEqual(node_diff.status, NodeDiff.UNCHANGED) self.assertFalse(node_diff.reprocess_new_node) self.assertListEqual(node_diff.changes, []) self.assertDictEqual(diff.get_nodes_to_supersede(), {}) self.assertDictEqual(diff.get_nodes_to_unpublish(), {})
def test_init_changes(self): """Tests creating a RecipeDiff when the newer definition has a variety of changes in it""" interface_1 = Interface() interface_1.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_1.add_parameter(JsonParameter('json_param_1', 'object')) interface_2 = Interface() interface_2.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface_2.add_parameter(JsonParameter('json_param_1', 'object')) interface_2.add_parameter( JsonParameter('json_param_2', 'object', required=False)) definition_1 = RecipeDefinition(interface_1) definition_1.add_job_node('A', 'job_type_1', '1.0', 1) definition_1.add_job_node('B', 'job_type_2', '2.0', 1) definition_1.add_job_node('C', 'job_type_3', '1.0', 2) definition_1.add_recipe_node('D', 'recipe_type_1', 1) definition_1.add_job_node('E', 'job_type_4', '1.0', 1) definition_1.add_dependency('A', 'B') definition_1.add_dependency('A', 'C') definition_1.add_dependency('B', 'E') definition_1.add_dependency('C', 'D') definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2 = RecipeDefinition(interface_2) # Nodes B and E are deleted definition_2.add_job_node('A', 'job_type_1', '1.0', 1) definition_2.add_job_node('C', 'job_type_3', '2.1', 1) # Change to job type version and revision definition_2.add_recipe_node('D', 'recipe_type_1', 1) definition_2.add_recipe_node('F', 'recipe_type_2', 5) # New node definition_2.add_dependency('A', 'C') definition_2.add_dependency('C', 'D') definition_2.add_dependency('D', 'F') definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition_2.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition_2.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') definition_2.add_recipe_input_connection('F', 'f_input_1', 'json_param_2') diff = RecipeDiff(definition_1, definition_2) # Non-breaking recipe input changes so recipe can be reprocessed self.assertTrue(diff.can_be_reprocessed) self.assertListEqual(diff.reasons, []) # Check each node for correct fields node_a = diff.graph['A'] self.assertEqual(node_a.status, NodeDiff.UNCHANGED) self.assertFalse(node_a.reprocess_new_node) self.assertListEqual(node_a.changes, []) node_b = diff.graph['B'] self.assertEqual(node_b.status, NodeDiff.DELETED) self.assertFalse(node_b.reprocess_new_node) self.assertListEqual(node_b.changes, []) node_c = diff.graph['C'] self.assertEqual(node_c.status, NodeDiff.CHANGED) self.assertTrue(node_c.reprocess_new_node) self.assertEqual(len(node_c.changes), 2) self.assertEqual(node_c.changes[0].name, 'JOB_TYPE_VERSION_CHANGE') self.assertEqual(node_c.changes[1].name, 'JOB_TYPE_REVISION_CHANGE') node_d = diff.graph['D'] self.assertEqual(node_d.status, NodeDiff.CHANGED) self.assertTrue(node_d.reprocess_new_node) self.assertEqual(len(node_d.changes), 1) self.assertEqual(node_d.changes[0].name, 'PARENT_CHANGED') node_e = diff.graph['E'] self.assertEqual(node_e.status, NodeDiff.DELETED) self.assertFalse(node_e.reprocess_new_node) self.assertListEqual(node_e.changes, []) node_f = diff.graph['F'] self.assertEqual(node_f.status, NodeDiff.NEW) self.assertTrue(node_f.reprocess_new_node) self.assertListEqual(node_f.changes, []) # Check nodes to copy, supersede, and unpublish self.assertSetEqual(set(diff.get_nodes_to_copy().keys()), {'A'}) self.assertSetEqual(set(diff.get_nodes_to_supersede().keys()), {'B', 'C', 'D', 'E'}) self.assertSetEqual(set(diff.get_nodes_to_unpublish().keys()), {'B', 'E'})