def test_convert_recipe_definition_to_v1_json_full(self): """Tests calling convert_recipe_definition_to_v1_json() with a full definition""" interface = Interface() interface.add_parameter(FileParameter('file_param_a', ['image/gif'])) interface.add_parameter(JsonParameter('json_param_a', 'object')) interface.add_parameter( JsonParameter('json_param_b', 'object', required=False)) definition = RecipeDefinition(interface) definition.add_job_node('A', 'job_type_1', '1.0', 1) definition.add_job_node('B', 'job_type_2', '2.0', 1) definition.add_job_node('C', 'job_type_3', '1.0', 2) definition.add_recipe_node('D', 'recipe_type_1', 1) definition.add_job_node('E', 'job_type_4', '1.0', 1) definition.add_dependency('A', 'B') definition.add_dependency('A', 'C') definition.add_dependency('B', 'E') definition.add_dependency('C', 'D') definition.add_recipe_input_connection('A', 'input_1', 'file_param_a') definition.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition.add_recipe_input_connection('D', 'd_input_2', 'json_param_a') json = convert_recipe_definition_to_v1_json(definition) RecipeDefinitionV1(definition=json.get_dict(), do_validate=True) # Revalidate job_names = {job_dict['name'] for job_dict in json.get_dict()['jobs']} self.assertSetEqual(job_names, {'A', 'B', 'C', 'E'}) # D is omitted (recipe node)
def test_convert_recipe_definition_to_v1_json_empty(self): """Tests calling convert_recipe_definition_to_v1_json() with an empty definition""" interface = Interface() definition = RecipeDefinition(interface) json = convert_recipe_definition_to_v1_json(definition) RecipeDefinitionV1(definition=json.get_dict(), do_validate=True) # Revalidate
def test_execute_with_recipe_legacy(self): """Tests calling ProcessRecipeInput.execute() successfully when a legacy sub-recipe has to get its data from its recipe """ workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0) file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0) manifest_a = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-a', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': { 'files': [], 'json': [] }, 'outputs': { 'files': [{ 'name': 'output_a', 'pattern': '*.png' }] } } } } job_type_a = job_test_utils.create_job_type(interface=manifest_a) output_data_a = Data() output_data_a.add_value(FileValue('output_a', [file_1.id])) output_data_a_dict = convert_data_to_v6_json(output_data_a).get_dict() manifest_b = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-b', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': { 'files': [], 'json': [] }, 'outputs': { 'files': [{ 'name': 'output_b', 'pattern': '*.png', 'multiple': True }] } } } } job_type_b = job_test_utils.create_job_type(interface=manifest_b) output_data_b = Data() output_data_b.add_value( FileValue('output_b', [file_2.id, file_3.id, file_4.id])) output_data_b_dict = convert_data_to_v6_json(output_data_b).get_dict() job_a = job_test_utils.create_job(job_type=job_type_a, num_exes=1, status='COMPLETED', output=output_data_a_dict) job_b = job_test_utils.create_job(job_type=job_type_b, num_exes=1, status='COMPLETED', output=output_data_b_dict) sub_recipe_interface_c = Interface() sub_recipe_interface_c.add_parameter( FileParameter('input_a', ['image/png'])) sub_recipe_interface_c.add_parameter( FileParameter('input_b', ['image/png'], multiple=True)) sub_recipe_def_c = RecipeDefinition(sub_recipe_interface_c) sub_recipe_def_dict_c = convert_recipe_definition_to_v1_json( sub_recipe_def_c).get_dict() sub_recipe_type_c = recipe_test_utils.create_recipe_type( definition=sub_recipe_def_dict_c) sub_recipe_c = recipe_test_utils.create_recipe( recipe_type=sub_recipe_type_c) definition = RecipeDefinition(Interface()) definition.add_job_node('node_a', job_type_a.name, job_type_a.version, job_type_a.revision_num) definition.add_job_node('node_b', job_type_b.name, job_type_b.version, job_type_b.revision_num) definition.add_recipe_node('node_c', sub_recipe_type_c.name, sub_recipe_type_c.revision_num) definition.add_dependency('node_c', 'node_a') definition.add_dependency_input_connection('node_c', 'input_a', 'node_a', 'output_a') definition.add_dependency('node_c', 'node_b') definition.add_dependency_input_connection('node_c', 'input_b', 'node_b', 'output_b') def_dict = convert_recipe_definition_to_v6_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type(definition=def_dict) recipe_data_dict = { 'version': '1.0', 'input_data': [], 'workspace_id': workspace.id } recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict) recipe_node_a = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='node_a', job=job_a) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='node_b', job=job_b) recipe_node_c = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='node_c', sub_recipe=sub_recipe_c) RecipeNode.objects.bulk_create( [recipe_node_a, recipe_node_b, recipe_node_c]) job_a.recipe = recipe job_a.save() job_b.recipe = recipe job_b.save() sub_recipe_c.recipe = recipe sub_recipe_c.save() # Create message message = ProcessRecipeInput() message.recipe_id = sub_recipe_c.id # Execute message result = message.execute() self.assertTrue(result) sub_recipe_c = Recipe.objects.get(id=sub_recipe_c.id) # Check for update_recipes message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'update_recipes') # Check sub-recipe for expected input_file_size self.assertEqual(sub_recipe_c.input_file_size, 24469.0) # Check sub-recipe for expected input data self.assertEqual( sub_recipe_c.input['version'], '1.0') # Should be legacy input data with workspace ID self.assertEqual(sub_recipe_c.input['workspace_id'], workspace.id) self.assertSetEqual(set(sub_recipe_c.get_input_data().values.keys()), {'input_a', 'input_b'}) self.assertListEqual( sub_recipe_c.get_input_data().values['input_a'].file_ids, [file_1.id]) self.assertListEqual( sub_recipe_c.get_input_data().values['input_b'].file_ids, [file_2.id, file_3.id, file_4.id]) # Make sure sub-recipe input file models are created input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id) self.assertEqual(len(input_files), 4) file_ids = {input_file.input_file_id for input_file in input_files} self.assertSetEqual(file_ids, {file_1.id, file_2.id, file_3.id, file_4.id}) # Test executing message again message_json_dict = message.to_json() message = ProcessRecipeInput.from_json(message_json_dict) result = message.execute() self.assertTrue(result) # Still should have update_recipes message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'update_recipes') # Make sure recipe input file models are unchanged input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id) self.assertEqual(len(input_files), 4)
def test_execute_with_recipe_missing_output(self): """Tests calling ProcessJobInput.execute() when a job has to get its data from its recipe but the previous job didn't provide it's output""" workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0) file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0) manifest_1 = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-a', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': {'files': [], 'json': []}, 'outputs': { 'files': [{'name': 'OUTPUT_A', 'pattern': '*.png'}] } } } } job_type_1 = job_test_utils.create_seed_job_type(manifest=manifest_1) manifest_2 = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-b', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': {'files': [{'name': 'INPUT_B', 'multiple': True}]}, 'outputs': { 'files': [{'name': 'OUTPUT_B', 'pattern': '*.png'}] } } } } job_type_2 = job_test_utils.create_seed_job_type(manifest=manifest_2) output_dict = { 'version': '1.0', 'output_data': [{ }] } job_1 = job_test_utils.create_job(job_type=job_type_1, num_exes=1, status='COMPLETED', output=output_dict) job_2 = job_test_utils.create_job(job_type=job_type_2, num_exes=0, status='PENDING', input_file_size=None, input=None) from recipe.definition.definition import RecipeDefinition from recipe.definition.json.definition_v1 import convert_recipe_definition_to_v1_json from recipe.test import utils as recipe_test_utils definition = RecipeDefinition(Interface()) definition.add_job_node('node_a', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('node_b', job_type_2.name, job_type_2.version, job_type_2.revision_num) definition.add_dependency('node_a', 'node_b') definition.add_dependency_input_connection('node_b', 'INPUT_B', 'node_a', 'OUTPUT_A') def_dict = convert_recipe_definition_to_v1_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6(definition=def_dict) recipe_data_dict = {'version': '1.0', 'input_data': [], 'workspace_id': workspace.id} recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_a', job=job_1) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_b', job=job_2) job_1.recipe = recipe job_1.save() job_2.recipe = recipe job_2.save() # Create message message = ProcessJobInput() message.job_id = job_2.id # Execute message result = message.execute() self.assertTrue(result) job_2 = Job.objects.get(id=job_2.id) # Check for queued jobs message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'cancel_jobs')
def test_execute_with_recipe(self): """Tests calling ProcessJobInput.execute() successfully when a job has to get its data from its recipe""" workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0) file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0) manifest_1 = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-a', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': {'files': [], 'json': []}, 'outputs': { 'files': [{'name': 'OUTPUT_A', 'pattern': '*.png'}] } } } } job_type_1 = job_test_utils.create_seed_job_type(manifest=manifest_1) manifest_2 = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-b', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': {'files': [{'name': 'INPUT_B', 'multiple': True}]}, 'outputs': { 'files': [{'name': 'OUTPUT_B', 'pattern': '*.png'}] } } } } job_type_2 = job_test_utils.create_seed_job_type(manifest=manifest_2) output_dict = { 'version': '1.0', 'output_data': [{ 'name': 'OUTPUT_A', 'file_ids': [file_1.id, file_2.id, file_3.id, file_4.id] }] } job_1 = job_test_utils.create_job(job_type=job_type_1, num_exes=1, status='COMPLETED', output=output_dict) job_2 = job_test_utils.create_job(job_type=job_type_2, num_exes=0, status='PENDING', input_file_size=None, input=None) from recipe.definition.definition import RecipeDefinition from recipe.definition.json.definition_v1 import convert_recipe_definition_to_v1_json from recipe.test import utils as recipe_test_utils definition = RecipeDefinition(Interface()) definition.add_job_node('node_a', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('node_b', job_type_2.name, job_type_2.version, job_type_2.revision_num) definition.add_dependency('node_a', 'node_b') definition.add_dependency_input_connection('node_b', 'INPUT_B', 'node_a', 'OUTPUT_A') def_dict = convert_recipe_definition_to_v1_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6(definition=def_dict) recipe_data_dict = {'version': '1.0', 'input_data': [], 'workspace_id': workspace.id} recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_a', job=job_1) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_b', job=job_2) job_1.recipe = recipe job_1.save() job_2.recipe = recipe job_2.save() # Create message message = ProcessJobInput() message.job_id = job_2.id # Execute message result = message.execute() self.assertTrue(result) job_2 = Job.objects.get(id=job_2.id) # Check for queued jobs message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'queued_jobs') self.assertFalse(message.new_messages[0].requeue) # Check job for expected input_file_size self.assertEqual(job_2.input_file_size, 24469.0) # Check job for expected input data self.assertSetEqual(set(job_2.get_input_data().values.keys()), {'INPUT_B'}) self.assertListEqual(job_2.get_input_data().values['INPUT_B'].file_ids, [file_1.id, file_2.id, file_3.id, file_4.id]) # Make sure job input file models are created job_input_files = JobInputFile.objects.filter(job_id=job_2.id) self.assertEqual(len(job_input_files), 4) file_ids = set() for job_input_file in job_input_files: self.assertEqual(job_input_file.job_input, 'INPUT_B') file_ids.add(job_input_file.input_file_id) self.assertSetEqual(file_ids, {file_1.id, file_2.id, file_3.id, file_4.id}) # Test executing message again message_json_dict = message.to_json() message = ProcessJobInput.from_json(message_json_dict) result = message.execute() self.assertTrue(result) # Still should have queued jobs message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'queued_jobs') self.assertFalse(message.new_messages[0].requeue) # Make sure job input file models are unchanged job_input_files = JobInputFile.objects.filter(job_id=job_2.id) self.assertEqual(len(job_input_files), 4)
def test_execute_with_recipe_legacy(self): """Tests calling ProcessJobInput.execute() successfully when a legacy job has to get its data from its recipe""" workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0) file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0) interface_1 = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [], 'output_data': [{ 'name': 'Output 1', 'type': 'files', 'media_type': 'image/png', }] } job_type_1 = job_test_utils.create_job_type(interface=interface_1) interface_2 = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Input 1', 'type': 'files', 'media_types': ['image/png'], }], 'output_data': [{ 'name': 'New Output 1', 'type': 'files', 'media_type': 'image/png', }] } job_type_2 = job_test_utils.create_job_type(interface=interface_2) output_dict = { 'version': '1.0', 'output_data': [{ 'name': 'Output 1', 'file_ids': [file_1.id, file_2.id, file_3.id, file_4.id] }] } job_1 = job_test_utils.create_job(job_type=job_type_1, num_exes=1, status='COMPLETED', output=output_dict) job_2 = job_test_utils.create_job(job_type=job_type_2, num_exes=0, status='PENDING', input_file_size=None, input=None) from recipe.definition.definition import RecipeDefinition from recipe.definition.json.definition_v1 import convert_recipe_definition_to_v1_json from recipe.test import utils as recipe_test_utils definition = RecipeDefinition(Interface()) definition.add_job_node('node_a', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('node_b', job_type_2.name, job_type_2.version, job_type_2.revision_num) definition.add_dependency('node_b', 'node_a') definition.add_dependency_input_connection('node_b', 'Input 1', 'node_a', 'Output 1') def_dict = convert_recipe_definition_to_v1_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type(definition=def_dict) recipe_data_dict = { 'version': '1.0', 'input_data': [], 'workspace_id': workspace.id } recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_a', job=job_1) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_b', job=job_2) job_1.recipe = recipe job_1.save() job_2.recipe = recipe job_2.save() # Create message message = ProcessJobInput() message.job_id = job_2.id # Execute message result = message.execute() self.assertTrue(result) job_2 = Job.objects.get(id=job_2.id) # Check for queued jobs message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'queued_jobs') self.assertFalse(message.new_messages[0].requeue) # Check job for expected input_file_size self.assertEqual(job_2.input_file_size, 24469.0) # Check job for expected output workspaces in job input data (legacy) self.assertDictEqual( job_2.input, { 'version': '1.0', 'input_data': [{ 'name': 'Input 1', 'file_ids': [file_1.id, file_2.id, file_3.id, file_4.id] }], 'output_data': [{ 'name': 'New Output 1', 'workspace_id': workspace.id }] }) # Make sure job input file models are created job_input_files = JobInputFile.objects.filter(job_id=job_2.id) self.assertEqual(len(job_input_files), 4) file_ids = set() for job_input_file in job_input_files: self.assertEqual(job_input_file.job_input, 'Input 1') file_ids.add(job_input_file.input_file_id) self.assertSetEqual(file_ids, {file_1.id, file_2.id, file_3.id, file_4.id}) # Test executing message again message_json_dict = message.to_json() message = ProcessJobInput.from_json(message_json_dict) result = message.execute() self.assertTrue(result) # Still should have queued jobs message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'queued_jobs') self.assertFalse(message.new_messages[0].requeue) # Make sure job input file models are unchanged job_input_files = JobInputFile.objects.filter(job_id=job_2.id) self.assertEqual(len(job_input_files), 4)