def test_json(self): """Tests converting a ProcessJobInput message to and from JSON""" job = job_test_utils.create_job(num_exes=0, status='PENDING', input_file_size=None, input=DataV6().get_dict()) # Create message message = ProcessJobInput() message.job_id = job.id # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = ProcessJobInput.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) job = Job.objects.get(id=job.id) self.assertEqual(len(new_message.new_messages), 1) self.assertEqual(new_message.new_messages[0].type, 'queued_jobs') self.assertFalse(new_message.new_messages[0].requeue) # Job should have input_file_size set to 0 (no input files) self.assertEqual(job.input_file_size, 0.0)
def test_execute_with_data(self): """Tests calling ProcessJobInput.execute() successfully when the job already has data populated""" workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) inputs = [{ 'name': 'Input 1', 'mediaTypes': ['text/plain'], }, { 'name': 'Input 2', 'mediaTypes': ['text/plain'], }] outputs = [{ 'name': 'Output 1', 'mediaType': 'image/png', }] manifest = job_test_utils.create_seed_manifest(command='my_command', inputs_files=inputs, outputs_files=outputs) job_type = job_test_utils.create_seed_job_type(manifest=manifest) input_dict = { 'version': '1.0', 'input_data': [{ 'name': 'Input 1', 'file_id': file_1.id }, { 'name': 'Input 2', 'file_id': file_2.id }], 'output_data': [{ 'name': 'Output 1', 'workspace_id': workspace.id }]} job = job_test_utils.create_job(job_type=job_type, num_exes=0, status='PENDING', input_file_size=None, input=input_dict) # Create message message = ProcessJobInput() message.job_id = job.id # Execute message result = message.execute() self.assertTrue(result) job = Job.objects.get(id=job.id) # Check for queued jobs message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'queued_jobs') self.assertFalse(message.new_messages[0].requeue) # Check job for expected input_file_size self.assertEqual(job.input_file_size, 1042.0) # Make sure job input file models are created job_input_files = JobInputFile.objects.filter(job_id=job.id) self.assertEqual(len(job_input_files), 2) for job_input_file in job_input_files: if job_input_file.job_input == 'Input 1': self.assertEqual(job_input_file.input_file_id, file_1.id) elif job_input_file.job_input == 'Input 2': self.assertEqual(job_input_file.input_file_id, file_2.id) else: self.fail('Invalid input name: %s' % job_input_file.job_input) # Test executing message again message_json_dict = message.to_json() message = ProcessJobInput.from_json(message_json_dict) result = message.execute() self.assertTrue(result) # Still should have queued jobs message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'queued_jobs') self.assertFalse(message.new_messages[0].requeue) # Make sure job input file models are unchanged job_input_files = JobInputFile.objects.filter(job_id=job.id) self.assertEqual(len(job_input_files), 2)
def test_execute_with_recipe_missing_output(self): """Tests calling ProcessJobInput.execute() when a job has to get its data from its recipe but the previous job didn't provide it's output""" workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0) file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0) manifest_1 = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-a', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': {'files': [], 'json': []}, 'outputs': { 'files': [{'name': 'OUTPUT_A', 'pattern': '*.png'}] } } } } job_type_1 = job_test_utils.create_seed_job_type(manifest=manifest_1) manifest_2 = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-b', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': {'files': [{'name': 'INPUT_B', 'multiple': True}]}, 'outputs': { 'files': [{'name': 'OUTPUT_B', 'pattern': '*.png'}] } } } } job_type_2 = job_test_utils.create_seed_job_type(manifest=manifest_2) output_dict = { 'version': '1.0', 'output_data': [{ }] } job_1 = job_test_utils.create_job(job_type=job_type_1, num_exes=1, status='COMPLETED', output=output_dict) job_2 = job_test_utils.create_job(job_type=job_type_2, num_exes=0, status='PENDING', input_file_size=None, input=None) from recipe.definition.definition import RecipeDefinition from recipe.definition.json.definition_v1 import convert_recipe_definition_to_v1_json from recipe.test import utils as recipe_test_utils definition = RecipeDefinition(Interface()) definition.add_job_node('node_a', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('node_b', job_type_2.name, job_type_2.version, job_type_2.revision_num) definition.add_dependency('node_a', 'node_b') definition.add_dependency_input_connection('node_b', 'INPUT_B', 'node_a', 'OUTPUT_A') def_dict = convert_recipe_definition_to_v1_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6(definition=def_dict) recipe_data_dict = {'version': '1.0', 'input_data': [], 'workspace_id': workspace.id} recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_a', job=job_1) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_b', job=job_2) job_1.recipe = recipe job_1.save() job_2.recipe = recipe job_2.save() # Create message message = ProcessJobInput() message.job_id = job_2.id # Execute message result = message.execute() self.assertTrue(result) job_2 = Job.objects.get(id=job_2.id) # Check for queued jobs message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'cancel_jobs')
def test_execute_with_recipe(self): """Tests calling ProcessJobInput.execute() successfully when a job has to get its data from its recipe""" workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0) file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0) manifest_1 = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-a', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': {'files': [], 'json': []}, 'outputs': { 'files': [{'name': 'OUTPUT_A', 'pattern': '*.png'}] } } } } job_type_1 = job_test_utils.create_seed_job_type(manifest=manifest_1) manifest_2 = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-b', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': {'files': [{'name': 'INPUT_B', 'multiple': True}]}, 'outputs': { 'files': [{'name': 'OUTPUT_B', 'pattern': '*.png'}] } } } } job_type_2 = job_test_utils.create_seed_job_type(manifest=manifest_2) output_dict = { 'version': '1.0', 'output_data': [{ 'name': 'OUTPUT_A', 'file_ids': [file_1.id, file_2.id, file_3.id, file_4.id] }] } job_1 = job_test_utils.create_job(job_type=job_type_1, num_exes=1, status='COMPLETED', output=output_dict) job_2 = job_test_utils.create_job(job_type=job_type_2, num_exes=0, status='PENDING', input_file_size=None, input=None) from recipe.definition.definition import RecipeDefinition from recipe.definition.json.definition_v1 import convert_recipe_definition_to_v1_json from recipe.test import utils as recipe_test_utils definition = RecipeDefinition(Interface()) definition.add_job_node('node_a', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('node_b', job_type_2.name, job_type_2.version, job_type_2.revision_num) definition.add_dependency('node_a', 'node_b') definition.add_dependency_input_connection('node_b', 'INPUT_B', 'node_a', 'OUTPUT_A') def_dict = convert_recipe_definition_to_v1_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6(definition=def_dict) recipe_data_dict = {'version': '1.0', 'input_data': [], 'workspace_id': workspace.id} recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_a', job=job_1) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_b', job=job_2) job_1.recipe = recipe job_1.save() job_2.recipe = recipe job_2.save() # Create message message = ProcessJobInput() message.job_id = job_2.id # Execute message result = message.execute() self.assertTrue(result) job_2 = Job.objects.get(id=job_2.id) # Check for queued jobs message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'queued_jobs') self.assertFalse(message.new_messages[0].requeue) # Check job for expected input_file_size self.assertEqual(job_2.input_file_size, 24469.0) # Check job for expected input data self.assertSetEqual(set(job_2.get_input_data().values.keys()), {'INPUT_B'}) self.assertListEqual(job_2.get_input_data().values['INPUT_B'].file_ids, [file_1.id, file_2.id, file_3.id, file_4.id]) # Make sure job input file models are created job_input_files = JobInputFile.objects.filter(job_id=job_2.id) self.assertEqual(len(job_input_files), 4) file_ids = set() for job_input_file in job_input_files: self.assertEqual(job_input_file.job_input, 'INPUT_B') file_ids.add(job_input_file.input_file_id) self.assertSetEqual(file_ids, {file_1.id, file_2.id, file_3.id, file_4.id}) # Test executing message again message_json_dict = message.to_json() message = ProcessJobInput.from_json(message_json_dict) result = message.execute() self.assertTrue(result) # Still should have queued jobs message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'queued_jobs') self.assertFalse(message.new_messages[0].requeue) # Make sure job input file models are unchanged job_input_files = JobInputFile.objects.filter(job_id=job_2.id) self.assertEqual(len(job_input_files), 4)
def test_execute_with_recipe_legacy(self): """Tests calling ProcessJobInput.execute() successfully when a legacy job has to get its data from its recipe""" workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0) file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0) interface_1 = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [], 'output_data': [{ 'name': 'Output 1', 'type': 'files', 'media_type': 'image/png', }] } job_type_1 = job_test_utils.create_job_type(interface=interface_1) interface_2 = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Input 1', 'type': 'files', 'media_types': ['image/png'], }], 'output_data': [{ 'name': 'New Output 1', 'type': 'files', 'media_type': 'image/png', }] } job_type_2 = job_test_utils.create_job_type(interface=interface_2) output_dict = { 'version': '1.0', 'output_data': [{ 'name': 'Output 1', 'file_ids': [file_1.id, file_2.id, file_3.id, file_4.id] }] } job_1 = job_test_utils.create_job(job_type=job_type_1, num_exes=1, status='COMPLETED', output=output_dict) job_2 = job_test_utils.create_job(job_type=job_type_2, num_exes=0, status='PENDING', input_file_size=None, input=None) from recipe.definition.definition import RecipeDefinition from recipe.definition.json.definition_v1 import convert_recipe_definition_to_v1_json from recipe.test import utils as recipe_test_utils definition = RecipeDefinition(Interface()) definition.add_job_node('node_a', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('node_b', job_type_2.name, job_type_2.version, job_type_2.revision_num) definition.add_dependency('node_b', 'node_a') definition.add_dependency_input_connection('node_b', 'Input 1', 'node_a', 'Output 1') def_dict = convert_recipe_definition_to_v1_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type(definition=def_dict) recipe_data_dict = { 'version': '1.0', 'input_data': [], 'workspace_id': workspace.id } recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_a', job=job_1) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='node_b', job=job_2) job_1.recipe = recipe job_1.save() job_2.recipe = recipe job_2.save() # Create message message = ProcessJobInput() message.job_id = job_2.id # Execute message result = message.execute() self.assertTrue(result) job_2 = Job.objects.get(id=job_2.id) # Check for queued jobs message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'queued_jobs') self.assertFalse(message.new_messages[0].requeue) # Check job for expected input_file_size self.assertEqual(job_2.input_file_size, 24469.0) # Check job for expected output workspaces in job input data (legacy) self.assertDictEqual( job_2.input, { 'version': '1.0', 'input_data': [{ 'name': 'Input 1', 'file_ids': [file_1.id, file_2.id, file_3.id, file_4.id] }], 'output_data': [{ 'name': 'New Output 1', 'workspace_id': workspace.id }] }) # Make sure job input file models are created job_input_files = JobInputFile.objects.filter(job_id=job_2.id) self.assertEqual(len(job_input_files), 4) file_ids = set() for job_input_file in job_input_files: self.assertEqual(job_input_file.job_input, 'Input 1') file_ids.add(job_input_file.input_file_id) self.assertSetEqual(file_ids, {file_1.id, file_2.id, file_3.id, file_4.id}) # Test executing message again message_json_dict = message.to_json() message = ProcessJobInput.from_json(message_json_dict) result = message.execute() self.assertTrue(result) # Still should have queued jobs message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'queued_jobs') self.assertFalse(message.new_messages[0].requeue) # Make sure job input file models are unchanged job_input_files = JobInputFile.objects.filter(job_id=job_2.id) self.assertEqual(len(job_input_files), 4)