def test_execute_with_recipe(self): """Tests calling PurgeJobs.execute() successfully with job as part of recipe""" recipe = recipe_test_utils.create_recipe() job_exe = job_test_utils.create_job_exe(status='COMPLETED') job = job_exe.job recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job, save=True) source_file = storage_test_utils.create_file(file_type='SOURCE') trigger = trigger_test_utils.create_trigger_event() PurgeResults.objects.create(source_file_id=source_file.id, trigger_event=trigger) # Add job to message message = PurgeJobs() message._purge_job_ids = [job.id] message.trigger_id = trigger.id message.source_file_id = source_file.id message.status_change = timezone.now() # Execute message result = message.execute() self.assertTrue(result) # Check that a new message to purge source file was created msgs = [msg for msg in message.new_messages if msg.type == 'purge_recipe'] self.assertEqual(len(msgs), 1) for msg in msgs: self.assertEqual(msg.recipe_id, recipe.id)
def test_execute_with_parent_recipe(self): """Tests calling PurgeRecipe.execute() successfully""" # Create recipes recipe = recipe_test_utils.create_recipe() parent_recipe = recipe_test_utils.create_recipe() recipe_test_utils.create_recipe_node(recipe=parent_recipe, node_name='A', sub_recipe=recipe, save=True) # Create message message = create_purge_recipe_message(recipe_id=recipe.id, trigger_id=self.trigger.id, source_file_id=self.file_1.id) # Execute message result = message.execute() self.assertTrue(result) # Test to see that a message to purge the parent recipe was sent msgs = [ msg for msg in message.new_messages if msg.type == 'purge_recipe' ] self.assertEqual(len(msgs), 1) for msg in msgs: self.assertEqual(msg.recipe_id, parent_recipe.id) # Assert models were deleted self.assertEqual(Recipe.objects.filter(id=recipe.id).count(), 0) self.assertEqual(RecipeNode.objects.filter(recipe=recipe).count(), 0)
def test_json(self): """Tests converting a ProcessCondition message to and from JSON""" definition = RecipeDefinition(Interface()) # TODO: once DataFilter is implemented, create a DataFilter object here that accepts the inputs definition.add_condition_node('node_a', Interface(), DataFilter(True)) definition_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type( definition=definition_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) condition = recipe_test_utils.create_recipe_condition(recipe=recipe, save=True) recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_a', condition=condition, save=True) # Create message message = create_process_condition_messages([condition.id])[0] # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = ProcessCondition.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) condition = RecipeCondition.objects.get(id=condition.id) self.assertEqual(len(new_message.new_messages), 1) self.assertEqual(new_message.new_messages[0].type, 'update_recipe') self.assertEqual(new_message.new_messages[0].root_recipe_id, recipe.id) self.assertTrue(condition.is_processed) self.assertIsNotNone(condition.processed) self.assertTrue(condition.is_accepted)
def test_get_nodes_to_create(self): """Tests calling Recipe.get_nodes_to_create()""" job_type = job_test_utils.create_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type() # Create recipe definition = RecipeDefinition(Interface()) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) definition.add_condition_node('B', Interface(), DataFilter(True)) definition.add_condition_node('C', Interface(), DataFilter(True)) definition.add_condition_node('D', Interface(), DataFilter(False)) definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_recipe_node('H', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_dependency('A', 'D') definition.add_dependency('A', 'E') definition.add_dependency('B', 'E') definition.add_dependency('B', 'F') definition.add_dependency('C', 'F') definition.add_dependency('D', 'G') definition.add_dependency('E', 'G') definition.add_dependency('E', 'H') definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) # Nodes A, B, and D already exist job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', save=True) condition_b = recipe_test_utils.create_recipe_condition( is_processed=True, is_accepted=True, save=False) condition_d = recipe_test_utils.create_recipe_condition( is_processed=True, is_accepted=False, save=False) RecipeCondition.objects.bulk_create([condition_b, condition_d]) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', condition=condition_b, save=False) recipe_node_d = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='D', condition=condition_d, save=False) RecipeNode.objects.bulk_create( [recipe_node_a, recipe_node_b, recipe_node_d]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) nodes_to_create = recipe_instance.get_nodes_to_create() self.assertSetEqual(set(nodes_to_create.keys()), {'C', 'E', 'H'})
def test_json(self): """Tests converting a ProcessCondition message to and from JSON""" definition = RecipeDefinition(Interface()) cond_interface_1 = Interface() cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer')) df1 = DataFilter(filter_list=[{ 'name': 'cond_int', 'type': 'integer', 'condition': '==', 'values': [0] }]) definition = RecipeDefinition(cond_interface_1) definition.add_condition_node('node_a', cond_interface_1, df1) definition.add_recipe_input_connection('node_a', 'cond_int', 'cond_int') definition_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_dict) data_1 = Data() data_1.add_value(JsonValue('cond_int', 0)) data_1_dict = convert_data_to_v6_json(data_1).get_dict() recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_1_dict) condition = recipe_test_utils.create_recipe_condition(recipe=recipe, save=True) recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_a', condition=condition, save=True) # Create message message = create_process_condition_messages([condition.id])[0] # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = ProcessCondition.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) condition = RecipeCondition.objects.get(id=condition.id) self.assertEqual(len(new_message.new_messages), 1) self.assertEqual(new_message.new_messages[0].type, 'update_recipe') self.assertEqual(new_message.new_messages[0].root_recipe_id, recipe.id) self.assertTrue(condition.is_processed) self.assertIsNotNone(condition.processed) self.assertTrue(condition.is_accepted)
def setUp(self): django.setup() # self.recipe_type = recipe_test_utils.create_recipe_type_v6(definition=recipe_test_utils.RECIPE_DEFINITION) self.recipe = recipe_test_utils.create_recipe() job_type_1 = job_test_utils.create_seed_job_type() job_1 = job_test_utils.create_job(job_type=job_type_1) job_type_2 = job_test_utils.create_seed_job_type() job_2 = job_test_utils.create_job(job_type=job_type_2) job_type_3 = job_test_utils.create_seed_job_type() job_3 = job_test_utils.create_job(job_type=job_type_3) self.recipe_node1 = recipe_test_utils.create_recipe_node(recipe=self.recipe, node_name='job-1', job=job_1) self.recipe_node2 = recipe_test_utils.create_recipe_node(recipe=self.recipe, node_name='job-2', job=job_2) self.recipe_node3 = recipe_test_utils.create_recipe_node(recipe=self.recipe, node_name='job-3', job=job_3) RecipeNode.objects.bulk_create([self.recipe_node1, self.recipe_node2, self.recipe_node3])
def test_json(self): """Tests coverting a UpdateRecipeMetrics message to and from JSON""" recipe = recipe_test_utils.create_recipe() job_1 = job_test_utils.create_job(status='FAILED') job_2 = job_test_utils.create_job(status='CANCELED') job_3 = job_test_utils.create_job(status='BLOCKED') job_4 = job_test_utils.create_job(status='BLOCKED') job_5 = job_test_utils.create_job(status='COMPLETED') recipe_node_1 = recipe_test_utils.create_recipe_node(recipe=recipe, job=job_1) recipe_node_2 = recipe_test_utils.create_recipe_node(recipe=recipe, job=job_2) recipe_node_3 = recipe_test_utils.create_recipe_node(recipe=recipe, job=job_3) recipe_node_4 = recipe_test_utils.create_recipe_node(recipe=recipe, job=job_4) recipe_node_5 = recipe_test_utils.create_recipe_node(recipe=recipe, job=job_5) RecipeNode.objects.bulk_create([ recipe_node_1, recipe_node_2, recipe_node_3, recipe_node_4, recipe_node_5 ]) # Add recipe to message message = UpdateRecipeMetrics() if message.can_fit_more(): message.add_recipe(recipe.id) # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = UpdateRecipeMetrics.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) recipe = Recipe.objects.get(id=recipe.id) self.assertEqual(recipe.jobs_total, 5) self.assertEqual(recipe.jobs_pending, 0) self.assertEqual(recipe.jobs_blocked, 2) self.assertEqual(recipe.jobs_queued, 0) self.assertEqual(recipe.jobs_running, 0) self.assertEqual(recipe.jobs_failed, 1) self.assertEqual(recipe.jobs_completed, 1) self.assertEqual(recipe.jobs_canceled, 1)
def test_convert_recipe_to_v6_json(self): """Tests calling convert_recipe_to_v6_json() successfully""" job_type_1 = job_test_utils.create_seed_job_type() job_type_2 = job_test_utils.create_seed_job_type() job_type_3 = job_test_utils.create_seed_job_type() job_type_4 = job_test_utils.create_seed_job_type() recipe_type_1 = recipe_test_utils.create_recipe_type_v6() interface = Interface() interface.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface.add_parameter(JsonParameter('json_param_1', 'object')) df1 = DataFilter(filter_list=[{'name': 'file_param_1', 'type': 'media-type', 'condition': '==', 'values': ['image/gif']}, {'name': 'json_param_1', 'type': 'object', 'condition': 'superset of', 'values': [{}]}], all=False) definition = RecipeDefinition(interface) definition.add_job_node('A', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('B', job_type_2.name, job_type_2.version, job_type_2.revision_num) definition.add_job_node('C', job_type_3.name, job_type_3.version, job_type_3.revision_num) definition.add_recipe_node('D', recipe_type_1.name, recipe_type_1.revision_num) definition.add_job_node('E', job_type_4.name, job_type_4.version, job_type_4.revision_num) definition.add_condition_node('F', interface, df1) #False definition.add_job_node('G', job_type_4.name, job_type_4.version, job_type_4.revision_num) definition.add_dependency('A', 'B') definition.add_dependency('A', 'C') definition.add_dependency('B', 'E') definition.add_dependency('C', 'D') definition.add_dependency('A', 'F') definition.add_dependency('F', 'G') definition.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') recipe = recipe_test_utils.create_recipe() job_a = job_test_utils.create_job(job_type=job_type_1, status='COMPLETED', save=False) job_b = job_test_utils.create_job(job_type=job_type_2, status='RUNNING', save=False) job_c = job_test_utils.create_job(job_type=job_type_3, status='COMPLETED', save=False) job_e = job_test_utils.create_job(job_type=job_type_4, status='PENDING', num_exes=0, save=False) Job.objects.bulk_create([job_a, job_b, job_c, job_e]) condition_f = recipe_test_utils.create_recipe_condition(is_processed=True, is_accepted=False, save=True) recipe_d = recipe_test_utils.create_recipe(recipe_type=recipe_type_1) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False) recipe_node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='B', job=job_b, save=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False) recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', sub_recipe=recipe_d, save=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', condition=condition_f, save=False) recipe_nodes = [recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f] recipe_instance = RecipeInstance(definition, recipe, recipe_nodes) json = convert_recipe_to_v6_json(recipe_instance) RecipeInstanceV6(json=json.get_dict(), do_validate=True) # Revalidate self.assertSetEqual(set(json.get_dict()['nodes'].keys()), {'A', 'B', 'C', 'D', 'E', 'F'})
def test_execute_with_sub_recipe(self): """Tests calling PurgeRecipe.execute() successfully""" # Create recipes sub_recipe_type = recipe_test_utils.create_recipe_type() definition = RecipeDefinition(Interface()) definition.add_recipe_node('A', sub_recipe_type.name, sub_recipe_type.revision_num) recipe_a = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_a.jobs_completed = 3 recipe_a.jobs_running = 2 recipe_a.jobs_total = 5 Recipe.objects.bulk_create([recipe_a]) definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_node_a = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='A', sub_recipe=recipe_a, save=False) RecipeNode.objects.bulk_create([recipe_node_a]) # Create message message = create_purge_recipe_message(recipe_id=recipe.id, trigger_id=self.trigger.id, source_file_id=self.file_1.id) # Execute message result = message.execute() self.assertTrue(result) # Test to see that a message to purge the parent recipe was sent msgs = [ msg for msg in message.new_messages if msg.type == 'purge_recipe' ] self.assertEqual(len(msgs), 1) for msg in msgs: self.assertEqual(msg.recipe_id, recipe_node_a.sub_recipe.id)
def test_execute_with_recipe_legacy(self): """Tests calling ProcessRecipeInput.execute() successfully when a legacy sub-recipe has to get its data from its recipe """ workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0) file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0) manifest_a = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-a', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': { 'files': [], 'json': [] }, 'outputs': { 'files': [{ 'name': 'output_a', 'pattern': '*.png' }] } } } } job_type_a = job_test_utils.create_job_type(interface=manifest_a) output_data_a = Data() output_data_a.add_value(FileValue('output_a', [file_1.id])) output_data_a_dict = convert_data_to_v6_json(output_data_a).get_dict() manifest_b = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-b', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': { 'files': [], 'json': [] }, 'outputs': { 'files': [{ 'name': 'output_b', 'pattern': '*.png', 'multiple': True }] } } } } job_type_b = job_test_utils.create_job_type(interface=manifest_b) output_data_b = Data() output_data_b.add_value( FileValue('output_b', [file_2.id, file_3.id, file_4.id])) output_data_b_dict = convert_data_to_v6_json(output_data_b).get_dict() job_a = job_test_utils.create_job(job_type=job_type_a, num_exes=1, status='COMPLETED', output=output_data_a_dict) job_b = job_test_utils.create_job(job_type=job_type_b, num_exes=1, status='COMPLETED', output=output_data_b_dict) sub_recipe_interface_c = Interface() sub_recipe_interface_c.add_parameter( FileParameter('input_a', ['image/png'])) sub_recipe_interface_c.add_parameter( FileParameter('input_b', ['image/png'], multiple=True)) sub_recipe_def_c = RecipeDefinition(sub_recipe_interface_c) sub_recipe_def_dict_c = convert_recipe_definition_to_v1_json( sub_recipe_def_c).get_dict() sub_recipe_type_c = recipe_test_utils.create_recipe_type( definition=sub_recipe_def_dict_c) sub_recipe_c = recipe_test_utils.create_recipe( recipe_type=sub_recipe_type_c) definition = RecipeDefinition(Interface()) definition.add_job_node('node_a', job_type_a.name, job_type_a.version, job_type_a.revision_num) definition.add_job_node('node_b', job_type_b.name, job_type_b.version, job_type_b.revision_num) definition.add_recipe_node('node_c', sub_recipe_type_c.name, sub_recipe_type_c.revision_num) definition.add_dependency('node_c', 'node_a') definition.add_dependency_input_connection('node_c', 'input_a', 'node_a', 'output_a') definition.add_dependency('node_c', 'node_b') definition.add_dependency_input_connection('node_c', 'input_b', 'node_b', 'output_b') def_dict = convert_recipe_definition_to_v6_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type(definition=def_dict) recipe_data_dict = { 'version': '1.0', 'input_data': [], 'workspace_id': workspace.id } recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict) recipe_node_a = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='node_a', job=job_a) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='node_b', job=job_b) recipe_node_c = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='node_c', sub_recipe=sub_recipe_c) RecipeNode.objects.bulk_create( [recipe_node_a, recipe_node_b, recipe_node_c]) job_a.recipe = recipe job_a.save() job_b.recipe = recipe job_b.save() sub_recipe_c.recipe = recipe sub_recipe_c.save() # Create message message = ProcessRecipeInput() message.recipe_id = sub_recipe_c.id # Execute message result = message.execute() self.assertTrue(result) sub_recipe_c = Recipe.objects.get(id=sub_recipe_c.id) # Check for update_recipes message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'update_recipes') # Check sub-recipe for expected input_file_size self.assertEqual(sub_recipe_c.input_file_size, 24469.0) # Check sub-recipe for expected input data self.assertEqual( sub_recipe_c.input['version'], '1.0') # Should be legacy input data with workspace ID self.assertEqual(sub_recipe_c.input['workspace_id'], workspace.id) self.assertSetEqual(set(sub_recipe_c.get_input_data().values.keys()), {'input_a', 'input_b'}) self.assertListEqual( sub_recipe_c.get_input_data().values['input_a'].file_ids, [file_1.id]) self.assertListEqual( sub_recipe_c.get_input_data().values['input_b'].file_ids, [file_2.id, file_3.id, file_4.id]) # Make sure sub-recipe input file models are created input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id) self.assertEqual(len(input_files), 4) file_ids = {input_file.input_file_id for input_file in input_files} self.assertSetEqual(file_ids, {file_1.id, file_2.id, file_3.id, file_4.id}) # Test executing message again message_json_dict = message.to_json() message = ProcessRecipeInput.from_json(message_json_dict) result = message.execute() self.assertTrue(result) # Still should have update_recipes message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'update_recipes') # Make sure recipe input file models are unchanged input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id) self.assertEqual(len(input_files), 4)
def test_execute_with_sub_recipes(self): """Tests calling UpdateBatchMetrics.execute() successfully with sub-recipes""" batch = batch_test_utils.create_batch() recipe_1 = recipe_test_utils.create_recipe(batch=batch) recipe_2 = recipe_test_utils.create_recipe(batch=batch) # Recipe 1 jobs job_1 = job_test_utils.create_job(status='FAILED', save=False) job_2 = job_test_utils.create_job(status='CANCELED', save=False) job_3 = job_test_utils.create_job(status='BLOCKED', save=False) job_4 = job_test_utils.create_job(status='BLOCKED', save=False) job_5 = job_test_utils.create_job(status='COMPLETED', save=False) # Recipe 2 jobs job_6 = job_test_utils.create_job(status='COMPLETED', save=False) job_7 = job_test_utils.create_job(status='COMPLETED', save=False) job_8 = job_test_utils.create_job(status='RUNNING', save=False) job_9 = job_test_utils.create_job(status='QUEUED', save=False) job_10 = job_test_utils.create_job(status='PENDING', save=False) job_11 = job_test_utils.create_job(status='PENDING', save=False) job_12 = job_test_utils.create_job(status='PENDING', save=False) job_13 = job_test_utils.create_job(status='CANCELED', save=False) job_14 = job_test_utils.create_job(status='BLOCKED', save=False) Job.objects.bulk_create([ job_1, job_2, job_3, job_4, job_5, job_6, job_7, job_8, job_9, job_10, job_11, job_12, job_13, job_14 ]) # Recipe 1 sub-recipes sub_recipe_1 = recipe_test_utils.create_recipe(save=False) sub_recipe_1.recipe = recipe_1 sub_recipe_1.jobs_total = 26 sub_recipe_1.jobs_pending = 3 sub_recipe_1.jobs_blocked = 4 sub_recipe_1.jobs_queued = 5 sub_recipe_1.jobs_running = 1 sub_recipe_1.jobs_failed = 2 sub_recipe_1.jobs_completed = 3 sub_recipe_1.jobs_canceled = 8 sub_recipe_1.is_completed = False sub_recipe_2 = recipe_test_utils.create_recipe(save=False) sub_recipe_2.recipe = recipe_1 sub_recipe_2.jobs_total = 30 sub_recipe_2.jobs_completed = 30 sub_recipe_2.is_completed = True # Recipe 2 sub-recipes sub_recipe_3 = recipe_test_utils.create_recipe(save=False) sub_recipe_3.recipe = recipe_2 sub_recipe_3.jobs_total = 21 sub_recipe_3.jobs_pending = 2 sub_recipe_3.jobs_blocked = 5 sub_recipe_3.jobs_queued = 0 sub_recipe_3.jobs_running = 3 sub_recipe_3.jobs_failed = 2 sub_recipe_3.jobs_completed = 8 sub_recipe_3.jobs_canceled = 1 sub_recipe_3.is_completed = False sub_recipe_4 = recipe_test_utils.create_recipe(save=False) sub_recipe_4.recipe = recipe_2 sub_recipe_4.jobs_total = 7 sub_recipe_4.jobs_completed = 7 sub_recipe_4.is_completed = True sub_recipe_5 = recipe_test_utils.create_recipe(save=False) sub_recipe_5.recipe = recipe_2 sub_recipe_5.jobs_total = 12 sub_recipe_5.jobs_completed = 12 sub_recipe_5.is_completed = True Recipe.objects.bulk_create([ sub_recipe_1, sub_recipe_2, sub_recipe_3, sub_recipe_4, sub_recipe_5 ]) # Recipe 1 nodes recipe_node_1 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_1) recipe_node_2 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_2) recipe_node_3 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_3) recipe_node_4 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_4) recipe_node_5 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_5) recipe_node_6 = recipe_test_utils.create_recipe_node( recipe=recipe_1, sub_recipe=sub_recipe_1) recipe_node_7 = recipe_test_utils.create_recipe_node( recipe=recipe_1, sub_recipe=sub_recipe_2) # Recipe 2 nodes recipe_node_8 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_6) recipe_node_9 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_7) recipe_node_10 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_8) recipe_node_11 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_9) recipe_node_12 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_10) recipe_node_13 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_11) recipe_node_14 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_12) recipe_node_15 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_13) recipe_node_16 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_14) recipe_node_17 = recipe_test_utils.create_recipe_node( recipe=recipe_2, sub_recipe=sub_recipe_3) recipe_node_18 = recipe_test_utils.create_recipe_node( recipe=recipe_2, sub_recipe=sub_recipe_4) recipe_node_19 = recipe_test_utils.create_recipe_node( recipe=recipe_2, sub_recipe=sub_recipe_5) RecipeNode.objects.bulk_create([ recipe_node_1, recipe_node_2, recipe_node_3, recipe_node_4, recipe_node_5, recipe_node_6, recipe_node_7, recipe_node_8, recipe_node_9, recipe_node_10, recipe_node_11, recipe_node_12, recipe_node_13, recipe_node_14, recipe_node_15, recipe_node_16, recipe_node_17, recipe_node_18, recipe_node_19 ]) # Generate recipe metrics Recipe.objects.update_recipe_metrics([ sub_recipe_1.id, sub_recipe_2.id, sub_recipe_3.id, sub_recipe_4.id, sub_recipe_5.id ]) Recipe.objects.update_recipe_metrics([recipe_1.id, recipe_2.id]) # Add batch to message message = UpdateBatchMetrics() if message.can_fit_more(): message.add_batch(batch.id) # Execute message result = message.execute() self.assertTrue(result) batch = Batch.objects.get(id=batch.id) self.assertEqual(batch.jobs_total, 110) self.assertEqual(batch.jobs_pending, 8) self.assertEqual(batch.jobs_blocked, 12) self.assertEqual(batch.jobs_queued, 6) self.assertEqual(batch.jobs_running, 5) self.assertEqual(batch.jobs_failed, 5) self.assertEqual(batch.jobs_completed, 63) self.assertEqual(batch.jobs_canceled, 11) self.assertEqual(batch.recipes_total, 7) self.assertEqual(batch.recipes_completed, 3) # Test executing message again message_json_dict = message.to_json() message = UpdateBatchMetrics.from_json(message_json_dict) result = message.execute() self.assertTrue(result)
def test_execute(self): """Tests calling ProcessCondition.execute() successfully""" workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) file_3 = storage_test_utils.create_file(workspace=workspace, file_size=65456.0) file_4 = storage_test_utils.create_file(workspace=workspace, file_size=24564165456.0) manifest_1 = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-a', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': { 'files': [], 'json': [] }, 'outputs': { 'files': [{ 'name': 'OUTPUT_A', 'pattern': '*.png', 'multiple': True }] } } } } job_type_1 = job_test_utils.create_job_type(interface=manifest_1) manifest_2 = { 'seedVersion': '1.0.0', 'job': { 'name': 'job-b', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': '', 'description': '', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': '', 'inputs': { 'files': [] }, 'outputs': { 'files': [{ 'name': 'OUTPUT_B', 'pattern': '*.png', 'multiple': True }] } } } } job_type_2 = job_test_utils.create_job_type(interface=manifest_2) output_1_dict = { 'version': '1.0', 'output_data': [{ 'name': 'OUTPUT_A', 'file_ids': [file_1.id, file_2.id] }] } output_2_dict = { 'version': '1.0', 'output_data': [{ 'name': 'OUTPUT_B', 'file_ids': [file_3.id, file_4.id] }] } cond_interface = Interface() cond_interface.add_parameter( FileParameter('INPUT_C_1', [], multiple=True)) cond_interface.add_parameter( FileParameter('INPUT_C_2', [], multiple=True)) definition = RecipeDefinition(Interface()) definition.add_job_node('node_a', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('node_b', job_type_2.name, job_type_2.version, job_type_2.revision_num) # TODO: once DataFilter is implemented, create a DataFilter object here that accepts the inputs definition.add_condition_node('node_c', cond_interface, DataFilter(True)) definition.add_dependency('node_a', 'node_c') definition.add_dependency('node_b', 'node_c') definition.add_dependency_input_connection('node_c', 'INPUT_C_1', 'node_a', 'OUTPUT_A') definition.add_dependency_input_connection('node_c', 'INPUT_C_2', 'node_b', 'OUTPUT_B') def_dict = convert_recipe_definition_to_v6_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type(definition=def_dict) recipe_data_dict = { 'version': '1.0', 'input_data': [], 'workspace_id': workspace.id } recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=recipe_data_dict) job_1 = job_test_utils.create_job(job_type=job_type_1, num_exes=1, status='COMPLETED', output=output_1_dict, recipe=recipe) job_2 = job_test_utils.create_job(job_type=job_type_2, num_exes=1, status='COMPLETED', output=output_2_dict, recipe=recipe) condition = recipe_test_utils.create_recipe_condition(recipe=recipe, save=True) node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_a', job=job_1, save=False) node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_b', job=job_2, save=False) node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_c', condition=condition, save=False) RecipeNode.objects.bulk_create([node_a, node_b, node_c]) # Create message message = create_process_condition_messages([condition.id])[0] # Execute message result = message.execute() self.assertTrue(result) condition = RecipeCondition.objects.get(id=condition.id) # Check for update_recipe message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'update_recipe') self.assertEqual(message.new_messages[0].root_recipe_id, recipe.id) # Check condition flags self.assertTrue(condition.is_processed) self.assertIsNotNone(condition.processed) self.assertTrue(condition.is_accepted) # Check condition for expected data self.assertSetEqual(set(condition.get_data().values.keys()), {'INPUT_C_1', 'INPUT_C_2'}) self.assertListEqual(condition.get_data().values['INPUT_C_1'].file_ids, [file_1.id, file_2.id]) self.assertListEqual(condition.get_data().values['INPUT_C_2'].file_ids, [file_3.id, file_4.id]) # Test executing message again message_json_dict = message.to_json() message = ProcessCondition.from_json(message_json_dict) result = message.execute() self.assertTrue(result) # Still should have update_recipe message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'update_recipe') self.assertEqual(message.new_messages[0].root_recipe_id, recipe.id)
def test_execute_with_recipe(self): """Tests calling CreateJobs.execute() successfully with a recipe that supersedes another recipe""" from batch.test import utils as batch_test_utils from recipe.models import RecipeNode from recipe.test import utils as recipe_test_utils job_type = job_test_utils.create_seed_job_type() recipe_type = recipe_test_utils.create_recipe_type() superseded_recipe = recipe_test_utils.create_recipe( recipe_type=recipe_type, is_superseded=True) node_name = 'recipe_node' superseded_job = job_test_utils.create_job(job_type=job_type, is_superseded=True) recipe_test_utils.create_recipe_node(recipe=superseded_recipe, node_name=node_name, job=superseded_job, save=True) batch = batch_test_utils.create_batch() recipe = recipe_test_utils.create_recipe( recipe_type=recipe_type, superseded_recipe=superseded_recipe, batch=batch) # Create and execute message message = create_jobs_message_for_recipe(recipe, node_name, job_type.name, job_type.version, job_type.revision_num, process_input=True) result = message.execute() self.assertTrue(result) new_job = Job.objects.get(job_type_id=job_type.id, recipe_id=recipe.id) self.assertEqual(new_job.event_id, recipe.event_id) self.assertEqual(new_job.recipe_id, recipe.id) self.assertEqual(new_job.root_recipe_id, superseded_recipe.id) self.assertEqual(new_job.root_superseded_job_id, superseded_job.id) self.assertEqual(new_job.superseded_job_id, superseded_job.id) self.assertEqual(new_job.batch_id, batch.id) node_count = RecipeNode.objects.filter(recipe_id=recipe.id, node_name=node_name, job_id=new_job.id).count() self.assertEqual(node_count, 1) # Check for process_job_input message (because process_input=True) self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'process_job_input') self.assertEqual(message.new_messages[0].job_id, new_job.id) # Test executing message again message.new_messages = [] result = message.execute() self.assertTrue(result) # Make sure a new job is not created self.assertEqual(Job.objects.filter(recipe_id=recipe.id).count(), 1) self.assertEqual( RecipeNode.objects.filter(recipe_id=recipe.id, node_name=node_name).count(), 1) # Check for same process_job_input message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'process_job_input') self.assertEqual(message.new_messages[0].job_id, new_job.id)
def test_execute(self): """Tests calling UpdateRecipeMetrics.execute() successfully""" recipe_1 = recipe_test_utils.create_recipe() job_1 = job_test_utils.create_job(status='FAILED') job_2 = job_test_utils.create_job(status='CANCELED') job_3 = job_test_utils.create_job(status='BLOCKED') job_4 = job_test_utils.create_job(status='BLOCKED') job_5 = job_test_utils.create_job(status='COMPLETED') recipe_node_1 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_1) recipe_node_2 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_2) recipe_node_3 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_3) recipe_node_4 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_4) recipe_node_5 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_5) RecipeNode.objects.bulk_create([ recipe_node_1, recipe_node_2, recipe_node_3, recipe_node_4, recipe_node_5 ]) batch = batch_test_utils.create_batch() recipe_2 = recipe_test_utils.create_recipe(batch=batch) job_6 = job_test_utils.create_job(status='COMPLETED') job_7 = job_test_utils.create_job(status='COMPLETED') job_8 = job_test_utils.create_job(status='RUNNING') job_9 = job_test_utils.create_job(status='QUEUED') job_10 = job_test_utils.create_job(status='PENDING') job_11 = job_test_utils.create_job(status='PENDING') job_12 = job_test_utils.create_job(status='PENDING') job_13 = job_test_utils.create_job(status='CANCELED') job_14 = job_test_utils.create_job(status='BLOCKED') recipe_node_6 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_6) recipe_node_7 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_7) recipe_node_8 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_8) recipe_node_9 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_9) recipe_node_10 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_10) recipe_node_11 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_11) recipe_node_12 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_12) recipe_node_13 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_13) recipe_node_14 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_14) RecipeNode.objects.bulk_create([ recipe_node_6, recipe_node_7, recipe_node_8, recipe_node_9, recipe_node_10, recipe_node_11, recipe_node_12, recipe_node_13, recipe_node_14 ]) # Add recipes to message message = UpdateRecipeMetrics() if message.can_fit_more(): message.add_recipe(recipe_1.id) if message.can_fit_more(): message.add_recipe(recipe_2.id) # Execute message result = message.execute() self.assertTrue(result) recipe_1 = Recipe.objects.get(id=recipe_1.id) self.assertEqual(recipe_1.jobs_total, 5) self.assertEqual(recipe_1.jobs_pending, 0) self.assertEqual(recipe_1.jobs_blocked, 2) self.assertEqual(recipe_1.jobs_queued, 0) self.assertEqual(recipe_1.jobs_running, 0) self.assertEqual(recipe_1.jobs_failed, 1) self.assertEqual(recipe_1.jobs_completed, 1) self.assertEqual(recipe_1.jobs_canceled, 1) recipe_2 = Recipe.objects.get(id=recipe_2.id) self.assertEqual(recipe_2.jobs_total, 9) self.assertEqual(recipe_2.jobs_pending, 3) self.assertEqual(recipe_2.jobs_blocked, 1) self.assertEqual(recipe_2.jobs_queued, 1) self.assertEqual(recipe_2.jobs_running, 1) self.assertEqual(recipe_2.jobs_failed, 0) self.assertEqual(recipe_2.jobs_completed, 2) self.assertEqual(recipe_2.jobs_canceled, 1) # Make sure message is created to update batch metrics self.assertEqual(len(message.new_messages), 1) msg = message.new_messages[0] self.assertEqual(msg.type, 'update_batch_metrics') self.assertListEqual(msg._batch_ids, [batch.id]) # Test executing message again message_json_dict = message.to_json() message = UpdateRecipeMetrics.from_json(message_json_dict) result = message.execute() self.assertTrue(result) recipe_1 = Recipe.objects.get(id=recipe_1.id) self.assertEqual(recipe_1.jobs_total, 5) self.assertEqual(recipe_1.jobs_pending, 0) self.assertEqual(recipe_1.jobs_blocked, 2) self.assertEqual(recipe_1.jobs_queued, 0) self.assertEqual(recipe_1.jobs_running, 0) self.assertEqual(recipe_1.jobs_failed, 1) self.assertEqual(recipe_1.jobs_completed, 1) self.assertEqual(recipe_1.jobs_canceled, 1) recipe_2 = Recipe.objects.get(id=recipe_2.id) self.assertEqual(recipe_2.jobs_total, 9) self.assertEqual(recipe_2.jobs_pending, 3) self.assertEqual(recipe_2.jobs_blocked, 1) self.assertEqual(recipe_2.jobs_queued, 1) self.assertEqual(recipe_2.jobs_running, 1) self.assertEqual(recipe_2.jobs_failed, 0) self.assertEqual(recipe_2.jobs_completed, 2) self.assertEqual(recipe_2.jobs_canceled, 1) # Make sure message is created to update batch metrics self.assertEqual(len(message.new_messages), 1) msg = message.new_messages[0] self.assertEqual(msg.type, 'update_batch_metrics') self.assertListEqual(msg._batch_ids, [batch.id])
def test_execute(self): """Tests calling UpdateRecipe.execute() successfully""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_a = job_test_utils.create_job(status='COMPLETED', input=data_dict, output=data_dict) recipe_b = recipe_test_utils.create_recipe() job_c = job_test_utils.create_job(status='PENDING') job_d = job_test_utils.create_job(status='BLOCKED') recipe_type_e = recipe_test_utils.create_recipe_type_v6() job_type_f = job_test_utils.create_seed_job_type() job_type_k = job_test_utils.create_seed_job_type() job_g = job_test_utils.create_job(status='FAILED', input=data_dict) job_h = job_test_utils.create_job(status='PENDING') condition_i = recipe_test_utils.create_recipe_condition(save=True) definition = RecipeDefinition(Interface()) definition.add_job_node('node_a', job_a.job_type.name, job_a.job_type.version, job_a.job_type_rev.revision_num) definition.add_recipe_node('node_b', recipe_b.recipe_type.name, recipe_b.recipe_type.revision_num) definition.add_job_node('node_c', job_c.job_type.name, job_c.job_type.version, job_c.job_type_rev.revision_num) definition.add_job_node('node_d', job_d.job_type.name, job_d.job_type.version, job_d.job_type_rev.revision_num) definition.add_recipe_node('node_e', recipe_type_e.name, recipe_type_e.revision_num) definition.add_job_node('node_f', job_type_f.name, job_type_f.version, job_type_f.revision_num) definition.add_job_node('node_g', job_g.job_type.name, job_g.job_type.version, job_g.job_type_rev.revision_num) definition.add_job_node('node_h', job_h.job_type.name, job_h.job_type.version, job_h.job_type_rev.revision_num) definition.add_condition_node('node_i', Interface(), DataFilter()) #True definition.add_condition_node('node_j', Interface(), DataFilter()) #True definition.add_job_node('node_k', job_type_k.name, job_type_k.version, job_type_k.revision_num) definition.add_dependency('node_a', 'node_c') definition.add_dependency('node_a', 'node_e') definition.add_dependency('node_a', 'node_g') definition.add_dependency('node_c', 'node_d') definition.add_dependency('node_e', 'node_f') definition.add_dependency('node_g', 'node_h') definition.add_dependency('node_a', 'node_i') definition.add_dependency('node_a', 'node_j') definition.add_dependency('node_i', 'node_k') definition.add_dependency('node_j', 'node_k') definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict) node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_a', job=job_a, save=False) node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_b', sub_recipe=recipe_b, save=False) node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_c', job=job_c, save=False) node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_d', job=job_d, save=False) node_g = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_g', job=job_g, save=False) node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_h', job=job_h, save=False) node_i = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_i', condition=condition_i, save=False) RecipeNode.objects.bulk_create([node_a, node_b, node_c, node_d, node_g, node_h, node_i]) forced_nodes = ForcedNodes() forced_nodes_e = ForcedNodes() forced_nodes_e.set_all_nodes() forced_nodes.add_subrecipe('node_e', forced_nodes_e) # Create and execute message message = create_update_recipe_message(recipe.id, forced_nodes=forced_nodes) result = message.execute() self.assertTrue(result) self.assertEqual(len(message.new_messages), 8) # Check messages blocked_jobs_msg = None pending_jobs_msg = None create_cond_msg = None create_jobs_msg = None create_recipes_msg = None process_condition_msg = None process_job_input_msg = None process_recipe_input_msg = None for msg in message.new_messages: if msg.type == 'blocked_jobs': blocked_jobs_msg = msg elif msg.type == 'pending_jobs': pending_jobs_msg = msg elif msg.type == 'create_conditions': create_cond_msg = msg elif msg.type == 'create_jobs': create_jobs_msg = msg elif msg.type == 'create_recipes': create_recipes_msg = msg elif msg.type == 'process_condition': process_condition_msg = msg elif msg.type == 'process_job_input': process_job_input_msg = msg elif msg.type == 'process_recipe_input': process_recipe_input_msg = msg self.assertIsNotNone(blocked_jobs_msg) self.assertIsNotNone(pending_jobs_msg) self.assertIsNotNone(create_cond_msg) self.assertIsNotNone(create_jobs_msg) self.assertIsNotNone(create_recipes_msg) self.assertIsNotNone(process_condition_msg) self.assertIsNotNone(process_job_input_msg) self.assertIsNotNone(process_recipe_input_msg) # Check message to change jobs to BLOCKED self.assertListEqual(blocked_jobs_msg._blocked_job_ids, [job_h.id]) # Check message to change jobs to PENDING self.assertListEqual(pending_jobs_msg._pending_job_ids, [job_d.id]) # Check message to create conditions self.assertEqual(create_cond_msg.recipe_id, recipe.id) self.assertEqual(create_cond_msg.root_recipe_id, recipe.root_superseded_recipe_id) condition = Condition('node_j', True) self.assertListEqual(create_cond_msg.conditions, [condition]) # Check message to create jobs self.assertEqual(create_jobs_msg.event_id, recipe.event_id) self.assertEqual(create_jobs_msg.create_jobs_type, RECIPE_TYPE) self.assertEqual(create_jobs_msg.recipe_id, recipe.id) self.assertEqual(create_jobs_msg.root_recipe_id, recipe.root_superseded_recipe_id) self.assertIsNone(create_jobs_msg.superseded_recipe_id) recipe_job = RecipeJob(job_type_f.name, job_type_f.version, job_type_f.revision_num, 'node_f', False) self.assertListEqual(create_jobs_msg.recipe_jobs, [recipe_job]) # Check message to create sub-recipes self.assertEqual(create_recipes_msg.event_id, recipe.event_id) msg_forced_nodes_dict = convert_forced_nodes_to_v6(create_recipes_msg.forced_nodes).get_dict() expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict() self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict) self.assertEqual(create_recipes_msg.create_recipes_type, SUB_RECIPE_TYPE) self.assertEqual(create_recipes_msg.recipe_id, recipe.id) self.assertEqual(create_recipes_msg.root_recipe_id, recipe.root_superseded_recipe_id) self.assertIsNone(create_recipes_msg.superseded_recipe_id) sub = SubRecipe(recipe_type_e.name, recipe_type_e.revision_num, 'node_e', True) self.assertListEqual(create_recipes_msg.sub_recipes, [sub]) # Check message to process condition self.assertEqual(process_condition_msg.condition_id, condition_i.id) # Check message to process job input self.assertEqual(process_job_input_msg.job_id, job_c.id) # Check message to process recipe input self.assertEqual(process_recipe_input_msg.recipe_id, recipe_b.id) # Test executing message again message_json_dict = message.to_json() message = UpdateRecipe.from_json(message_json_dict) result = message.execute() self.assertTrue(result) # Make sure the same messages are returned self.assertEqual(len(message.new_messages), 8) blocked_jobs_msg = None pending_jobs_msg = None create_cond_msg = None create_jobs_msg = None create_recipes_msg = None process_condition_msg = None process_job_input_msg = None process_recipe_input_msg = None for msg in message.new_messages: if msg.type == 'blocked_jobs': blocked_jobs_msg = msg elif msg.type == 'pending_jobs': pending_jobs_msg = msg elif msg.type == 'create_conditions': create_cond_msg = msg elif msg.type == 'create_jobs': create_jobs_msg = msg elif msg.type == 'create_recipes': create_recipes_msg = msg elif msg.type == 'process_condition': process_condition_msg = msg elif msg.type == 'process_job_input': process_job_input_msg = msg elif msg.type == 'process_recipe_input': process_recipe_input_msg = msg self.assertIsNotNone(blocked_jobs_msg) self.assertIsNotNone(pending_jobs_msg) self.assertIsNotNone(create_cond_msg) self.assertIsNotNone(create_jobs_msg) self.assertIsNotNone(create_recipes_msg) self.assertIsNotNone(process_condition_msg) self.assertIsNotNone(process_job_input_msg) self.assertIsNotNone(process_recipe_input_msg) # Check message to change jobs to BLOCKED self.assertListEqual(blocked_jobs_msg._blocked_job_ids, [job_h.id]) # Check message to change jobs to PENDING self.assertListEqual(pending_jobs_msg._pending_job_ids, [job_d.id]) # Check message to create conditions self.assertEqual(create_cond_msg.recipe_id, recipe.id) self.assertEqual(create_cond_msg.root_recipe_id, recipe.root_superseded_recipe_id) condition = Condition('node_j', True) self.assertListEqual(create_cond_msg.conditions, [condition]) # Check message to create jobs self.assertEqual(create_jobs_msg.event_id, recipe.event_id) self.assertEqual(create_jobs_msg.create_jobs_type, RECIPE_TYPE) self.assertEqual(create_jobs_msg.recipe_id, recipe.id) self.assertEqual(create_jobs_msg.root_recipe_id, recipe.root_superseded_recipe_id) self.assertIsNone(create_jobs_msg.superseded_recipe_id) recipe_job = RecipeJob(job_type_f.name, job_type_f.version, job_type_f.revision_num, 'node_f', False) self.assertListEqual(create_jobs_msg.recipe_jobs, [recipe_job]) # Check message to create sub-recipes self.assertEqual(create_recipes_msg.event_id, recipe.event_id) msg_forced_nodes_dict = convert_forced_nodes_to_v6(create_recipes_msg.forced_nodes).get_dict() expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict() self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict) self.assertEqual(create_recipes_msg.create_recipes_type, SUB_RECIPE_TYPE) self.assertEqual(create_recipes_msg.recipe_id, recipe.id) self.assertEqual(create_recipes_msg.root_recipe_id, recipe.root_superseded_recipe_id) self.assertIsNone(create_recipes_msg.superseded_recipe_id) sub = SubRecipe(recipe_type_e.name, recipe_type_e.revision_num, 'node_e', True) self.assertListEqual(create_recipes_msg.sub_recipes, [sub]) # Check message to process condition self.assertEqual(process_condition_msg.condition_id, condition_i.id) # Check message to process job input self.assertEqual(process_job_input_msg.job_id, job_c.id) # Check message to process recipe input self.assertEqual(process_recipe_input_msg.recipe_id, recipe_b.id)
def test_has_completed_false(self): """Tests calling Recipe.has_completed() when an entire recipe has not completed""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() definition = RecipeDefinition(Interface()) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('B', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('C', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('D', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('H', job_type.name, job_type.version, job_type.revision_num) definition.add_dependency('A', 'C') definition.add_dependency('A', 'E') definition.add_dependency('A', 'H') definition.add_dependency('C', 'D') definition.add_dependency('G', 'H') job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_c = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_d = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_e = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_f = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_h = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h]) recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_b.is_completed = True recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_g.is_completed = False Recipe.objects.bulk_create([recipe_b, recipe_g]) definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False, is_original=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False, is_original=False) recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', job=job_d, save=False, is_original=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', job=job_f, save=False) recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='H', job=job_h, save=False) recipe_node_g = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='G', sub_recipe=recipe_g, save=False, is_original=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False) RecipeNode.objects.bulk_create([ recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h ]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) self.assertFalse(recipe_instance.has_completed())
def test_get_jobs_to_update(self): """Tests calling Recipe.get_jobs_to_update()""" job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() definition = RecipeDefinition(Interface()) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('B', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('C', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('D', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('H', job_type.name, job_type.version, job_type.revision_num) definition.add_dependency('A', 'C') definition.add_dependency('A', 'E') definition.add_dependency('A', 'H') definition.add_dependency('B', 'E') definition.add_dependency('B', 'G') definition.add_dependency('C', 'D') definition.add_dependency('E', 'F') definition.add_dependency('G', 'H') job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', save=False) job_c = job_test_utils.create_job(job_type=job_type, status='CANCELED', num_exes=0, save=False) job_d = job_test_utils.create_job(job_type=job_type, status='PENDING', num_exes=0, save=False) job_e = job_test_utils.create_job(job_type=job_type, status='BLOCKED', num_exes=0, save=False) job_f = job_test_utils.create_job(job_type=job_type, status='PENDING', num_exes=0, save=False) job_h = job_test_utils.create_job(job_type=job_type, status='PENDING', num_exes=0, save=False) Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h]) recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_b.jobs_completed = 3 recipe_b.jobs_running = 2 recipe_b.jobs_total = 5 recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_g.jobs_completed = 2 recipe_g.jobs_failed = 1 recipe_g.jobs_total = 3 Recipe.objects.bulk_create([recipe_b, recipe_g]) definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False) recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', job=job_d, save=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', job=job_f, save=False) recipe_node_g = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='G', sub_recipe=recipe_g, save=False) recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='H', job=job_h, save=False) RecipeNode.objects.bulk_create([ recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h ]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) results = recipe_instance.get_jobs_to_update() self.assertSetEqual(set(results['BLOCKED']), {job_d.id, job_h.id}) self.assertSetEqual(set(results['PENDING']), {job_e.id})
def test_convert_recipe_to_v6_json(self): """Tests calling convert_recipe_to_v6_json() successfully""" job_type_1 = job_test_utils.create_job_type() job_type_2 = job_test_utils.create_job_type() job_type_3 = job_test_utils.create_job_type() job_type_4 = job_test_utils.create_job_type() recipe_type_1 = recipe_test_utils.create_recipe_type() interface = Interface() interface.add_parameter(FileParameter('file_param_1', ['image/gif'])) interface.add_parameter(JsonParameter('json_param_1', 'object')) definition = RecipeDefinition(interface) definition.add_job_node('A', job_type_1.name, job_type_1.version, job_type_1.revision_num) definition.add_job_node('B', job_type_2.name, job_type_2.version, job_type_2.revision_num) definition.add_job_node('C', job_type_3.name, job_type_3.version, job_type_3.revision_num) definition.add_recipe_node('D', recipe_type_1.name, recipe_type_1.revision_num) definition.add_job_node('E', job_type_4.name, job_type_4.version, job_type_4.revision_num) definition.add_dependency('A', 'B') definition.add_dependency('A', 'C') definition.add_dependency('B', 'E') definition.add_dependency('C', 'D') definition.add_recipe_input_connection('A', 'input_1', 'file_param_1') definition.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1') definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2') definition.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1') definition.add_recipe_input_connection('D', 'd_input_2', 'json_param_1') recipe = recipe_test_utils.create_recipe() job_a = job_test_utils.create_job(job_type=job_type_1, status='COMPLETED', save=False) job_b = job_test_utils.create_job(job_type=job_type_2, status='RUNNING', save=False) job_c = job_test_utils.create_job(job_type=job_type_3, status='COMPLETED', save=False) job_e = job_test_utils.create_job(job_type=job_type_4, status='PENDING', num_exes=0, save=False) Job.objects.bulk_create([job_a, job_b, job_c, job_e]) recipe_d = recipe_test_utils.create_recipe(recipe_type=recipe_type_1) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False) recipe_node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='B', job=job_b, save=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False) recipe_node_d = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='D', sub_recipe=recipe_d, save=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_nodes = [ recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e ] recipe_instance = RecipeInstance(definition, recipe_nodes) json = convert_recipe_to_v6_json(recipe_instance) RecipeInstanceV6(json=json.get_dict(), do_validate=True) # Revalidate self.assertSetEqual(set(json.get_dict()['nodes'].keys()), {'A', 'B', 'C', 'D', 'E'})
def test_execute_recipe_superseded(self): """Tests calling CreateJobs.execute() successfully for jobs within a recipe that supersedes another recipe""" from batch.test import utils as batch_test_utils from recipe.models import RecipeNode from recipe.test import utils as recipe_test_utils batch = batch_test_utils.create_batch() event = trigger_test_utils.create_trigger_event() job_type_1 = job_test_utils.create_seed_job_type() job_type_2 = job_test_utils.create_seed_job_type() superseded_recipe = recipe_test_utils.create_recipe() superseded_job_1 = job_test_utils.create_job(job_type=job_type_1, is_superseded=True) superseded_job_2 = job_test_utils.create_job(job_type=job_type_2, is_superseded=True) recipe_test_utils.create_recipe_node(recipe=superseded_recipe, node_name='node_1', job=superseded_job_1, save=True) recipe_test_utils.create_recipe_node(recipe=superseded_recipe, node_name='node_2', job=superseded_job_2, save=True) recipe = recipe_test_utils.create_recipe( recipe_type=superseded_recipe.recipe_type, superseded_recipe=superseded_recipe, event=event, batch=batch) recipe_jobs = [ RecipeJob(job_type_1.name, job_type_1.version, job_type_1.revision_num, 'node_1', False), RecipeJob(job_type_2.name, job_type_2.version, job_type_2.revision_num, 'node_2', True) ] # Create and execute message message = create_jobs_messages_for_recipe(recipe, recipe_jobs)[0] result = message.execute() self.assertTrue(result) self.assertEqual( Job.objects.filter(recipe_id=recipe.id, event_id=recipe.event_id).count(), 2) recipe_nodes = RecipeNode.objects.select_related('job').filter( recipe_id=recipe.id) self.assertEqual(len(recipe_nodes), 2) for recipe_node in recipe_nodes: if recipe_node.node_name == 'node_1': job_1 = recipe_node.job self.assertEqual(job_1.job_type_id, job_type_1.id) self.assertEqual(job_1.event_id, event.id) self.assertEqual(job_1.batch_id, batch.id) self.assertEqual(job_1.recipe_id, recipe.id) self.assertEqual(job_1.root_recipe_id, superseded_recipe.id) self.assertEqual(job_1.superseded_job_id, superseded_job_1.id) self.assertEqual(job_1.root_superseded_job_id, superseded_job_1.id) elif recipe_node.node_name == 'node_2': job_2 = recipe_node.job self.assertEqual(job_2.job_type_id, job_type_2.id) self.assertEqual(job_2.event_id, event.id) self.assertEqual(job_2.batch_id, batch.id) self.assertEqual(job_2.recipe_id, recipe.id) self.assertEqual(job_2.root_recipe_id, superseded_recipe.id) self.assertEqual(job_2.superseded_job_id, superseded_job_2.id) self.assertEqual(job_2.root_superseded_job_id, superseded_job_2.id) else: self.fail('%s is the wrong node name' % recipe_node.node_name) # Should be two messages, one for processing job input and one for updating metrics for the recipe self.assertEqual(len(message.new_messages), 2) process_job_input_msg = None update_metrics_msg = None for msg in message.new_messages: if msg.type == 'process_job_input': process_job_input_msg = msg elif msg.type == 'update_recipe_metrics': update_metrics_msg = msg self.assertIsNotNone(process_job_input_msg) self.assertIsNotNone(update_metrics_msg) # Check message to process job input for new job 2 self.assertEqual(process_job_input_msg.job_id, job_2.id) # Check message to update recipe metrics for the recipe containing the new jobs self.assertListEqual(update_metrics_msg._recipe_ids, [recipe.id]) # Test executing message again message_json_dict = message.to_json() message = CreateJobs.from_json(message_json_dict) result = message.execute() self.assertTrue(result) self.assertEqual( Job.objects.filter(recipe_id=recipe.id, event_id=recipe.event_id).count(), 2) recipe_nodes = RecipeNode.objects.select_related('job').filter( recipe_id=recipe.id) self.assertEqual(len(recipe_nodes), 2) for recipe_node in recipe_nodes: if recipe_node.node_name == 'node_1': job_1 = recipe_node.job self.assertEqual(job_1.job_type_id, job_type_1.id) self.assertEqual(job_1.event_id, event.id) self.assertEqual(job_1.batch_id, batch.id) self.assertEqual(job_1.recipe_id, recipe.id) self.assertEqual(job_1.root_recipe_id, superseded_recipe.id) self.assertEqual(job_1.superseded_job_id, superseded_job_1.id) self.assertEqual(job_1.root_superseded_job_id, superseded_job_1.id) elif recipe_node.node_name == 'node_2': job_2 = recipe_node.job self.assertEqual(job_2.job_type_id, job_type_2.id) self.assertEqual(job_2.event_id, event.id) self.assertEqual(job_2.batch_id, batch.id) self.assertEqual(job_2.recipe_id, recipe.id) self.assertEqual(job_2.root_recipe_id, superseded_recipe.id) self.assertEqual(job_2.superseded_job_id, superseded_job_2.id) self.assertEqual(job_2.root_superseded_job_id, superseded_job_2.id) else: self.fail('%s is the wrong node name' % recipe_node.node_name) # Should be two messages, one for processing job input and one for updating metrics for the recipe self.assertEqual(len(message.new_messages), 2) process_job_input_msg = None update_metrics_msg = None for msg in message.new_messages: if msg.type == 'process_job_input': process_job_input_msg = msg elif msg.type == 'update_recipe_metrics': update_metrics_msg = msg self.assertIsNotNone(process_job_input_msg) self.assertIsNotNone(update_metrics_msg) # Check message to process job input for new job 2 self.assertEqual(process_job_input_msg.job_id, job_2.id) # Check message to update recipe metrics for the recipe containing the new jobs self.assertListEqual(update_metrics_msg._recipe_ids, [recipe.id])
def test_get_nodes_to_process_input(self): """Tests calling Recipe.get_nodes_to_process_input()""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() # Create recipe definition = RecipeDefinition(Interface()) cond_interface_1 = Interface() cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer')) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) df1 = DataFilter(filter_list=[{ 'name': 'cond_int', 'type': 'integer', 'condition': '==', 'values': [0] }, { 'name': 'cond_int', 'type': 'integer', 'condition': '!=', 'values': [0] }], all=False) #always True df2 = DataFilter(filter_list=[{ 'name': 'cond_int', 'type': 'integer', 'condition': '==', 'values': [0] }, { 'name': 'cond_int', 'type': 'integer', 'condition': '!=', 'values': [0] }], all=True) #always False definition.add_condition_node('B', cond_interface_1, df1) #True definition.add_condition_node('C', cond_interface_1, df1) #True definition.add_condition_node('D', cond_interface_1, df2) #False definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_recipe_node('H', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_dependency('A', 'D') definition.add_dependency('A', 'E') definition.add_dependency('B', 'E') definition.add_dependency('B', 'F') definition.add_dependency('C', 'F') definition.add_dependency('D', 'G') definition.add_dependency('E', 'G') definition.add_dependency('E', 'H') definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict) # Nodes A, B, and D already exist job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', input=data_dict, output=data_dict, save=True) condition_b = recipe_test_utils.create_recipe_condition( is_processed=True, is_accepted=True, save=False) condition_d = recipe_test_utils.create_recipe_condition( is_processed=True, is_accepted=False, save=False) RecipeCondition.objects.bulk_create([condition_b, condition_d]) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', condition=condition_b, save=False) recipe_node_d = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='D', condition=condition_d, save=False) RecipeNode.objects.bulk_create( [recipe_node_a, recipe_node_b, recipe_node_d]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) nodes_to_process = recipe_instance.get_nodes_to_process_input() self.assertSetEqual(set(nodes_to_process.keys()), {'C', 'E'})
def test_execute_with_sub_recipes(self): """Tests calling UpdateRecipeMetrics.execute() successfully with sub-recipes""" recipe_1 = recipe_test_utils.create_recipe() batch = batch_test_utils.create_batch() recipe_2 = recipe_test_utils.create_recipe(batch=batch) # Recipe 1 jobs job_1 = job_test_utils.create_job(status='FAILED', save=False) job_2 = job_test_utils.create_job(status='CANCELED', save=False) job_3 = job_test_utils.create_job(status='BLOCKED', save=False) job_4 = job_test_utils.create_job(status='BLOCKED', save=False) job_5 = job_test_utils.create_job(status='COMPLETED', save=False) # Recipe 2 jobs job_6 = job_test_utils.create_job(status='COMPLETED', save=False) job_7 = job_test_utils.create_job(status='COMPLETED', save=False) job_8 = job_test_utils.create_job(status='RUNNING', save=False) job_9 = job_test_utils.create_job(status='QUEUED', save=False) job_10 = job_test_utils.create_job(status='PENDING', save=False) job_11 = job_test_utils.create_job(status='PENDING', save=False) job_12 = job_test_utils.create_job(status='PENDING', save=False) job_13 = job_test_utils.create_job(status='CANCELED', save=False) job_14 = job_test_utils.create_job(status='BLOCKED', save=False) Job.objects.bulk_create([ job_1, job_2, job_3, job_4, job_5, job_6, job_7, job_8, job_9, job_10, job_11, job_12, job_13, job_14 ]) # Recipe 1 sub-recipes sub_recipe_1 = recipe_test_utils.create_recipe(save=False) sub_recipe_1.jobs_total = 26 sub_recipe_1.jobs_pending = 3 sub_recipe_1.jobs_blocked = 4 sub_recipe_1.jobs_queued = 5 sub_recipe_1.jobs_running = 1 sub_recipe_1.jobs_failed = 2 sub_recipe_1.jobs_completed = 3 sub_recipe_1.jobs_canceled = 8 sub_recipe_1.is_completed = False sub_recipe_2 = recipe_test_utils.create_recipe(save=False) sub_recipe_2.jobs_total = 30 sub_recipe_2.jobs_completed = 30 sub_recipe_2.is_completed = True # Recipe 2 sub-recipes sub_recipe_3 = recipe_test_utils.create_recipe(save=False) sub_recipe_3.jobs_total = 21 sub_recipe_3.jobs_pending = 2 sub_recipe_3.jobs_blocked = 5 sub_recipe_3.jobs_queued = 0 sub_recipe_3.jobs_running = 3 sub_recipe_3.jobs_failed = 2 sub_recipe_3.jobs_completed = 8 sub_recipe_3.jobs_canceled = 1 sub_recipe_3.is_completed = False sub_recipe_4 = recipe_test_utils.create_recipe(save=False) sub_recipe_4.jobs_total = 7 sub_recipe_4.jobs_completed = 7 sub_recipe_4.is_completed = True sub_recipe_5 = recipe_test_utils.create_recipe(save=False) sub_recipe_5.jobs_total = 12 sub_recipe_5.jobs_completed = 12 sub_recipe_5.is_completed = True Recipe.objects.bulk_create([ sub_recipe_1, sub_recipe_2, sub_recipe_3, sub_recipe_4, sub_recipe_5 ]) # Recipe 1 nodes recipe_node_1 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_1) recipe_node_2 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_2) recipe_node_3 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_3) recipe_node_4 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_4) recipe_node_5 = recipe_test_utils.create_recipe_node(recipe=recipe_1, job=job_5) recipe_node_6 = recipe_test_utils.create_recipe_node( recipe=recipe_1, sub_recipe=sub_recipe_1) recipe_node_7 = recipe_test_utils.create_recipe_node( recipe=recipe_1, sub_recipe=sub_recipe_2) # Recipe 2 nodes recipe_node_8 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_6) recipe_node_9 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_7) recipe_node_10 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_8) recipe_node_11 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_9) recipe_node_12 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_10) recipe_node_13 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_11) recipe_node_14 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_12) recipe_node_15 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_13) recipe_node_16 = recipe_test_utils.create_recipe_node(recipe=recipe_2, job=job_14) recipe_node_17 = recipe_test_utils.create_recipe_node( recipe=recipe_2, sub_recipe=sub_recipe_3) recipe_node_18 = recipe_test_utils.create_recipe_node( recipe=recipe_2, sub_recipe=sub_recipe_4) recipe_node_19 = recipe_test_utils.create_recipe_node( recipe=recipe_2, sub_recipe=sub_recipe_5) RecipeNode.objects.bulk_create([ recipe_node_1, recipe_node_2, recipe_node_3, recipe_node_4, recipe_node_5, recipe_node_6, recipe_node_7, recipe_node_8, recipe_node_9, recipe_node_10, recipe_node_11, recipe_node_12, recipe_node_13, recipe_node_14, recipe_node_15, recipe_node_16, recipe_node_17, recipe_node_18, recipe_node_19 ]) # Add recipes to message message = UpdateRecipeMetrics() if message.can_fit_more(): message.add_recipe(recipe_1.id) if message.can_fit_more(): message.add_recipe(recipe_2.id) # Execute message result = message.execute() self.assertTrue(result) recipe_1 = Recipe.objects.get(id=recipe_1.id) self.assertEqual(recipe_1.jobs_total, 61) self.assertEqual(recipe_1.jobs_pending, 3) self.assertEqual(recipe_1.jobs_blocked, 6) self.assertEqual(recipe_1.jobs_queued, 5) self.assertEqual(recipe_1.jobs_running, 1) self.assertEqual(recipe_1.jobs_failed, 3) self.assertEqual(recipe_1.jobs_completed, 34) self.assertEqual(recipe_1.jobs_canceled, 9) self.assertEqual(recipe_1.sub_recipes_total, 2) self.assertEqual(recipe_1.sub_recipes_completed, 1) recipe_2 = Recipe.objects.get(id=recipe_2.id) self.assertEqual(recipe_2.jobs_total, 49) self.assertEqual(recipe_2.jobs_pending, 5) self.assertEqual(recipe_2.jobs_blocked, 6) self.assertEqual(recipe_2.jobs_queued, 1) self.assertEqual(recipe_2.jobs_running, 4) self.assertEqual(recipe_2.jobs_failed, 2) self.assertEqual(recipe_2.jobs_completed, 29) self.assertEqual(recipe_2.jobs_canceled, 2) self.assertEqual(recipe_2.sub_recipes_total, 3) self.assertEqual(recipe_2.sub_recipes_completed, 2) # Make sure message is created to update batch metrics self.assertEqual(len(message.new_messages), 1) msg = message.new_messages[0] self.assertEqual(msg.type, 'update_batch_metrics') self.assertListEqual(msg._batch_ids, [batch.id]) # Test executing message again message_json_dict = message.to_json() message = UpdateRecipeMetrics.from_json(message_json_dict) result = message.execute() self.assertTrue(result) recipe_1 = Recipe.objects.get(id=recipe_1.id) self.assertEqual(recipe_1.jobs_total, 61) self.assertEqual(recipe_1.jobs_pending, 3) self.assertEqual(recipe_1.jobs_blocked, 6) self.assertEqual(recipe_1.jobs_queued, 5) self.assertEqual(recipe_1.jobs_running, 1) self.assertEqual(recipe_1.jobs_failed, 3) self.assertEqual(recipe_1.jobs_completed, 34) self.assertEqual(recipe_1.jobs_canceled, 9) self.assertEqual(recipe_1.sub_recipes_total, 2) self.assertEqual(recipe_1.sub_recipes_completed, 1) recipe_2 = Recipe.objects.get(id=recipe_2.id) self.assertEqual(recipe_2.jobs_total, 49) self.assertEqual(recipe_2.jobs_pending, 5) self.assertEqual(recipe_2.jobs_blocked, 6) self.assertEqual(recipe_2.jobs_queued, 1) self.assertEqual(recipe_2.jobs_running, 4) self.assertEqual(recipe_2.jobs_failed, 2) self.assertEqual(recipe_2.jobs_completed, 29) self.assertEqual(recipe_2.jobs_canceled, 2) self.assertEqual(recipe_2.sub_recipes_total, 3) self.assertEqual(recipe_2.sub_recipes_completed, 2) # Make sure message is created to update batch metrics self.assertEqual(len(message.new_messages), 1) msg = message.new_messages[0] self.assertEqual(msg.type, 'update_batch_metrics') self.assertListEqual(msg._batch_ids, [batch.id])
def test_get_original_leaf_nodes(self): """Tests calling Recipe.get_original_leaf_nodes()""" job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() definition = RecipeDefinition(Interface()) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('B', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('C', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('D', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('H', job_type.name, job_type.version, job_type.revision_num) definition.add_dependency('A', 'C') definition.add_dependency('A', 'E') definition.add_dependency('A', 'H') definition.add_dependency('C', 'D') definition.add_dependency('G', 'H') job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', save=False, is_superseded=True) job_c = job_test_utils.create_job(job_type=job_type, status='CANCELED', num_exes=0, save=False) job_d = job_test_utils.create_job(job_type=job_type, status='PENDING', num_exes=0, save=False) job_e = job_test_utils.create_job(job_type=job_type, status='BLOCKED', num_exes=0, save=False) job_f = job_test_utils.create_job(job_type=job_type, status='PENDING', num_exes=0, save=False) job_h = job_test_utils.create_job(job_type=job_type, status='PENDING', num_exes=0, save=False) Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h]) recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_b.jobs_completed = 3 recipe_b.jobs_running = 2 recipe_b.jobs_total = 5 recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_g.jobs_completed = 2 recipe_g.jobs_failed = 1 recipe_g.jobs_total = 3 Recipe.objects.bulk_create([recipe_b, recipe_g]) definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False, is_original=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False, is_original=False) recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', job=job_d, save=False, is_original=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', job=job_f, save=False) recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='H', job=job_h, save=False) recipe_node_g = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='G', sub_recipe=recipe_g, save=False, is_original=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False) RecipeNode.objects.bulk_create([ recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h ]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) results = recipe_instance.get_original_leaf_nodes() self.assertEqual(len(results.values()), 4) leaf_jobs = [ node.job.id for node in results.values() if node.node_type == JobNodeDefinition.NODE_TYPE ] leaf_recipes = [ node.recipe.id for node in results.values() if node.node_type == RecipeNodeDefinition.NODE_TYPE ] self.assertItemsEqual( leaf_jobs, [recipe_node_e.job.id, recipe_node_f.job.id, recipe_node_h.job.id]) self.assertItemsEqual(leaf_recipes, [recipe_node_b.sub_recipe.id])
def test_execute_with_top_level_recipe(self): """Tests calling UpdateRecipeMetrics.execute() successfully where a message needs to be sent to update a top-level recipe """ batch = batch_test_utils.create_batch() top_recipe = recipe_test_utils.create_recipe(batch=batch) recipe = recipe_test_utils.create_recipe(batch=batch) recipe.recipe = top_recipe recipe.save() recipe_node_1 = recipe_test_utils.create_recipe_node(recipe=top_recipe, sub_recipe=recipe) # Recipe jobs job_1 = job_test_utils.create_job(status='FAILED', save=False) job_2 = job_test_utils.create_job(status='CANCELED', save=False) job_3 = job_test_utils.create_job(status='BLOCKED', save=False) job_4 = job_test_utils.create_job(status='BLOCKED', save=False) job_5 = job_test_utils.create_job(status='COMPLETED', save=False) Job.objects.bulk_create([job_1, job_2, job_3, job_4, job_5]) # Recipe nodes recipe_node_2 = recipe_test_utils.create_recipe_node(recipe=recipe, job=job_1) recipe_node_3 = recipe_test_utils.create_recipe_node(recipe=recipe, job=job_2) recipe_node_4 = recipe_test_utils.create_recipe_node(recipe=recipe, job=job_3) recipe_node_5 = recipe_test_utils.create_recipe_node(recipe=recipe, job=job_4) recipe_node_6 = recipe_test_utils.create_recipe_node(recipe=recipe, job=job_5) RecipeNode.objects.bulk_create([ recipe_node_1, recipe_node_2, recipe_node_3, recipe_node_4, recipe_node_5, recipe_node_6 ]) # Add recipes to message message = UpdateRecipeMetrics() if message.can_fit_more(): message.add_recipe(recipe.id) # Execute message result = message.execute() self.assertTrue(result) recipe = Recipe.objects.get(id=recipe.id) self.assertEqual(recipe.jobs_total, 5) self.assertEqual(recipe.jobs_pending, 0) self.assertEqual(recipe.jobs_blocked, 2) self.assertEqual(recipe.jobs_queued, 0) self.assertEqual(recipe.jobs_running, 0) self.assertEqual(recipe.jobs_failed, 1) self.assertEqual(recipe.jobs_completed, 1) self.assertEqual(recipe.jobs_canceled, 1) self.assertEqual(recipe.sub_recipes_total, 0) self.assertEqual(recipe.sub_recipes_completed, 0) # Make sure message is created to update top-level recipe metrics # There should be no message to update batch metrics since we did not update a top-level recipe self.assertEqual(len(message.new_messages), 1) msg = message.new_messages[0] self.assertEqual(msg.type, 'update_recipe_metrics') self.assertListEqual(msg._recipe_ids, [top_recipe.id]) # Test executing message again message_json_dict = message.to_json() message = UpdateRecipeMetrics.from_json(message_json_dict) result = message.execute() self.assertTrue(result) recipe = Recipe.objects.get(id=recipe.id) self.assertEqual(recipe.jobs_total, 5) self.assertEqual(recipe.jobs_pending, 0) self.assertEqual(recipe.jobs_blocked, 2) self.assertEqual(recipe.jobs_queued, 0) self.assertEqual(recipe.jobs_running, 0) self.assertEqual(recipe.jobs_failed, 1) self.assertEqual(recipe.jobs_completed, 1) self.assertEqual(recipe.jobs_canceled, 1) self.assertEqual(recipe.sub_recipes_total, 0) self.assertEqual(recipe.sub_recipes_completed, 0) # Make sure message is created to update top-level recipe metrics # There should be no message to update batch metrics since we did not update a top-level recipe self.assertEqual(len(message.new_messages), 1) msg = message.new_messages[0] self.assertEqual(msg.type, 'update_recipe_metrics') self.assertListEqual(msg._recipe_ids, [top_recipe.id])
def test_has_completed_true(self): """Tests calling Recipe.has_completed() when an entire recipe has completed""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() cond_interface_1 = Interface() cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer')) df2 = DataFilter(filter_list=[{ 'name': 'cond_int', 'type': 'integer', 'condition': '==', 'values': [0] }, { 'name': 'cond_int', 'type': 'integer', 'condition': '!=', 'values': [0] }], all=True) #always False definition = RecipeDefinition(Interface()) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('B', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('C', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('D', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('H', job_type.name, job_type.version, job_type.revision_num) definition.add_condition_node('I', cond_interface_1, df2) #False definition.add_job_node('J', job_type.name, job_type.version, job_type.revision_num) definition.add_dependency('A', 'C') definition.add_dependency('A', 'E') definition.add_dependency('A', 'H') definition.add_dependency('C', 'D') definition.add_dependency('G', 'H') definition.add_dependency('A', 'I') definition.add_dependency('I', 'J') job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_c = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_d = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_e = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_f = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_h = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h]) condition_i = recipe_test_utils.create_recipe_condition( is_processed=True, is_accepted=False, save=True) recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_b.is_completed = True recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_g.is_completed = True Recipe.objects.bulk_create([recipe_b, recipe_g]) definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False, is_original=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False, is_original=False) recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', job=job_d, save=False, is_original=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', job=job_f, save=False) recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='H', job=job_h, save=False) recipe_node_i = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='I', condition=condition_i, save=False) recipe_node_g = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='G', sub_recipe=recipe_g, save=False, is_original=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False) RecipeNode.objects.bulk_create([ recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h, recipe_node_i ]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) self.assertTrue(recipe_instance.has_completed())
def test_execute_subrecipes_superseded(self): """Tests calling CreateRecipes.execute() successfully when creating sub-recipes that supersede other sub-recipes """ # Creates definitions for sub-recipe A and sub-recipe B batch = batch_test_utils.create_batch() event = trigger_test_utils.create_trigger_event() top_recipe_type = recipe_test_utils.create_recipe_type() job_type_a_1 = job_test_utils.create_seed_job_type() job_type_a_2 = job_test_utils.create_seed_job_type() sub_definition_a = RecipeDefinition(Interface()) sub_definition_a.add_job_node('node_1', job_type_a_1.name, job_type_a_1.version, job_type_a_1.revision_num) sub_definition_a.add_job_node('node_2', job_type_a_2.name, job_type_a_2.version, job_type_a_2.revision_num) sub_definition_a.add_dependency('node_1', 'node_2') sub_definition_a_dict = convert_recipe_definition_to_v6_json( sub_definition_a).get_dict() recipe_type_a = recipe_test_utils.create_recipe_type( definition=sub_definition_a_dict) job_type_b_x = job_test_utils.create_seed_job_type() job_type_b_y = job_test_utils.create_seed_job_type() recipe_type_b_z = recipe_test_utils.create_recipe_type() sub_definition_b = RecipeDefinition(Interface()) sub_definition_b.add_job_node('node_x', job_type_b_x.name, job_type_b_x.version, job_type_b_x.revision_num) sub_definition_b.add_job_node('node_y', job_type_b_y.name, job_type_b_y.version, job_type_b_y.revision_num) sub_definition_b.add_recipe_node('node_z', recipe_type_b_z.name, recipe_type_b_z.revision_num) sub_definition_b.add_dependency('node_x', 'node_z') sub_definition_b.add_dependency('node_y', 'node_z') sub_definition_b_dict = convert_recipe_definition_to_v6_json( sub_definition_b).get_dict() recipe_type_b = recipe_test_utils.create_recipe_type( definition=sub_definition_b_dict) # Create previous recipe containing sub-recipe A and B in order to be superseded prev_recipe_a = recipe_test_utils.create_recipe( recipe_type=recipe_type_a, save=False) prev_job_a_1 = job_test_utils.create_job(job_type=job_type_a_1, save=False) prev_job_a_2 = job_test_utils.create_job(job_type=job_type_a_2, save=False) prev_recipe_b = recipe_test_utils.create_recipe( recipe_type=recipe_type_b, save=False) prev_job_b_x = job_test_utils.create_job(job_type=job_type_b_x, save=False) prev_job_b_y = job_test_utils.create_job(job_type=job_type_b_y, save=False) prev_recipe_b_z = recipe_test_utils.create_recipe( recipe_type=recipe_type_b_z, save=False) prev_top_recipe = recipe_test_utils.create_recipe( recipe_type=top_recipe_type, save=False) new_top_recipe = recipe_test_utils.create_recipe( recipe_type=top_recipe_type, save=False) Job.objects.bulk_create( [prev_job_a_1, prev_job_a_2, prev_job_b_x, prev_job_b_y]) Recipe.objects.bulk_create([ prev_recipe_a, prev_recipe_b, prev_recipe_b_z, prev_top_recipe, new_top_recipe ]) recipe_node_a = recipe_test_utils.create_recipe_node( recipe=prev_top_recipe, sub_recipe=prev_recipe_a, node_name='node_a', save=False) recipe_node_a_1 = recipe_test_utils.create_recipe_node( recipe=prev_recipe_a, job=prev_job_a_1, node_name='node_1', save=False) recipe_node_a_2 = recipe_test_utils.create_recipe_node( recipe=prev_recipe_a, job=prev_job_a_2, node_name='node_2', save=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=prev_top_recipe, sub_recipe=prev_recipe_b, node_name='node_b', save=False) recipe_node_b_x = recipe_test_utils.create_recipe_node( recipe=prev_recipe_b, job=prev_job_b_x, node_name='node_x', save=False) recipe_node_b_y = recipe_test_utils.create_recipe_node( recipe=prev_recipe_b, job=prev_job_b_y, node_name='node_y', save=False) recipe_node_b_z = recipe_test_utils.create_recipe_node( recipe=prev_recipe_b, sub_recipe=prev_recipe_b_z, node_name='node_z', save=False) RecipeNode.objects.bulk_create([ recipe_node_a, recipe_node_a_1, recipe_node_a_2, recipe_node_b, recipe_node_b_x, recipe_node_b_y, recipe_node_b_z ]) # Create message to create sub-recipes A and B for new_top_recipe which supersedes prev_top_recipe sub_recipes = [ SubRecipe(recipe_type_a.name, recipe_type_a.revision_num, 'node_a', True), SubRecipe(recipe_type_b.name, recipe_type_b.revision_num, 'node_b', False) ] forced_nodes = ForcedNodes() sub_forced_nodes_b = ForcedNodes() sub_forced_nodes_y = ForcedNodes() sub_forced_nodes_b.add_subrecipe('node_y', sub_forced_nodes_y) forced_nodes.add_subrecipe('node_b', sub_forced_nodes_b) message = create_subrecipes_messages( new_top_recipe.id, new_top_recipe.root_superseded_recipe_id, sub_recipes, event.id, superseded_recipe_id=prev_top_recipe.id, forced_nodes=forced_nodes, batch_id=batch.id)[0] # Execute message result = message.execute() self.assertTrue(result) # Check for new sub-recipes qry = RecipeNode.objects.select_related('sub_recipe') recipe_nodes = qry.filter( recipe_id=new_top_recipe.id).order_by('node_name') self.assertEqual(len(recipe_nodes), 2) self.assertEqual(recipe_nodes[0].node_name, 'node_a') self.assertEqual(recipe_nodes[1].node_name, 'node_b') sub_recipe_a = recipe_nodes[0].sub_recipe sub_recipe_b = recipe_nodes[1].sub_recipe self.assertEqual(sub_recipe_a.recipe_type_id, recipe_type_a.id) self.assertEqual(sub_recipe_a.superseded_recipe_id, prev_recipe_a.id) self.assertEqual(sub_recipe_a.root_superseded_recipe_id, prev_recipe_a.id) self.assertEqual(sub_recipe_b.recipe_type_id, recipe_type_b.id) self.assertEqual(sub_recipe_b.superseded_recipe_id, prev_recipe_b.id) self.assertEqual(sub_recipe_b.root_superseded_recipe_id, prev_recipe_b.id) # Check for sub-recipes to contain correct copied nodes # Nodes 1 and 2 in sub-recipe A should be copied recipe_nodes = RecipeNode.objects.select_related('job').filter( recipe_id=sub_recipe_a.id).order_by('node_name') self.assertEqual(len(recipe_nodes), 2) self.assertEqual(recipe_nodes[0].node_name, 'node_1') self.assertFalse(recipe_nodes[0].is_original) self.assertEqual(recipe_nodes[0].job_id, prev_job_a_1.id) self.assertEqual(recipe_nodes[1].node_name, 'node_2') self.assertFalse(recipe_nodes[1].is_original) self.assertEqual(recipe_nodes[1].job_id, prev_job_a_2.id) # Node X in sub-recipe B should be copied recipe_nodes = RecipeNode.objects.select_related('sub_recipe').filter( recipe_id=sub_recipe_b.id) self.assertEqual(len(recipe_nodes), 1) self.assertEqual(recipe_nodes[0].node_name, 'node_x') self.assertFalse(recipe_nodes[0].is_original) # Should be four messages, two for superseding recipe nodes, one for processing recipe input, and one for # updating metrics for the recipe containing the new sub-recipes self.assertEqual(len(message.new_messages), 4) supersede_recipe_a_msg = None supersede_recipe_b_msg = None process_recipe_input_msg = None update_metrics_msg = None for msg in message.new_messages: if msg.type == 'supersede_recipe_nodes': if msg._recipe_ids[0] == prev_recipe_a.id: supersede_recipe_a_msg = msg if msg._recipe_ids[0] == prev_recipe_b.id: supersede_recipe_b_msg = msg elif msg.type == 'process_recipe_input': process_recipe_input_msg = msg elif msg.type == 'update_recipe_metrics': update_metrics_msg = msg self.assertIsNotNone(supersede_recipe_a_msg) self.assertIsNotNone(supersede_recipe_b_msg) self.assertIsNotNone(process_recipe_input_msg) self.assertIsNotNone(update_metrics_msg) # Check message for superseding previous sub-recipe A self.assertFalse(supersede_recipe_a_msg.supersede_all) self.assertSetEqual(supersede_recipe_a_msg.supersede_jobs, set()) self.assertSetEqual(supersede_recipe_a_msg.supersede_subrecipes, set()) self.assertFalse(supersede_recipe_a_msg.unpublish_all) self.assertSetEqual(supersede_recipe_a_msg.unpublish_jobs, set()) self.assertFalse(supersede_recipe_a_msg.supersede_recursive_all) self.assertSetEqual(supersede_recipe_a_msg.supersede_recursive, set()) self.assertFalse(supersede_recipe_a_msg.unpublish_recursive_all) self.assertSetEqual(supersede_recipe_a_msg.unpublish_recursive, set()) # Check message for superseding previous sub-recipe B self.assertFalse(supersede_recipe_b_msg.supersede_all) self.assertSetEqual(supersede_recipe_b_msg.supersede_jobs, {'node_y'}) self.assertSetEqual(supersede_recipe_b_msg.supersede_subrecipes, {'node_z'}) self.assertFalse(supersede_recipe_b_msg.unpublish_all) self.assertSetEqual(supersede_recipe_b_msg.unpublish_jobs, set()) self.assertFalse(supersede_recipe_b_msg.supersede_recursive_all) self.assertSetEqual(supersede_recipe_b_msg.supersede_recursive, {'node_z'}) self.assertFalse(supersede_recipe_b_msg.unpublish_recursive_all) self.assertSetEqual(supersede_recipe_b_msg.unpublish_recursive, set()) # Check message to process recipe input for new sub-recipe A self.assertEqual(process_recipe_input_msg.recipe_id, sub_recipe_a.id) # Check message to update recipe metrics for the recipe containing the new sub-recipes self.assertListEqual(update_metrics_msg._recipe_ids, [new_top_recipe.id]) # Test executing message again message_json_dict = message.to_json() message = CreateRecipes.from_json(message_json_dict) result = message.execute() self.assertTrue(result) # Check sub-recipes to make sure we didn't create them again a second time qry = RecipeNode.objects.select_related('sub_recipe') recipe_nodes = qry.filter( recipe_id=new_top_recipe.id).order_by('node_name') self.assertEqual(len(recipe_nodes), 2) self.assertEqual(recipe_nodes[0].node_name, 'node_a') self.assertEqual(recipe_nodes[1].node_name, 'node_b') sub_recipe_a = recipe_nodes[0].sub_recipe sub_recipe_b = recipe_nodes[1].sub_recipe self.assertEqual(sub_recipe_a.recipe_type_id, recipe_type_a.id) self.assertEqual(sub_recipe_a.superseded_recipe_id, prev_recipe_a.id) self.assertEqual(sub_recipe_a.root_superseded_recipe_id, prev_recipe_a.id) self.assertEqual(sub_recipe_b.recipe_type_id, recipe_type_b.id) self.assertEqual(sub_recipe_b.superseded_recipe_id, prev_recipe_b.id) self.assertEqual(sub_recipe_b.root_superseded_recipe_id, prev_recipe_b.id) # Check for sub-recipes to contain correct copied nodes # Nodes 1 and 2 in sub-recipe A should be copied recipe_nodes = RecipeNode.objects.select_related('job').filter( recipe_id=sub_recipe_a.id).order_by('node_name') self.assertEqual(len(recipe_nodes), 2) self.assertEqual(recipe_nodes[0].node_name, 'node_1') self.assertFalse(recipe_nodes[0].is_original) self.assertEqual(recipe_nodes[0].job_id, prev_job_a_1.id) self.assertEqual(recipe_nodes[1].node_name, 'node_2') self.assertFalse(recipe_nodes[1].is_original) self.assertEqual(recipe_nodes[1].job_id, prev_job_a_2.id) # Node X in sub-recipe B should be copied recipe_nodes = RecipeNode.objects.select_related('sub_recipe').filter( recipe_id=sub_recipe_b.id) self.assertEqual(len(recipe_nodes), 1) self.assertEqual(recipe_nodes[0].node_name, 'node_x') self.assertFalse(recipe_nodes[0].is_original) # Check messages again # Should be four messages, two for superseding recipe nodes, one for processing recipe input, and one for # updating metrics for the recipe containing the new sub-recipes self.assertEqual(len(message.new_messages), 4) supersede_recipe_a_msg = None supersede_recipe_b_msg = None process_recipe_input_msg = None update_metrics_msg = None for msg in message.new_messages: if msg.type == 'supersede_recipe_nodes': if msg._recipe_ids[0] == prev_recipe_a.id: supersede_recipe_a_msg = msg if msg._recipe_ids[0] == prev_recipe_b.id: supersede_recipe_b_msg = msg elif msg.type == 'process_recipe_input': process_recipe_input_msg = msg elif msg.type == 'update_recipe_metrics': update_metrics_msg = msg self.assertIsNotNone(supersede_recipe_a_msg) self.assertIsNotNone(supersede_recipe_b_msg) self.assertIsNotNone(process_recipe_input_msg) self.assertIsNotNone(update_metrics_msg) # Check message for superseding previous sub-recipe A self.assertFalse(supersede_recipe_a_msg.supersede_all) self.assertSetEqual(supersede_recipe_a_msg.supersede_jobs, set()) self.assertSetEqual(supersede_recipe_a_msg.supersede_subrecipes, set()) self.assertFalse(supersede_recipe_a_msg.unpublish_all) self.assertSetEqual(supersede_recipe_a_msg.unpublish_jobs, set()) self.assertFalse(supersede_recipe_a_msg.supersede_recursive_all) self.assertSetEqual(supersede_recipe_a_msg.supersede_recursive, set()) self.assertFalse(supersede_recipe_a_msg.unpublish_recursive_all) self.assertSetEqual(supersede_recipe_a_msg.unpublish_recursive, set()) # Check message for superseding previous sub-recipe B self.assertFalse(supersede_recipe_b_msg.supersede_all) self.assertSetEqual(supersede_recipe_b_msg.supersede_jobs, {'node_y'}) self.assertSetEqual(supersede_recipe_b_msg.supersede_subrecipes, {'node_z'}) self.assertFalse(supersede_recipe_b_msg.unpublish_all) self.assertSetEqual(supersede_recipe_b_msg.unpublish_jobs, set()) self.assertFalse(supersede_recipe_b_msg.supersede_recursive_all) self.assertSetEqual(supersede_recipe_b_msg.supersede_recursive, {'node_z'}) self.assertFalse(supersede_recipe_b_msg.unpublish_recursive_all) self.assertSetEqual(supersede_recipe_b_msg.unpublish_recursive, set()) # Check message to process recipe input for new sub-recipe A self.assertEqual(process_recipe_input_msg.recipe_id, sub_recipe_a.id) # Check message to update recipe metrics for the recipe containing the new sub-recipes self.assertListEqual(update_metrics_msg._recipe_ids, [new_top_recipe.id])