def test_uuid_use_properties(self): """Tests setting UUIDs on products with different property values.""" inputs_json=[ {'name': 'property1', 'type': 'string'}, {'name': 'property2', 'type': 'string'} ] manifest = job_test_utils.create_seed_manifest(name='test-job', inputs_json=inputs_json, command='my_command') manifest['job']['interface']['inputs']['files'] = [] job_type = job_test_utils.create_seed_job_type(manifest=manifest) job1 = job_test_utils.create_job(job_type=job_type) job_exe1 = job_test_utils.create_job_exe(job=job1) data1 = job_exe1.job.get_input_data() data1.add_value(JsonValue('property1', 'value1')) data1.add_value(JsonValue('property2', 'value2')) job_exe1.job.input = convert_data_to_v6_json(data1).get_dict() job2 = job_test_utils.create_job(job_type=job_type) job_exe2 = job_test_utils.create_job_exe(job=job2) data2 = job_exe2.job.get_input_data() data2.add_value(JsonValue('property1', 'diffvalue1')) data2.add_value(JsonValue('property2', 'value2')) job_exe2.job.input = convert_data_to_v6_json(data2).get_dict() products1 = ProductFile.objects.upload_files(self.files, [self.source_file.id], job_exe1, self.workspace) products2 = ProductFile.objects.upload_files(self.files, [self.source_file.id], job_exe2, self.workspace) # Make sure the product files have different UUIDs self.assertIsNotNone(products1[0].uuid) self.assertIsNotNone(products1[1].uuid) self.assertNotEqual(products1[0].uuid, products2[0].uuid) self.assertNotEqual(products1[1].uuid, products2[1].uuid)
def process_manual_ingested_source_file(self, ingest_id, source_file, when, recipe_type_id): """Processes a manual ingest where a strike or scan is not involved. All database changes are made in an atomic transaction :param ingest_id: :type ingest_id: int :param source_file: The source file that was ingested :type source_file: :class:`source.models.SourceFile` :param when: When the source file was ingested :type when: :class:`datetime.datetime` :param recipe_type_id: id of the Recipe type to kick off :type recipe_type_id: int """ recipe_type = RecipeType.objects.get(id=recipe_type_id) if recipe_type and recipe_type.is_active: recipe_data = Data() input_name = recipe_type.get_definition().get_input_keys()[0] recipe_data.add_value(FileValue(input_name, [source_file.id])) event = self._create_trigger_event(None, source_file, when) ingest_event = self._create_ingest_event(ingest_id, None, source_file, when) messages = create_recipes_messages( recipe_type.name, recipe_type.revision_num, convert_data_to_v6_json(recipe_data).get_dict(), event.id, ingest_event.id) CommandMessageManager().send_messages(messages) else: logger.info( 'No recipe type found for id %s or recipe type is inactive' % recipe_type_id)
def test_json_forced_nodes(self): """Tests coverting a ProcessRecipeInput message to and from JSON with forced nodes provided""" data_dict = convert_data_to_v6_json(Data()).get_dict() recipe = recipe_test_utils.create_recipe(input=data_dict) forced_nodes = ForcedNodes() forced_nodes.set_all_nodes() forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict() # Create message message = create_process_recipe_input_messages( [recipe.id], forced_nodes=forced_nodes)[0] # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = ProcessRecipeInput.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) recipe = Recipe.objects.get(id=recipe.id) self.assertEqual(len(new_message.new_messages), 1) msg = new_message.new_messages[0] self.assertEqual(msg.type, 'update_recipe') self.assertEqual(msg.root_recipe_id, recipe.id) self.assertDictEqual( convert_forced_nodes_to_v6(msg.forced_nodes).get_dict(), forced_nodes_dict) # Recipe should have input_file_size set to 0 (no input files) self.assertEqual(recipe.input_file_size, 0.0)
def test_json(self): """Tests converting an UpdateRecipe message to and from JSON""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_failed = job_test_utils.create_job(status='FAILED', input=data_dict) job_pending = job_test_utils.create_job(status='PENDING') definition = RecipeDefinition(Interface()) definition.add_job_node('job_failed', job_failed.job_type.name, job_failed.job_type.version, job_failed.job_type_rev.revision_num) definition.add_job_node('job_pending', job_pending.job_type.name, job_pending.job_type.version, job_pending.job_type_rev.revision_num) definition.add_dependency('job_failed', 'job_pending') definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_failed', job=job_failed) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_pending', job=job_pending) # Create message message = create_update_recipe_message(recipe.id) # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = UpdateRecipe.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) # Check for message to set job_pending to BLOCKED self.assertEqual(len(new_message.new_messages), 1) msg = new_message.new_messages[0] self.assertEqual(msg.type, 'blocked_jobs') self.assertListEqual(msg._blocked_job_ids, [job_pending.id])
def _find_existing_jobs(self): """Searches to determine if this message already ran and the jobs already exist :returns: The list of job models found :rtype: :func:`list` """ if self.create_jobs_type == INPUT_DATA_TYPE: jobs = Job.objects.filter(job_type__name=self.job_type_name, job_type__version=self.job_type_version, job_type_rev__revision_num=self.job_type_rev_num, event_id=self.event_id, input=convert_data_to_v6_json(self.input_data).get_dict()) elif self.create_jobs_type == RECIPE_TYPE: from recipe.models import RecipeNode node_names = [recipe_job.node_name for recipe_job in self.recipe_jobs] qry = RecipeNode.objects.select_related('job') qry = qry.filter(recipe_id=self.recipe_id, node_name__in=node_names, job__event_id=self.event_id) jobs_by_node = {recipe_node.node_name: recipe_node.job for recipe_node in qry} jobs = jobs_by_node.values() if jobs_by_node: # Set up process input dict for recipe_job in self.recipe_jobs: job = jobs_by_node[recipe_job.node_name] self._process_input[job.id] = recipe_job.process_input return jobs
def create_dataset_members(self, dataset, data_list): """Creates a dataset member :param dataset: The dataset the member is a part of :type dataset: :class:`data.models.DataSet` :param data_list: Data definitions of the dataset members :type data_list: [:class:`data.data.data.Data`] """ with transaction.atomic(): dataset_members = [] datasetfiles = [] existing_scale_ids = DataSetFile.objects.get_file_ids( dataset_ids=[dataset.id]) for d in data_list: dataset_member = DataSetMember() dataset_member.dataset = dataset dataset_member.data = convert_data_to_v6_json(d).get_dict() dataset_member.file_ids = list(data_util.get_file_ids(d)) dataset_members.append(dataset_member) datasetfiles.extend( DataSetFile.objects.create_dataset_files( dataset, d, existing_scale_ids)) existing_scale_ids.append(dataset_member.file_ids) DataSetFile.objects.bulk_create(datasetfiles) return DataSetMember.objects.bulk_create(dataset_members)
def convert_definition_to_v6_json(definition): """Returns the v6 dataset definition JSON for the given definition :param definition: The dataset definition :type definition: :class:`data.dataset.dataset.DataSetDefinition` :returns: The v6 dataset definition JSON :rtype: :class:`data.dataset.json.dataset_v6.DataSetDefinitionV6` """ def_dict = { 'version': SCHEMA_VERSION } if definition.parameters: interface_dict = convert_interface_to_v6_json(definition.parameters).get_dict() def_dict['parameters'] = rest_utils.strip_schema_version(interface_dict) if definition.global_parameters: interface_dict = convert_interface_to_v6_json(definition.global_parameters).get_dict() def_dict['global_parameters'] = rest_utils.strip_schema_version(interface_dict) if definition.global_data: data_dict = convert_data_to_v6_json(definition.global_data).get_dict() def_dict['global_data'] = rest_utils.strip_schema_version(data_dict) return DataSetDefinitionV6(definition=def_dict, do_validate=False)
def to_json(self): """See :meth:`messaging.messages.message.CommandMessage.to_json` """ json_dict = {'event_id': self.event_id, 'create_jobs_type': self.create_jobs_type} if self.create_jobs_type == INPUT_DATA_TYPE: json_dict['job_type_name'] = self.job_type_name json_dict['job_type_version'] = self.job_type_version json_dict['job_type_rev_num'] = self.job_type_rev_num json_dict['input_data'] = convert_data_to_v6_json(self.input_data).get_dict() elif self.create_jobs_type == RECIPE_TYPE: json_dict['recipe_id'] = self.recipe_id if self.root_recipe_id: json_dict['root_recipe_id'] = self.root_recipe_id if self.superseded_recipe_id: json_dict['superseded_recipe_id'] = self.superseded_recipe_id if self.batch_id: json_dict['batch_id'] = self.batch_id recipe_jobs = [] for recipe_job in self.recipe_jobs: recipe_jobs.append({'job_type_name': recipe_job.job_type_name, 'job_type_version': recipe_job.job_type_version, 'job_type_rev_num': recipe_job.job_type_rev_num, 'node_name': recipe_job.node_name, 'process_input': recipe_job.process_input}) json_dict['recipe_jobs'] = recipe_jobs json_dict['recipe_config'] = self.recipe_config if self.ingest_event_id: json_dict['ingest_event_id'] = self.ingest_event_id return json_dict
def execute(self): """See :meth:`messaging.messages.message.CommandMessage.execute` """ condition = RecipeCondition.objects.get_condition_with_interfaces( self.condition_id) if not condition.is_processed: definition = condition.recipe.recipe_type_rev.get_definition() # Get condition data from dependencies in the recipe recipe_input_data = condition.recipe.get_input_data() node_outputs = RecipeNode.objects.get_recipe_node_outputs( condition.recipe_id) for node_output in node_outputs.values(): if node_output.node_type == 'condition' and node_output.id == condition.id: node_name = node_output.node_name break # Set data on the condition model try: data = definition.generate_node_input_data( node_name, recipe_input_data, node_outputs) RecipeCondition.objects.set_condition_data_v6( condition, data, node_name) except InvalidData: logger.exception( 'Recipe %d created invalid input data for condition %d. Message will not re-run.', condition.recipe_id, self.condition_id) return True # Process filter and set whether condition was accepted data_filter = definition.graph[node_name].data_filter is_accepted = data_filter.is_data_accepted(data) RecipeCondition.objects.set_processed(condition.id, is_accepted) # Log results filter_str = json.dumps(data_filter.filter_list, sort_keys=True, indent=4, separators=(',', ': ')) data_str = json.dumps(convert_data_to_v6_json(data).get_dict(), sort_keys=True, indent=4, separators=(',', ': ')) logger.info( 'Condition %d (recipe %d at %s) evaluated to %s:\nCondition: %s\nInput Data: %s', condition.id, condition.recipe_id, node_name, is_accepted, filter_str, data_str) # Create message to update the condition's recipe from recipe.messages.update_recipe import create_update_recipe_message root_recipe_id = condition.recipe.root_recipe_id if condition.recipe.root_recipe_id else condition.recipe_id logger.info( 'Processed data for condition %d, sending message to update recipe %d', self.condition_id, root_recipe_id) self.new_messages.append(create_update_recipe_message(root_recipe_id)) return True
def get_dict(self): """Returns the internal dictionary that represents these job results :returns: The dictionary representing the results :rtype: dict """ return convert_data_to_v6_json(self._results_data).get_dict()
def test_convert_data_to_v6_json(self): """Tests calling convert_data_to_v6_json()""" # Try interface with nothing set data = Data() json = convert_data_to_v6_json(data) DataV6(data=json.get_dict(), do_validate=True) # Revalidate # Try data with a variety of values data = Data() data.add_value(FileValue('input_a', [1234])) data.add_value(FileValue('input_b', [1235, 1236])) data.add_value(JsonValue('input_c', 'hello')) data.add_value(JsonValue('input_d', 11.9)) json = convert_data_to_v6_json(data) DataV6(data=json.get_dict(), do_validate=True) # Revalidate self.assertSetEqual(set(json.get_data().values.keys()), {'input_a', 'input_b', 'input_c', 'input_d'})
def get_dict(self): """Returns the internal dictionary that represents this job data :returns: The internal dictionary :rtype: dict """ return convert_data_to_v6_json(self._new_data).get_dict()
def get_v6_data_json(self): """Returns the data for this datasetmember as v6 json with the version stripped :returns: The v6 JSON output data dict for this datasetmember :rtype: dict """ return rest_utils.strip_schema_version( convert_data_to_v6_json(self.get_data()).get_dict())
def test_json(self): """Tests converting a ProcessCondition message to and from JSON""" definition = RecipeDefinition(Interface()) cond_interface_1 = Interface() cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer')) df1 = DataFilter(filter_list=[{ 'name': 'cond_int', 'type': 'integer', 'condition': '==', 'values': [0] }]) definition = RecipeDefinition(cond_interface_1) definition.add_condition_node('node_a', cond_interface_1, df1) definition.add_recipe_input_connection('node_a', 'cond_int', 'cond_int') definition_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_dict) data_1 = Data() data_1.add_value(JsonValue('cond_int', 0)) data_1_dict = convert_data_to_v6_json(data_1).get_dict() recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_1_dict) condition = recipe_test_utils.create_recipe_condition(recipe=recipe, save=True) recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_a', condition=condition, save=True) # Create message message = create_process_condition_messages([condition.id])[0] # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = ProcessCondition.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) condition = RecipeCondition.objects.get(id=condition.id) self.assertEqual(len(new_message.new_messages), 1) self.assertEqual(new_message.new_messages[0].type, 'update_recipe') self.assertEqual(new_message.new_messages[0].root_recipe_id, recipe.id) self.assertTrue(condition.is_processed) self.assertIsNotNone(condition.processed) self.assertTrue(condition.is_accepted)
def setUp(self): django.setup() def upload_files(file_uploads): for file_upload in file_uploads: file_upload.file.save() def delete_files(files): for scale_file in files: scale_file.save() self.workspace = storage_test_utils.create_workspace() self.workspace.upload_files = MagicMock(side_effect=upload_files) self.workspace.delete_files = MagicMock(side_effect=delete_files) self.source_file = source_test_utils.create_source(file_name='input1.txt', workspace=self.workspace) inputs_json=[ {'name': 'property1', 'type': 'string'}, {'name': 'property2', 'type': 'string'} ] manifest = job_test_utils.create_seed_manifest(inputs_json=inputs_json, command='my_command') manifest['job']['interface']['inputs']['files'] = [] job_type = job_test_utils.create_seed_job_type(manifest=manifest) self.job_exe = job_test_utils.create_job_exe(job_type=job_type) data = self.job_exe.job.get_input_data() data.add_value(JsonValue('property1', 'value1')) data.add_value(JsonValue('property2', 'value2')) self.job_exe.job.input = convert_data_to_v6_json(data).get_dict() self.job_exe.job.source_sensor_class = 'classA' self.job_exe.job.source_sensor = '1' self.job_exe.job.source_collection = '12345' self.job_exe.job.source_task = 'my-task' self.job_exe.job.save() self.job_exe_no = job_test_utils.create_job_exe() self.local_path_1 = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'local/1/file.txt') self.local_path_2 = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'local/2/file.json') self.local_path_3 = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'local/3/file.h5') self.files = [ ProductFileMetadata(output_name='output_name_1', local_path=self.local_path_1, remote_path='remote/1/file.txt'), ProductFileMetadata(output_name='output_name_2', local_path=self.local_path_2, media_type='application/x-custom-json', remote_path='remote/2/file.json', source_sensor_class='classB', source_sensor='2', source_collection='12346', source_task='my-task-2'), ] self.files_no = [ ProductFileMetadata(output_name='output_name_3', local_path=self.local_path_3, media_type='image/x-hdf5-image', remote_path='remote/3/file.h5') ]
def process_ingested_source_file_cm(self, ingest_id, source, source_file, when): """Processes the given ingested source file by kicking off its recipe. All database changes are made in an atomic transaction. :param source: The strike that triggered the ingest :type scan: `object` :param source_file: The source file that was ingested :type source_file: :class:`source.models.SourceFile` :param when: When the source file was ingested :type when: :class:`datetime.datetime` """ # Create the recipe handler associated with the ingest strike/scan source_recipe_config = source.configuration['recipe'] recipe_name = source_recipe_config['name'] recipe_revision = source_recipe_config[ 'revision_num'] if 'revision_num' in source_recipe_config else None recipe_type = RecipeType.objects.get(name=recipe_name) if recipe_revision: recipe_type = RecipeTypeRevision.objects.get_revision( recipe_name, recipe_revision).recipe_type if len(recipe_type.get_definition().get_input_keys()) == 0: logger.info( 'No inputs defined for recipe %s. Recipe will not be run.' % recipe_name) return if recipe_type and recipe_type.is_active: # Assuming one input per recipe, so pull the first defined input you find recipe_data = Data() input_name = recipe_type.get_definition().get_input_keys()[0] recipe_data.add_value(FileValue(input_name, [source_file.id])) event = self._create_trigger_event(source, source_file, when) ingest_event = self._create_ingest_event(ingest_id, source, source_file, when) # This can cause a race condition with a slow DB. messages = create_recipes_messages( recipe_type.name, recipe_type.revision_num, convert_data_to_v6_json(recipe_data).get_dict(), event.id, ingest_event.id) CommandMessageManager().send_messages(messages) else: logger.info( 'No recipe type found for %s %s or recipe type is inactive' % (recipe_name, recipe_revision))
def test_execute_invalid_data(self): """Tests calling CreateJobs.execute() when the input data is invalid""" manifest = { 'seedVersion': '1.0.0', 'job': { 'name': 'name', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': 'Title', 'description': 'This is a description', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'command': 'the command', 'inputs': { 'files': [{ 'name': 'input_a' }] } } } } job_type = job_test_utils.create_seed_job_type(manifest=manifest) event = trigger_test_utils.create_trigger_event() # Data does not provide required input_a so it is invalid data_dict = convert_data_to_v6_json(Data()).get_dict() # Create and execute message message = create_jobs_message(job_type.name, job_type.version, job_type.revision_num, event.id, count=10, input_data_dict=data_dict) result = message.execute() self.assertTrue(result) self.assertEqual( Job.objects.filter(job_type_id=job_type.id, event_id=event.id).count(), 0) # Should be no new messages self.assertEqual(len(message.new_messages), 0)
def test_queue_job_timestamps(self): """Tests that job attributes are updated when a job is queued.""" data_dict = convert_data_to_v6_json(Data()).get_dict() job = job_test_utils.create_job(num_exes=1, status='CANCELED', input=data_dict, started=timezone.now(), ended=timezone.now()) Job.objects.update_jobs_to_queued([job], timezone.now(), requeue=True) job = Job.objects.get(pk=job.id) self.assertEqual(job.status, 'QUEUED') self.assertIsNotNone(job.queued) self.assertIsNone(job.started) self.assertIsNone(job.ended)
def test_process_event_range(self, mock_Queue, mock_msg_mgr): """Tests processing an event that requires catching up for a range of previous days.""" event = job_test_utils.create_clock_event( occurred=datetime.datetime(2015, 1, 10, 10, tzinfo=utc)) last = job_test_utils.create_clock_event( occurred=datetime.datetime(2015, 1, 7, 12, tzinfo=utc)) self.processor.process_event(event, last) i = 1 for call_args in mock_Queue.objects.queue_new_job_v6.call_args_list: args = call_args[0] self.assertEqual(self.job_type, args[0]) self.assertEqual(event, args[2]) inputs = convert_data_to_v6_json(args[1]) if i == 1: self.assertDictEqual( { u'files': {}, u'json': { u'DAY': '2015-01-07' }, u'version': u'7' }, inputs.get_dict()) if i == 2: self.assertDictEqual( { u'files': {}, u'json': { u'DAY': '2015-01-08' }, u'version': u'7' }, inputs.get_dict()) if i == 3: self.assertDictEqual( { u'files': {}, u'json': { u'DAY': '2015-01-09' }, u'version': u'7' }, inputs.get_dict()) i += 1 self.assertEqual(mock_Queue.objects.queue_new_job_v6.call_count, 3)
def test_json_no_recipe(self): """Tests converting a CreateJobs message (without a recipe) to and from JSON""" manifest = { 'seedVersion': '1.0.0', 'job': { 'name': 'name', 'jobVersion': '1.0.0', 'packageVersion': '1.0.0', 'title': 'Title', 'description': 'This is a description', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10 } } job_type = job_test_utils.create_seed_job_type(manifest=manifest) event = trigger_test_utils.create_trigger_event() data_dict = convert_data_to_v6_json(Data()).get_dict() count = 10 # Create message message = create_jobs_message(job_type.name, job_type.version, job_type.revision_num, event.id, count=count, input_data_dict=data_dict) # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = CreateJobs.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) self.assertEqual( Job.objects.filter(job_type_id=job_type.id, event_id=event.id).count(), count) # Check for process_job_input messages self.assertEqual(len(new_message.new_messages), count) for msg in new_message.new_messages: self.assertEqual(msg.type, 'process_job_input')
def test_json_forced_nodes(self): """Tests converting an UpdateRecipe message to and from JSON when forced nodes are provided""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_completed = job_test_utils.create_job(status='COMPLETED', input=data_dict, output=data_dict) sub_recipe_type = recipe_test_utils.create_recipe_type_v6() definition = RecipeDefinition(Interface()) definition.add_job_node('job_completed', job_completed.job_type.name, job_completed.job_type.version, job_completed.job_type_rev.revision_num) definition.add_recipe_node('the_sub_recipe', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_dependency('job_completed', 'the_sub_recipe') definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict) recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_completed', job=job_completed) forced_nodes = ForcedNodes() sub_forced_nodes = ForcedNodes() sub_forced_nodes.set_all_nodes() forced_nodes.add_subrecipe('the_sub_recipe', sub_forced_nodes) # Create message message = create_update_recipe_message(recipe.id, forced_nodes=forced_nodes) # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = UpdateRecipe.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) # Check for message to create sub-recipe self.assertEqual(len(new_message.new_messages), 1) msg = new_message.new_messages[0] self.assertEqual(msg.type, 'create_recipes') self.assertEqual(msg.event_id, recipe.event_id) msg_forced_nodes_dict = convert_forced_nodes_to_v6(msg.forced_nodes).get_dict() expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict() self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict) self.assertEqual(msg.create_recipes_type, SUB_RECIPE_TYPE) self.assertEqual(msg.recipe_id, recipe.id) self.assertEqual(msg.root_recipe_id, recipe.root_superseded_recipe_id) self.assertIsNone(msg.superseded_recipe_id) sub = SubRecipe(sub_recipe_type.name, sub_recipe_type.revision_num, 'the_sub_recipe', True) self.assertListEqual(msg.sub_recipes, [sub])
def test_process_event_first(self, mock_Queue, mock_msg_mgr): """Tests processing an event that was never triggered before.""" event = job_test_utils.create_clock_event( occurred=datetime.datetime(2015, 1, 10, 12, tzinfo=utc)) self.processor.process_event(event, None) call_args = mock_Queue.objects.queue_new_job_v6.call_args[0] inputs = convert_data_to_v6_json(call_args[1]) self.assertEqual(self.job_type, call_args[0]) self.assertDictEqual( { u'files': {}, u'json': { u'DAY': '2015-01-09' }, u'version': u'7' }, inputs.get_dict()) self.assertEqual(event, call_args[2])
def execute(self): """See :meth:`messaging.messages.message.CommandMessage.execute` """ files_to_delete = ScaleFile.objects.filter_files(job_ids=[self.job_id]) if files_to_delete: # Construct input data files = [] workspaces = [] for f in files_to_delete: files.append({ 'id': f.id, 'file_path': f.file_path, 'workspace': f.workspace.name }) if f.workspace.name not in [ k for wrkspc in workspaces for k in wrkspc.keys() ]: workspaces.append( {f.workspace.name: f.workspace.json_config}) inputs = Data() inputs.add_value(JsonValue('job_id', str(self.job_id))) inputs.add_value(JsonValue('trigger_id', str(self.trigger_id))) inputs.add_value( JsonValue('source_file_id', str(self.source_file_id))) inputs.add_value(JsonValue('purge', str(self.purge))) inputs.add_value(JsonValue('files', json.dumps(files))) inputs.add_value(JsonValue('workspaces', json.dumps(workspaces))) inputs_json = convert_data_to_v6_json(inputs) # Send message to create system job to delete files msg = create_jobs_message(job_type_name="scale-delete-files", job_type_version="1.0.0", event_id=self.trigger_id, job_type_rev_num=1, input_data_dict=inputs_json.get_dict()) self.new_messages.append(msg) return True
def test_json(self): """Tests coverting a ProcessRecipeInput message to and from JSON""" data_dict = convert_data_to_v6_json(Data()).get_dict() recipe = recipe_test_utils.create_recipe(input=data_dict) # Create message message = ProcessRecipeInput() message.recipe_id = recipe.id # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = ProcessRecipeInput.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) recipe = Recipe.objects.get(id=recipe.id) self.assertEqual(len(new_message.new_messages), 1) self.assertEqual(new_message.new_messages[0].type, 'update_recipes') # Recipe should have input_file_size set to 0 (no input files) self.assertEqual(recipe.input_file_size, 0.0)
def test_execute_with_data(self): """Tests calling ProcessRecipeInput.execute() successfully when the recipe already has data populated""" workspace = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace, file_size=10485760.0) file_2 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0) file_3 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0) recipe_interface = Interface() recipe_interface.add_parameter(FileParameter('input_a', ['text/plain'])) recipe_interface.add_parameter( FileParameter('input_b', ['text/plain'], multiple=True)) definition = RecipeDefinition(recipe_interface) definition_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type( definition=definition_dict) data = Data() data.add_value(FileValue('input_a', [file_1.id])) data.add_value(FileValue('input_b', [file_2.id, file_3.id])) data_dict = convert_data_to_v6_json(data).get_dict() recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict) # Create message message = ProcessRecipeInput() message.recipe_id = recipe.id # Execute message result = message.execute() self.assertTrue(result) recipe = Recipe.objects.get(id=recipe.id) # Check for update_recipes message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'update_recipes') # Check recipe for expected input_file_size self.assertEqual(recipe.input_file_size, 1052.0) # Make sure recipe input file models are created recipe_input_files = RecipeInputFile.objects.filter( recipe_id=recipe.id) self.assertEqual(len(recipe_input_files), 3) for recipe_input_file in recipe_input_files: if recipe_input_file.input_file_id == file_1.id: self.assertEqual(recipe_input_file.recipe_input, 'input_a') elif recipe_input_file.input_file_id == file_2.id: self.assertEqual(recipe_input_file.recipe_input, 'input_b') elif recipe_input_file.input_file_id == file_3.id: self.assertEqual(recipe_input_file.recipe_input, 'input_b') else: self.fail('Invalid input file ID: %s' % recipe_input_file.input_file_id) # Test executing message again message_json_dict = message.to_json() message = ProcessRecipeInput.from_json(message_json_dict) result = message.execute() self.assertTrue(result) # Still should have update_recipes message self.assertEqual(len(message.new_messages), 1) self.assertEqual(message.new_messages[0].type, 'update_recipes') # Make sure recipe input file models are unchanged recipe_input_files = RecipeInputFile.objects.filter( recipe_id=recipe.id) self.assertEqual(len(recipe_input_files), 3)
def test_has_completed_true(self): """Tests calling Recipe.has_completed() when an entire recipe has completed""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() cond_interface_1 = Interface() cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer')) df2 = DataFilter(filter_list=[{ 'name': 'cond_int', 'type': 'integer', 'condition': '==', 'values': [0] }, { 'name': 'cond_int', 'type': 'integer', 'condition': '!=', 'values': [0] }], all=True) #always False definition = RecipeDefinition(Interface()) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('B', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('C', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('D', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('H', job_type.name, job_type.version, job_type.revision_num) definition.add_condition_node('I', cond_interface_1, df2) #False definition.add_job_node('J', job_type.name, job_type.version, job_type.revision_num) definition.add_dependency('A', 'C') definition.add_dependency('A', 'E') definition.add_dependency('A', 'H') definition.add_dependency('C', 'D') definition.add_dependency('G', 'H') definition.add_dependency('A', 'I') definition.add_dependency('I', 'J') job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_c = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_d = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_e = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_f = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_h = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h]) condition_i = recipe_test_utils.create_recipe_condition( is_processed=True, is_accepted=False, save=True) recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_b.is_completed = True recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_g.is_completed = True Recipe.objects.bulk_create([recipe_b, recipe_g]) definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False, is_original=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False, is_original=False) recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', job=job_d, save=False, is_original=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', job=job_f, save=False) recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='H', job=job_h, save=False) recipe_node_i = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='I', condition=condition_i, save=False) recipe_node_g = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='G', sub_recipe=recipe_g, save=False, is_original=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False) RecipeNode.objects.bulk_create([ recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h, recipe_node_i ]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) self.assertTrue(recipe_instance.has_completed())
def test_has_completed_false(self): """Tests calling Recipe.has_completed() when an entire recipe has not completed""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() definition = RecipeDefinition(Interface()) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('B', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('C', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('D', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_job_node('H', job_type.name, job_type.version, job_type.revision_num) definition.add_dependency('A', 'C') definition.add_dependency('A', 'E') definition.add_dependency('A', 'H') definition.add_dependency('C', 'D') definition.add_dependency('G', 'H') job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_c = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_d = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_e = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_f = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) job_h = job_test_utils.create_job(job_type=job_type, status='COMPLETED', output=data_dict, save=False) Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h]) recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_b.is_completed = True recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type, save=False) recipe_g.is_completed = False Recipe.objects.bulk_create([recipe_b, recipe_g]) definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False, is_original=False) recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False, is_original=False) recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', job=job_d, save=False, is_original=False) recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False) recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', job=job_f, save=False) recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='H', job=job_h, save=False) recipe_node_g = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='G', sub_recipe=recipe_g, save=False, is_original=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False) RecipeNode.objects.bulk_create([ recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h ]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) self.assertFalse(recipe_instance.has_completed())
def test_get_nodes_to_process_input(self): """Tests calling Recipe.get_nodes_to_process_input()""" data_dict = convert_data_to_v6_json(Data()).get_dict() job_type = job_test_utils.create_seed_job_type() sub_recipe_type = recipe_test_utils.create_recipe_type_v6() # Create recipe definition = RecipeDefinition(Interface()) cond_interface_1 = Interface() cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer')) definition.add_job_node('A', job_type.name, job_type.version, job_type.revision_num) df1 = DataFilter(filter_list=[{ 'name': 'cond_int', 'type': 'integer', 'condition': '==', 'values': [0] }, { 'name': 'cond_int', 'type': 'integer', 'condition': '!=', 'values': [0] }], all=False) #always True df2 = DataFilter(filter_list=[{ 'name': 'cond_int', 'type': 'integer', 'condition': '==', 'values': [0] }, { 'name': 'cond_int', 'type': 'integer', 'condition': '!=', 'values': [0] }], all=True) #always False definition.add_condition_node('B', cond_interface_1, df1) #True definition.add_condition_node('C', cond_interface_1, df1) #True definition.add_condition_node('D', cond_interface_1, df2) #False definition.add_job_node('E', job_type.name, job_type.version, job_type.revision_num) definition.add_job_node('F', job_type.name, job_type.version, job_type.revision_num) definition.add_recipe_node('G', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_recipe_node('H', sub_recipe_type.name, sub_recipe_type.revision_num) definition.add_dependency('A', 'D') definition.add_dependency('A', 'E') definition.add_dependency('B', 'E') definition.add_dependency('B', 'F') definition.add_dependency('C', 'F') definition.add_dependency('D', 'G') definition.add_dependency('E', 'G') definition.add_dependency('E', 'H') definition_json_dict = convert_recipe_definition_to_v6_json( definition).get_dict() recipe_type = recipe_test_utils.create_recipe_type_v6( definition=definition_json_dict) recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict) # Nodes A, B, and D already exist job_a = job_test_utils.create_job(job_type=job_type, status='COMPLETED', input=data_dict, output=data_dict, save=True) condition_b = recipe_test_utils.create_recipe_condition( is_processed=True, is_accepted=True, save=False) condition_d = recipe_test_utils.create_recipe_condition( is_processed=True, is_accepted=False, save=False) RecipeCondition.objects.bulk_create([condition_b, condition_d]) recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False) recipe_node_b = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='B', condition=condition_b, save=False) recipe_node_d = recipe_test_utils.create_recipe_node( recipe=recipe, node_name='D', condition=condition_d, save=False) RecipeNode.objects.bulk_create( [recipe_node_a, recipe_node_b, recipe_node_d]) recipe_instance = Recipe.objects.get_recipe_instance(recipe.id) nodes_to_process = recipe_instance.get_nodes_to_process_input() self.assertSetEqual(set(nodes_to_process.keys()), {'C', 'E'})
def json(self): """Accessor for json in results""" return convert_data_to_v6_json(self._results_data).get_dict()['json']
def files(self): """Accessor for files in results""" return convert_data_to_v6_json(self._results_data).get_dict()['files']