def test_invalid_trigger_rule(self): '''Tests calling RecipeTypeManager.edit_recipe_type() with a new invalid trigger rule''' # Create recipe_type name = 'test-recipe' version = '1.0' title = 'Test Recipe' desc = 'Test description' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=recipe_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) trigger_rule_id = trigger_rule.id new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=recipe_test_utils.MOCK_ERROR_TYPE, configuration=self.new_trigger_config.get_dict()) recipe_type = RecipeType.objects.create_recipe_type(name, version, title, desc, self.recipe_def, trigger_rule) with transaction.atomic(): recipe_type = RecipeType.objects.select_for_update().get(pk=recipe_type.id) # Edit the recipe self.assertRaises(InvalidRecipeConnection, RecipeType.objects.edit_recipe_type, recipe_type.id, None, None, self.new_recipe_def, new_trigger_rule, False) recipe_type = RecipeType.objects.select_related('trigger_rule').get(pk=recipe_type.id) # Check results self.assertEqual(recipe_type.title, title) self.assertEqual(recipe_type.description, desc) self.assertDictEqual(recipe_type.get_recipe_definition().get_dict(), self.recipe_def.get_dict()) self.assertEqual(recipe_type.revision_num, 1) self.assertEqual(recipe_type.trigger_rule_id, trigger_rule_id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule_id) self.assertTrue(trigger_rule.is_active) num_of_revs = RecipeTypeRevision.objects.filter(recipe_type_id=recipe_type.id).count() self.assertEqual(num_of_revs, 1)
def test_change_to_both(self): """Tests calling RecipeTypeManager.edit_recipe_type() with a change to both the definition and trigger rule""" # Create recipe_type name = 'test-recipe' version = '1.0' title = 'Test Recipe' desc = 'Test description' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=recipe_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) trigger_rule_id = trigger_rule.id new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=recipe_test_utils.MOCK_TYPE, configuration=self.new_trigger_config.get_dict()) new_trigger_rule_id = new_trigger_rule.id recipe_type = RecipeType.objects.create_recipe_type(name, version, title, desc, self.recipe_def, trigger_rule) with transaction.atomic(): recipe_type = RecipeType.objects.select_for_update().get(pk=recipe_type.id) # Edit the recipe RecipeType.objects.edit_recipe_type(recipe_type.id, None, None, self.new_recipe_def, new_trigger_rule, False) recipe_type = RecipeType.objects.select_related('trigger_rule').get(pk=recipe_type.id) # Check results self.assertEqual(recipe_type.title, title) self.assertEqual(recipe_type.description, desc) self.assertDictEqual(recipe_type.get_recipe_definition().get_dict(), self.new_recipe_def.get_dict()) self.assertEqual(recipe_type.revision_num, 2) self.assertEqual(recipe_type.trigger_rule_id, new_trigger_rule_id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule_id) self.assertFalse(trigger_rule.is_active) new_trigger_rule = TriggerRule.objects.get(pk=new_trigger_rule_id) self.assertTrue(new_trigger_rule.is_active) # New revision due to definition change num_of_revs = RecipeTypeRevision.objects.filter(recipe_type_id=recipe_type.id).count() self.assertEqual(num_of_revs, 2)
def test_change_to_both(self): '''Tests calling JobTypeManager.edit_job_type() with a change to both the definition and the trigger rule ''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.new_trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule) # Call test JobType.objects.edit_job_type(job_type.id, self.new_job_interface, new_trigger_rule, False) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_job_interface.get_dict()) self.assertEqual(job_type.revision_num, 2) self.assertEqual(job_type.trigger_rule_id, new_trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertFalse(trigger_rule.is_active) new_trigger_rule = TriggerRule.objects.get(pk=new_trigger_rule.id) self.assertTrue(new_trigger_rule.is_active) # New revision due to definition change num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 2)
def test_successful_job_creation(self): """Tests successfully processing a parse that triggers job creation.""" # Set up data configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', 'data_types': ['type1', 'type2'], }, 'data': { 'input_data_name': self.input_name, 'workspace_name': self.workspace.name }, } rule_model = trigger_test_utils.create_trigger_rule( trigger_type='PARSE', configuration=configuration) self.job_type_1.trigger_rule = rule_model self.job_type_1.save() # Call method to test ParseTriggerHandler().process_parsed_source_file(self.source_file) # Check results queue_1 = Queue.objects.get(job_type=self.job_type_1.id) job_1 = Job.objects.get(pk=queue_1.job_id) self.assertEqual(job_1.data['input_data'][0]['name'], self.input_name) self.assertEqual(job_1.data['input_data'][0]['file_id'], self.source_file.id)
def test_change_simple_with_trigger(self): """Tests calling RecipeTypeManager.edit_recipe_type() with only basic attributes and a previous trigger rule""" # Create recipe_type name = 'test-recipe' version = '1.0' title = 'Test Recipe' desc = 'Test description' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=recipe_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) trigger_rule_id = trigger_rule.id recipe_type = RecipeType.objects.create_recipe_type(name, version, title, desc, self.recipe_def, trigger_rule) with transaction.atomic(): recipe_type = RecipeType.objects.select_for_update().get(pk=recipe_type.id) # Edit the recipe new_title = 'New title' new_desc = 'New description' RecipeType.objects.edit_recipe_type(recipe_type.id, new_title, new_desc, None, None, False) recipe_type = RecipeType.objects.select_related('trigger_rule').get(pk=recipe_type.id) # Check results self.assertEqual(recipe_type.title, new_title) self.assertEqual(recipe_type.description, new_desc) self.assertDictEqual(recipe_type.get_recipe_definition().get_dict(), self.recipe_def.get_dict()) self.assertEqual(recipe_type.revision_num, 1) self.assertEqual(recipe_type.trigger_rule_id, trigger_rule_id) num_of_revs = RecipeTypeRevision.objects.filter(recipe_type_id=recipe_type.id).count() self.assertEqual(num_of_revs, 1)
def test_successful_recipe_creation(self): """Tests successfully processing a parse that triggers recipe creation.""" # Set up data configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', }, 'data': { 'input_data_name': self.input_name, 'workspace_name': self.workspace.name }, } rule_model = trigger_test_utils.create_trigger_rule( trigger_type='PARSE', configuration=configuration) self.recipe_type_1.trigger_rule = rule_model self.recipe_type_1.save() # Call method to test ParseTriggerHandler().process_parsed_source_file(self.source_file) # Check results...ensure first job is queued queue_1 = Queue.objects.get(job_type=self.job_type_2.id) job_exe_1 = JobExecution.objects.select_related().get( pk=queue_1.job_exe_id) job_1 = job_exe_1.job self.assertEqual(job_1.data['input_data'][0]['name'], self.input_name) self.assertEqual(job_1.data['input_data'][0]['file_id'], self.source_file.id) self.assertEqual(job_1.data['output_data'][0]['name'], self.output_name) self.assertEqual(job_1.data['output_data'][0]['workspace_id'], self.workspace.id)
def create_clock_rule(name=None, rule_type='CLOCK', event_type=None, schedule='PT1H0M0S', is_active=True): """Creates a scale clock trigger rule model for unit testing :returns: The trigger rule model :rtype: :class:`trigger.models.TriggerRule` """ if not event_type: global RULE_EVENT_COUNTER event_type = 'TEST_EVENT_%i' % RULE_EVENT_COUNTER RULE_EVENT_COUNTER += 1 config = { 'version': '1.0', 'event_type': event_type, 'schedule': schedule, } return trigger_test_utils.create_trigger_rule(name=name, trigger_type=rule_type, configuration=config, is_active=is_active)
def test_successful_recipe_creation(self): '''Tests successfully processing a parse that triggers recipe creation.''' # Set up data configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', }, 'data': { 'input_data_name': self.input_name, 'workspace_name': self.workspace.name }, } rule_model = trigger_test_utils.create_trigger_rule(trigger_type='PARSE', configuration=configuration) self.recipe_type_1.trigger_rule = rule_model self.recipe_type_1.save() # Call method to test ParseTriggerHandler().process_parsed_source_file(self.source_file) # Check results...ensure first job is queued queue_1 = Queue.objects.get(job_type=self.job_type_2.id) job_exe_1 = JobExecution.objects.select_related().get(pk=queue_1.job_exe_id) job_1 = job_exe_1.job self.assertEqual(job_1.data['input_data'][0]['name'], self.input_name) self.assertEqual(job_1.data['input_data'][0]['file_id'], self.source_file.id) self.assertEqual(job_1.data['output_data'][0]['name'], self.output_name) self.assertEqual(job_1.data['output_data'][0]['workspace_id'], self.workspace.id)
def test_successful_job_creation(self): '''Tests successfully processing an ingest that triggers job creation.''' # Set up data configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', 'data_types': ['type1', 'type2'], }, 'data': { 'input_data_name': self.input_name, 'workspace_name': self.workspace.name }, } rule_model = trigger_test_utils.create_trigger_rule( trigger_type='INGEST', configuration=configuration) self.job_type_1.trigger_rule = rule_model self.job_type_1.save() # Call method to test IngestTriggerHandler().process_ingested_source_file( self.source_file, now()) # Check results queue_1 = Queue.objects.get(job_type=self.job_type_1.id) job_exe_1 = JobExecution.objects.select_related().get( pk=queue_1.job_exe_id) job_1 = job_exe_1.job self.assertEqual(job_1.data['input_data'][0]['name'], self.input_name) self.assertEqual(job_1.data['input_data'][0]['file_id'], self.source_file.id)
def test_successful_job_creation(self): """Tests successfully processing an ingest that triggers job creation.""" # Set up data configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', 'data_types': ['type1', 'type2'], }, 'data': { 'input_data_name': self.input_name, 'workspace_name': self.workspace.name }, } rule_model = trigger_test_utils.create_trigger_rule(trigger_type='INGEST', configuration=configuration) self.job_type_1.trigger_rule = rule_model self.job_type_1.save() # Call method to test IngestTriggerHandler().process_ingested_source_file(self.source_file, now()) # Check results queue_1 = Queue.objects.get(job_type=self.job_type_1.id) job_exe_1 = JobExecution.objects.select_related().get(pk=queue_1.job_exe_id) job_1 = job_exe_1.job self.assertEqual(job_1.data['input_data'][0]['name'], self.input_name) self.assertEqual(job_1.data['input_data'][0]['file_id'], self.source_file.id)
def test_remove_trigger_rule(self): """Tests calling RecipeTypeManager.edit_recipe_type() that removes the trigger rule""" # Create recipe_type name = 'test-recipe' version = '1.0' title = 'Test Recipe' desc = 'Test description' trigger_rule = trigger_test_utils.create_trigger_rule( trigger_type=recipe_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) trigger_rule_id = trigger_rule.id recipe_type = RecipeType.objects.create_recipe_type( name, version, title, desc, self.recipe_def, trigger_rule) with transaction.atomic(): recipe_type = RecipeType.objects.select_for_update().get( pk=recipe_type.id) # Edit the recipe RecipeType.objects.edit_recipe_type(recipe_type.id, None, None, None, None, True) recipe_type = RecipeType.objects.select_related('trigger_rule').get( pk=recipe_type.id) # Check results self.assertEqual(recipe_type.title, title) self.assertEqual(recipe_type.description, desc) self.assertDictEqual(recipe_type.get_recipe_definition().get_dict(), self.recipe_def.get_dict()) self.assertEqual(recipe_type.revision_num, 1) self.assertIsNone(recipe_type.trigger_rule) trigger_rule = TriggerRule.objects.get(pk=trigger_rule_id) self.assertFalse(trigger_rule.is_active) num_of_revs = RecipeTypeRevision.objects.filter( recipe_type_id=recipe_type.id).count() self.assertEqual(num_of_revs, 1)
def test_change_simple_with_trigger(self): """Tests calling RecipeTypeManager.edit_recipe_type() with only basic attributes and a previous trigger rule""" # Create recipe_type name = 'test-recipe' version = '1.0' title = 'Test Recipe' desc = 'Test description' trigger_rule = trigger_test_utils.create_trigger_rule( trigger_type=recipe_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) trigger_rule_id = trigger_rule.id recipe_type = RecipeType.objects.create_recipe_type( name, version, title, desc, self.recipe_def, trigger_rule) with transaction.atomic(): recipe_type = RecipeType.objects.select_for_update().get( pk=recipe_type.id) # Edit the recipe new_title = 'New title' new_desc = 'New description' RecipeType.objects.edit_recipe_type(recipe_type.id, new_title, new_desc, None, None, False) recipe_type = RecipeType.objects.select_related('trigger_rule').get( pk=recipe_type.id) # Check results self.assertEqual(recipe_type.title, new_title) self.assertEqual(recipe_type.description, new_desc) self.assertDictEqual(recipe_type.get_recipe_definition().get_dict(), self.recipe_def.get_dict()) self.assertEqual(recipe_type.revision_num, 1) self.assertEqual(recipe_type.trigger_rule_id, trigger_rule_id) num_of_revs = RecipeTypeRevision.objects.filter( recipe_type_id=recipe_type.id).count() self.assertEqual(num_of_revs, 1)
def create_seed_job_type(manifest=None, priority=50, max_tries=3, max_scheduled=None, is_active=True, is_operational=True, trigger_rule=None, configuration=None): if not manifest: manifest = { 'seedVersion': '1.0.0', 'job': { 'name': 'image-watermark', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Image Watermarker', 'description': 'Processes an input PNG and outputs watermarked PNG.', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 30, 'interface': { 'command': '${INPUT_IMAGE} ${OUTPUT_DIR}', 'inputs': { 'files': [{'name': 'INPUT_IMAGE'}] }, 'outputs': { 'files': [{'name': 'OUTPUT_IMAGE', 'pattern': '*_watermark.png'}] } }, 'resources': { 'scalar': [ {'name': 'cpus', 'value': 1.0}, {'name': 'mem', 'value': 64.0} ] }, 'errors': [ { 'code': 1, 'name': 'image-corrupt', 'title': 'Image Corrupt', 'description': 'Image input is not recognized as a valid PNG.', 'category': 'data' } ] } } if not trigger_rule: trigger_rule = trigger_test_utils.create_trigger_rule() if not configuration: configuration = { 'version': '1.0', 'default_settings': {} } job_type = JobType.objects.create(name=manifest['job']['name'], version=manifest['job']['jobVersion'], manifest=manifest, priority=priority, timeout=manifest['job']['timeout'], max_tries=max_tries, max_scheduled=max_scheduled, is_active=is_active, is_operational=is_operational, trigger_rule=trigger_rule, configuration=configuration) JobTypeRevision.objects.create_job_type_revision(job_type) return job_type
def create_job_type(name=None, version=None, category=None, interface=None, priority=50, timeout=3600, max_tries=3, max_scheduled=None, cpus=1.0, mem=1.0, disk=1.0, error_mapping=None, is_active=True, is_system=False, is_operational=True, trigger_rule=None, configuration=None): """Creates a job type model for unit testing :returns: The job type model :rtype: :class:`job.models.JobType` """ if not name: global JOB_TYPE_NAME_COUNTER name = 'test-job-type-%i' % JOB_TYPE_NAME_COUNTER JOB_TYPE_NAME_COUNTER += 1 if not version: global JOB_TYPE_VERSION_COUNTER version = '%i.0.0' % JOB_TYPE_VERSION_COUNTER JOB_TYPE_VERSION_COUNTER += 1 if not category: global JOB_TYPE_CATEGORY_COUNTER category = 'test-category-%i' % JOB_TYPE_CATEGORY_COUNTER JOB_TYPE_CATEGORY_COUNTER += 1 if not interface: interface = { 'version': '1.4', 'command': 'test_cmd', 'command_arguments': 'test_arg', 'env_vars': [], 'mounts': [], 'settings': [], 'input_data': [], 'output_data': [], 'shared_resources': [], } if not error_mapping: error_mapping = { 'version': '1.0', 'exit_codes': {} } if not trigger_rule: trigger_rule = trigger_test_utils.create_trigger_rule() if not configuration: configuration = { 'version': '1.0', 'default_settings': {} } job_type = JobType.objects.create(name=name, version=version, category=category, manifest=interface, priority=priority, timeout=timeout, max_tries=max_tries, max_scheduled=max_scheduled, cpus_required=cpus, mem_const_required=mem, disk_out_const_required=disk, error_mapping=error_mapping, is_active=is_active, is_system=is_system, is_operational=is_operational, trigger_rule=trigger_rule, configuration=configuration) JobTypeRevision.objects.create_job_type_revision(job_type) return job_type
def setUp(self): django.setup() self.workspace = storage_test_utils.create_workspace() self.error = error_test_utils.create_error() self.interface = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }]} self.job_interface = JobInterface(self.interface) self.error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '1': self.error.name, } }) self.configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain' }, 'data': { 'input_data_name': 'Test Input 1', 'workspace_name': self.workspace.name } } self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration) self.trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) self.invalid_trigger_config = job_test_utils.MockErrorTriggerRuleConfiguration(job_test_utils.MOCK_ERROR_TYPE, self.configuration) self.invalid_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE, configuration=self.trigger_config.get_dict())
def create_job_type(name=None, version=None, category=None, interface=None, priority=50, timeout=3600, max_tries=3, max_scheduled=None, cpus=1.0, mem=1.0, disk=1.0, error_mapping=None, is_active=True, is_operational=True, trigger_rule=None, configuration=None): """Creates a job type model for unit testing :returns: The job type model :rtype: :class:`job.models.JobType` """ if not name: global JOB_TYPE_NAME_COUNTER name = 'test-job-type-%i' % JOB_TYPE_NAME_COUNTER JOB_TYPE_NAME_COUNTER += 1 if not version: global JOB_TYPE_VERSION_COUNTER version = '%i.0.0' % JOB_TYPE_VERSION_COUNTER JOB_TYPE_VERSION_COUNTER += 1 if not category: global JOB_TYPE_CATEGORY_COUNTER category = 'test-category-%i' % JOB_TYPE_CATEGORY_COUNTER JOB_TYPE_CATEGORY_COUNTER += 1 if not interface: interface = { 'version': '1.2', 'command': 'test_cmd', 'command_arguments': 'test_arg', 'env_vars': [], 'settings': [], 'input_data': [], 'output_data': [], 'shared_resources': [], } if not error_mapping: error_mapping = { 'version': '1.0', 'exit_codes': {} } if not trigger_rule: trigger_rule = trigger_test_utils.create_trigger_rule() if not configuration: configuration = { 'version': '1.0', 'default_settings': {} } job_type = JobType.objects.create(name=name, version=version, category=category, interface=interface, priority=priority, timeout=timeout, max_tries=max_tries, max_scheduled=max_scheduled, cpus_required=cpus, mem_required=mem, disk_out_const_required=disk, error_mapping=error_mapping, is_active=is_active, is_operational=is_operational, trigger_rule=trigger_rule, configuration=configuration) JobTypeRevision.objects.create_job_type_revision(job_type) return job_type
def create_recipe_type(name=None, version=None, title=None, description=None, definition=None, trigger_rule=None): '''Creates a recipe type for unit testing :returns: The RecipeType model :rtype: :class:`recipe.models.RecipeType` ''' if not name: global NAME_COUNTER name = 'test-recipe-type-%i' % NAME_COUNTER NAME_COUNTER = NAME_COUNTER + 1 if not version: global VERSION_COUNTER version = '%i.0.0' % VERSION_COUNTER VERSION_COUNTER = VERSION_COUNTER + 1 if not title: global TITLE_COUNTER title = 'Test Recipe Type %i' % TITLE_COUNTER TITLE_COUNTER = TITLE_COUNTER + 1 if not description: global DESCRIPTION_COUNTER description = 'Test Description %i' % DESCRIPTION_COUNTER DESCRIPTION_COUNTER = DESCRIPTION_COUNTER + 1 if not definition: definition = { 'version': '1.0', 'input_data': [], 'jobs': [], } if not trigger_rule: trigger_rule = trigger_test_utils.create_trigger_rule() recipe_type = RecipeType() recipe_type.name = name recipe_type.version = version recipe_type.title = title recipe_type.description = description recipe_type.definition = definition recipe_type.trigger_rule = trigger_rule recipe_type.save() RecipeTypeRevision.objects.create_recipe_type_revision(recipe_type) return recipe_type
def test_invalid_trigger_rule(self): '''Tests calling JobTypeManager.create_job_type() with an invalid trigger rule''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE, configuration=self.trigger_config.get_dict()) # Call test self.assertRaises(InvalidConnection, JobType.objects.create_job_type, name, version, self.job_interface, trigger_rule, self.error_mapping)
def create_recipe_type(name=None, version=None, title=None, description=None, definition=None, trigger_rule=None): """Creates a recipe type for unit testing :returns: The RecipeType model :rtype: :class:`recipe.models.RecipeType` """ if not name: global NAME_COUNTER name = 'test-recipe-type-%i' % NAME_COUNTER NAME_COUNTER += 1 if not version: global VERSION_COUNTER version = '%i.0.0' % VERSION_COUNTER VERSION_COUNTER += 1 if not title: global TITLE_COUNTER title = 'Test Recipe Type %i' % TITLE_COUNTER TITLE_COUNTER += 1 if not description: global DESCRIPTION_COUNTER description = 'Test Description %i' % DESCRIPTION_COUNTER DESCRIPTION_COUNTER += 1 if not definition: definition = { 'version': '1.0', 'input_data': [], 'jobs': [], } if not trigger_rule: trigger_rule = trigger_test_utils.create_trigger_rule() recipe_type = RecipeType() recipe_type.name = name recipe_type.version = version recipe_type.title = title recipe_type.description = description recipe_type.definition = definition recipe_type.trigger_rule = trigger_rule recipe_type.save() RecipeTypeRevision.objects.create_recipe_type_revision(recipe_type) return recipe_type
def test_invalid_trigger_rule(self): '''Tests calling JobTypeManager.edit_job_type() with a new invalid trigger rule''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE, configuration=self.new_trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule) # Call test self.assertRaises(InvalidConnection, JobType.objects.edit_job_type, job_type.id, self.new_job_interface, new_trigger_rule, False) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertEqual(job_type.trigger_rule_id, trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertTrue(trigger_rule.is_active) num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 1)
def test_uneditable_field(self): '''Tests calling JobTypeManager.edit_job_type() to change an uneditable field''' name = 'my-job-type' version = '1.0' title = 'my title' new_title = 'my new title' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title) # Call test self.assertRaises(Exception, JobType.objects.edit_job_type, job_type.id, title=new_title, is_system=True) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) # No change self.assertEqual(job_type.title, title)
def setUp(self): django.setup() self.job_type1 = job_test_utils.create_job_type() self.job_type2 = job_test_utils.create_job_type() self.workspace = storage_test_utils.create_workspace() self.trigger_config = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', }, 'data': { 'input_data_name': 'input_file', 'workspace_name': self.workspace.name, } } self.trigger_rule = trigger_test_utils.create_trigger_rule( trigger_type='PARSE', is_active=True, configuration=self.trigger_config) self.definition = { 'version': '1.0', 'input_data': [], 'jobs': [{ 'name': 'Job 1', 'job_type': { 'name': self.job_type1.name, 'version': self.job_type1.version, }, }, { 'name': 'Job 2', 'job_type': { 'name': self.job_type2.name, 'version': self.job_type2.version, }, }], } self.recipe_type = recipe_test_utils.create_recipe_type( name='my-type', definition=self.definition, trigger_rule=self.trigger_rule)
def test_successful_with_trigger_rule(self): '''Tests calling JobTypeManager.create_job_type() successfully with a trigger rule and error mapping''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) # Call test job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, self.error_mapping) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertEqual(job_type.trigger_rule_id, trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertTrue(trigger_rule.is_active) self.assertDictEqual(job_type.get_error_interface().get_dict(), self.error_mapping.get_dict())
def test_change_general_fields(self): '''Tests calling JobTypeManager.edit_job_type() with a change to some general fields''' name = 'my-job-type' version = '1.0' title = 'my title' priority = 12 error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '-15': self.error.name, } }) new_title = 'my new title' new_priority = 13 new_error_mapping = ErrorInterface({ 'version': '1.0', 'exit_codes': { '-16': self.error.name, } }) new_is_paused = True trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title, priority=priority, error_mapping=error_mapping) # Call test JobType.objects.edit_job_type(job_type.id, title=new_title, priority=new_priority, error_mapping=new_error_mapping, is_paused=new_is_paused) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertEqual(job_type.trigger_rule_id, trigger_rule.id) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertTrue(trigger_rule.is_active) self.assertEqual(job_type.title, new_title) self.assertEqual(job_type.priority, new_priority) self.assertDictEqual(job_type.get_error_interface().get_dict(), new_error_mapping.get_dict()) self.assertEqual(job_type.is_paused, new_is_paused) self.assertIsNotNone(job_type.paused)
def create_clock_rule(name=None, rule_type='CLOCK', event_type=None, schedule='PT1H0M0S', is_active=True): '''Creates a scale clock trigger rule model for unit testing :returns: The trigger rule model :rtype: :class:`trigger.models.TriggerRule` ''' if not event_type: global RULE_EVENT_COUNTER event_type = 'TEST_EVENT_%i' % RULE_EVENT_COUNTER RULE_EVENT_COUNTER = RULE_EVENT_COUNTER + 1 config = { 'version': '1.0', 'event_type': event_type, 'schedule': schedule, } return trigger_test_utils.create_trigger_rule(name=name, trigger_type=rule_type, configuration=config, is_active=is_active)
def test_valid_feature_with_parse_rule(self): """Tests calling save_parse_results with valid arguments and parse rules in place""" # Setup parse rule workspace = storage_utils.create_workspace() configuration = {u'version': u'1.0', u'condition': {u'media_type': u'text/plain', u'data_types': [u'type']}, u'data': {u'input_data_name': u'my_input', u'workspace_name': workspace.name}} rule_model = trigger_utils.create_trigger_rule(trigger_type='PARSE', configuration=configuration) interface = {'version': '1.0', 'command': '', 'command_arguments': '', 'input_data': [{'name': 'my_input', 'type': 'file'}]} job_type = job_utils.create_job_type(interface=interface) job_type.trigger_rule = rule_model job_type.save() # Call method to test SourceFile.objects.save_parse_results(self.src_file.id, FEATURE_GEOJSON, self.started, self.ended, [], None, None) # Ensure there's an event for the parse evt = TriggerEvent.objects.first() self.assertEqual(evt.description[u'version'], u'1.0') self.assertEqual(evt.description[u'file_name'], u'text.txt')
def test_remove_trigger_rule(self): '''Tests calling JobTypeManager.edit_job_type() that removes the trigger rule''' name = 'my-job-type' version = '1.0' trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE, configuration=self.trigger_config.get_dict()) job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule) # Call test JobType.objects.edit_job_type(job_type.id, None, None, True) # Check results job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id) self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict()) self.assertEqual(job_type.revision_num, 1) self.assertIsNone(job_type.trigger_rule) trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id) self.assertFalse(trigger_rule.is_active) num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count() self.assertEqual(num_of_revs, 1)
def setUp(self): django.setup() self.job_type1 = job_test_utils.create_job_type() self.job_type2 = job_test_utils.create_job_type() self.workspace = storage_test_utils.create_workspace() self.trigger_config = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', }, 'data': { 'input_data_name': 'input_file', 'workspace_name': self.workspace.name, } } self.trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type='PARSE', is_active=True, configuration=self.trigger_config) self.definition = { 'version': '1.0', 'input_data': [], 'jobs': [{ 'name': 'Job 1', 'job_type': { 'name': self.job_type1.name, 'version': self.job_type1.version, }, }, { 'name': 'Job 2', 'job_type': { 'name': self.job_type2.name, 'version': self.job_type2.version, }, }], } self.recipe_type = recipe_test_utils.create_recipe_type(name='my-type', definition=self.definition, trigger_rule=self.trigger_rule)
def create_seed_job_type(manifest=None, priority=50, max_tries=3, max_scheduled=None, is_active=True, is_operational=True, trigger_rule=None, configuration=None, docker_image='fake'): if not manifest: global JOB_TYPE_NAME_COUNTER name = 'test-job-type-%i' % JOB_TYPE_NAME_COUNTER JOB_TYPE_NAME_COUNTER += 1 manifest = { 'seedVersion': '1.0.0', 'job': { 'name': name, 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Image Watermarker', 'description': 'Processes an input PNG and outputs watermarked PNG.', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 30, 'interface': { 'command': '${INPUT_IMAGE} ${OUTPUT_DIR}', 'inputs': { 'files': [{ 'name': 'INPUT_IMAGE' }] }, 'outputs': { 'files': [{ 'name': 'OUTPUT_IMAGE', 'pattern': '*_watermark.png' }] }, 'mounts': [{ 'name': 'MOUNT_PATH', 'path': '/the/container/path', 'mode': 'ro' }], 'settings': [{ 'name': 'VERSION', 'secret': False }, { 'name': 'DB_HOST', 'secret': False }, { 'name': 'DB_PASS', 'secret': True }] }, 'resources': { 'scalar': [{ 'name': 'cpus', 'value': 1.0 }, { 'name': 'mem', 'value': 64.0 }] }, 'errors': [{ 'code': 1, 'name': 'image-corrupt', 'title': 'Image Corrupt', 'description': 'Image input is not recognized as a valid PNG.', 'category': 'data' }] } } if not trigger_rule: trig_config = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', }, 'data': { 'input_data_name': 'INPUT_IMAGE', 'workspace_name': storage_test_utils.create_workspace().name, } } trigger_rule = trigger_test_utils.create_trigger_rule( configuration=trig_config) if not configuration: configuration = {'version': '1.0', 'default_settings': {}} job_type = JobType.objects.create(name=manifest['job']['name'], version=manifest['job']['jobVersion'], manifest=manifest, priority=priority, timeout=manifest['job']['timeout'], max_tries=max_tries, max_scheduled=max_scheduled, is_active=is_active, is_operational=is_operational, trigger_rule=trigger_rule, configuration=configuration, docker_image=docker_image) version_array = job_type.get_job_version_array( manifest['job']['jobVersion']) job_type.version_array = version_array job_type.save() JobTypeRevision.objects.create_job_type_revision(job_type) return job_type
def setUp(self): django.setup() self.workspace = storage_test_utils.create_workspace() self.file = storage_test_utils.create_file() configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', }, 'data': { 'input_data_name': 'Recipe Input', 'workspace_name': self.workspace.name, }, } self.rule = trigger_test_utils.create_trigger_rule(configuration=configuration) self.event = trigger_test_utils.create_trigger_event(rule=self.rule) interface_1 = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }], } self.job_type_1 = job_test_utils.create_job_type(interface=interface_1) self.definition_1 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'type': 'file', 'media_types': ['text/plain'], }], 'jobs': [{ 'name': 'Job 1', 'job_type': { 'name': self.job_type_1.name, 'version': self.job_type_1.version, }, 'recipe_inputs': [{ 'recipe_input': 'Recipe Input', 'job_input': 'Test Input 1', }], }], } RecipeDefinition(self.definition_1).validate_job_interfaces() self.recipe_type = recipe_test_utils.create_recipe_type(definition=self.definition_1, trigger_rule=self.rule) self.interface_2 = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 2', 'type': 'files', 'media_types': ['image/tiff'], }], 'output_data': [{ 'name': 'Test Output 2', 'type': 'file', }], } self.job_type_2 = job_test_utils.create_job_type(interface=self.interface_2) self.definition_2 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'type': 'file', 'media_types': ['text/plain'], }], 'jobs': [{ 'name': 'Job 1', 'job_type': { 'name': self.job_type_1.name, 'version': self.job_type_1.version, }, 'recipe_inputs': [{ 'recipe_input': 'Recipe Input', 'job_input': 'Test Input 1', }] }, { 'name': 'Job 2', 'job_type': { 'name': self.job_type_2.name, 'version': self.job_type_2.version, }, 'dependencies': [{ 'name': 'Job 1', 'connections': [{ 'output': 'Test Output 1', 'input': 'Test Input 2', }], }], }], } self.data = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': self.file.id, }], 'workspace_id': self.workspace.id, }
def setUp(self): django.setup() self.workspace = storage_test_utils.create_workspace() self.file = storage_test_utils.create_file() configuration = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', }, 'data': { 'input_data_name': 'Recipe Input', 'workspace_name': self.workspace.name, }, } self.rule = trigger_test_utils.create_trigger_rule( configuration=configuration) self.event = trigger_test_utils.create_trigger_event(rule=self.rule) interface_1 = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 1', 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': 'Test Output 1', 'type': 'files', 'media_type': 'image/png', }], } self.job_type_1 = job_test_utils.create_job_type(interface=interface_1) self.definition_1 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'type': 'file', 'media_types': ['text/plain'], }], 'jobs': [{ 'name': 'Job 1', 'job_type': { 'name': self.job_type_1.name, 'version': self.job_type_1.version, }, 'recipe_inputs': [{ 'recipe_input': 'Recipe Input', 'job_input': 'Test Input 1', }], }], } RecipeDefinition(self.definition_1).validate_job_interfaces() self.recipe_type = recipe_test_utils.create_recipe_type( definition=self.definition_1, trigger_rule=self.rule) self.interface_2 = { 'version': '1.0', 'command': 'my_command', 'command_arguments': 'args', 'input_data': [{ 'name': 'Test Input 2', 'type': 'files', 'media_types': ['image/tiff'], }], 'output_data': [{ 'name': 'Test Output 2', 'type': 'file', }], } self.job_type_2 = job_test_utils.create_job_type( interface=self.interface_2) self.definition_2 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'type': 'file', 'media_types': ['text/plain'], }], 'jobs': [{ 'name': 'Job 1', 'job_type': { 'name': self.job_type_1.name, 'version': self.job_type_1.version, }, 'recipe_inputs': [{ 'recipe_input': 'Recipe Input', 'job_input': 'Test Input 1', }] }, { 'name': 'Job 2', 'job_type': { 'name': self.job_type_2.name, 'version': self.job_type_2.version, }, 'dependencies': [{ 'name': 'Job 1', 'connections': [{ 'output': 'Test Output 1', 'input': 'Test Input 2', }], }], }], } self.data = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': self.file.id, }], 'workspace_id': self.workspace.id, }
def setUp(self): django.setup() self.input_name = 'Test Input' self.output_name = 'Test Output' self.workspace = storage_test_utils.create_workspace() # This job trigger should not match due to a different media type job_trigger_config = { 'version': '1.0', 'condition': { 'media_type': 'image/png', }, 'data': { 'input_data_name': self.input_name, 'workspace_name': self.workspace.name }, } job_trigger_rule = trigger_test_utils.create_trigger_rule( configuration=job_trigger_config) interface_1 = { 'version': '1.0', 'command': 'my_cmd', 'command_arguments': 'args', 'input_data': [{ 'name': self.input_name, 'type': 'file', }], } self.job_type_1 = job_test_utils.create_job_type( interface=interface_1, trigger_rule=job_trigger_rule) interface_2 = { 'version': '1.0', 'command': 'my_cmd', 'command_arguments': 'args', 'input_data': [{ 'name': self.input_name, 'type': 'file', }], 'output_data': [{ 'name': self.output_name, 'type': 'file', }], } self.job_type_2 = job_test_utils.create_job_type( interface=interface_2, trigger_rule=job_trigger_rule) # create a recipe that runs both jobs definition_1 = { 'version': '1.0', 'input_data': [{ 'name': self.input_name, 'type': 'file', 'required': True, }], 'jobs': [{ 'name': 'Job 2', 'job_type': { 'name': self.job_type_2.name, 'version': self.job_type_2.version, }, 'recipe_inputs': [{ 'recipe_input': self.input_name, 'job_input': self.input_name, }], }, { 'name': 'Job 1', 'job_type': { 'name': self.job_type_1.name, 'version': self.job_type_1.version, }, 'dependencies': [{ 'name': 'Job 2', 'connections': [{ 'output': self.output_name, 'input': self.input_name, }], }], }], } self.when_parsed = now() self.file_name = 'my_file.txt' self.data_type = 'test_file_type' self.media_type = 'text/plain' self.source_file = ScaleFile.objects.create(file_name=self.file_name, file_type='SOURCE', media_type=self.media_type, file_size=10, data_type=self.data_type, file_path='the_path', workspace=self.workspace) self.source_file.add_data_type_tag('type1') self.source_file.add_data_type_tag('type2') self.source_file.add_data_type_tag('type3') self.source_file.parsed = now() recipe_trigger_config = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', }, 'data': { 'input_data_name': self.input_name, 'workspace_name': self.workspace.name }, } self.trigger_rule = trigger_test_utils.create_trigger_rule( configuration=recipe_trigger_config) self.recipe_type_1 = recipe_test_utils.create_recipe_type( definition=definition_1, trigger_rule=self.trigger_rule)
def setUp(self): django.setup() self.input_name = 'Test Input' self.output_name = 'Test Output' self.workspace = storage_test_utils.create_workspace() # This job trigger should not match due to a different media type job_trigger_config = { 'version': '1.0', 'condition': { 'media_type': 'image/png', }, 'data': { 'input_data_name': self.input_name, 'workspace_name': self.workspace.name }, } job_trigger_rule = trigger_test_utils.create_trigger_rule(configuration=job_trigger_config) interface_1 = { 'version': '1.0', 'command': 'my_cmd', 'command_arguments': 'args', 'input_data': [{ 'name': self.input_name, 'type': 'file', }], } self.job_type_1 = job_test_utils.create_job_type(interface=interface_1, trigger_rule=job_trigger_rule) interface_2 = { 'version': '1.0', 'command': 'my_cmd', 'command_arguments': 'args', 'input_data': [{ 'name': self.input_name, 'type': 'file', }], 'output_data': [{ 'name': self.output_name, 'type': 'file', }], } self.job_type_2 = job_test_utils.create_job_type(interface=interface_2, trigger_rule=job_trigger_rule) # create a recipe that runs both jobs definition_1 = { 'version': '1.0', 'input_data': [{ 'name': self.input_name, 'type': 'file', 'required': True, }], 'jobs': [{ 'name': 'Job 2', 'job_type': { 'name': self.job_type_2.name, 'version': self.job_type_2.version, }, 'recipe_inputs': [{ 'recipe_input': self.input_name, 'job_input': self.input_name, }], }, { 'name': 'Job 1', 'job_type': { 'name': self.job_type_1.name, 'version': self.job_type_1.version, }, 'dependencies': [{ 'name': 'Job 2', 'connections': [{ 'output': self.output_name, 'input': self.input_name, }], }], }], } self.when_parsed = now() self.file_name = 'my_file.txt' self.data_type = 'test_file_type' self.media_type = 'text/plain' self.source_file = SourceFile.objects.create(file_name=self.file_name, media_type=self.media_type, file_size=10, data_type=self.data_type, file_path='the_path', workspace=self.workspace) self.source_file.add_data_type_tag('type1') self.source_file.add_data_type_tag('type2') self.source_file.add_data_type_tag('type3') self.source_file.parsed = now() recipe_trigger_config = { 'version': '1.0', 'condition': { 'media_type': 'text/plain', }, 'data': { 'input_data_name': self.input_name, 'workspace_name': self.workspace.name }, } self.trigger_rule = trigger_test_utils.create_trigger_rule(configuration=recipe_trigger_config) self.recipe_type_1 = recipe_test_utils.create_recipe_type(definition=definition_1, trigger_rule=self.trigger_rule)