def edit_job_type_v6(job_type, manifest_dict=None, docker_image=None, icon_code=None, is_active=None, is_paused=None, max_scheduled=None, configuration_dict=None): """Updates a job type, including creating a new revision for unit testing """ manifest = SeedManifest(manifest_dict, do_validate=True) configuration = None if configuration_dict: configuration = JobConfigurationV6( configuration_dict, do_validate=True).get_configuration() JobType.objects.edit_job_type_v6(job_type.id, manifest=manifest, docker_image=docker_image, icon_code=icon_code, is_active=is_active, is_paused=is_paused, max_scheduled=max_scheduled, configuration=configuration)
def test_init_validation(self): """Tests creating and validating a Seed manifest JSON""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'image-watermark', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Image Watermarker', 'description': 'Processes an input PNG and outputs watermarked PNG.', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 30, 'interface': { 'command': '${INPUT_IMAGE} ${OUTPUT_DIR}', 'inputs': { 'files': [{ 'name': 'INPUT_IMAGE' }] }, 'outputs': { 'files': [{ 'name': 'OUTPUT_IMAGE', 'pattern': '*_watermark.png' }] } }, 'resources': { 'scalar': [{ 'name': 'cpus', 'value': 1 }, { 'name': 'mem', 'value': 64 }] }, 'errors': [{ 'code': 1, 'name': 'image-Corrupt-1', 'description': 'Image input is not recognized as a valid PNG.', 'category': 'data' }, { 'code': 2, 'name': 'algorithm-failure' }] } } # No exception is success SeedManifest(manifest_dict, do_validate=True)
def get_job_interface(self): """Returns the interface for this queued job :returns: The job interface :rtype: :class:`job.configuration.interface.job_interface.JobInterface` """ return SeedManifest(self.interface, do_validate=False)
def test_remove_secret_settings(self): """Tests calling JobConfiguration.remove_secret_settings()""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'settings': [{ 'name': 'setting_a' }, { 'name': 'secret_setting_a', 'secret': True }, { 'name': 'secret_setting_b', 'secret': True }, { 'name': 'secret_setting_c', 'secret': True }] } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() configuration.add_setting('setting_a', 'value_1') configuration.add_setting('secret_setting_a', 'secret_value_1') configuration.add_setting('secret_setting_b', 'secret_value_2') configuration.add_setting('setting_d', 'value_4') secret_settings = configuration.remove_secret_settings(manifest) self.assertDictEqual( secret_settings, { 'secret_setting_a': 'secret_value_1', 'secret_setting_b': 'secret_value_2' }) self.assertDictEqual(configuration.settings, { 'setting_a': 'value_1', 'setting_d': 'value_4' })
def test_validation(self): """Tests creating and validating the Seed manifest in delete_files_job_type.json""" json_file_name = 'delete_files_job_type.json' storage_dir = os.path.abspath(os.path.join( __file__, "../../..")) # storage/test/fixtures json_file = os.path.join(storage_dir, 'fixtures', json_file_name) with open(json_file) as json_data: d = json.load(json_data) manifest_dict = d[0]['fields']['manifest'] # No exception is success SeedManifest(manifest_dict, do_validate=True)
def create(interface_dict, do_validate=True): """Instantiate an instance of the JobInterface based on inferred type :param interface_dict: deserialized JSON interface :type interface_dict: dict :param do_validate: whether schema validation should be applied :type do_validate: bool :return: instance of the job interface appropriate for input data :rtype: :class:`job.configuration.interface.job_interface.JobInterface` or :class:`job.seed.manifest.SeedManifest` """ if JobInterfaceSunset.is_seed_dict(interface_dict): return SeedManifest(interface_dict, do_validate=do_validate) else: return JobInterface(interface_dict, do_validate=do_validate)
def test_no_default_workspace(self): """Tests calling JobConfiguration.validate() to validate output workspaces""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'outputs': { 'files': [{ 'name': 'output_a', 'mediaType': 'image/gif', 'pattern': '*_a.gif' }, { 'name': 'output_b', 'mediaType': 'image/gif', 'pattern': '*_a.gif' }] } } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() # No workspaces defined for outputs warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 2) self.assertEqual(warnings[0].name, 'MISSING_WORKSPACE') self.assertEqual(warnings[1].name, 'MISSING_WORKSPACE')
def test_validate_priority(self): """Tests calling JobConfiguration.validate() to validate priority""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10 } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() configuration.priority = 100 warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 0) configuration.priority = 0 with self.assertRaises(InvalidJobConfiguration) as context: configuration.validate(manifest) self.assertEqual(context.exception.error.name, 'INVALID_PRIORITY') configuration.priority = -1 with self.assertRaises(InvalidJobConfiguration) as context: configuration.validate(manifest) self.assertEqual(context.exception.error.name, 'INVALID_PRIORITY')
def queue_jobs(self, jobs, requeue=False, priority=None): """Queues the given jobs. The caller must have obtained model locks on the job models in an atomic transaction. Any jobs that are not in a valid status for being queued, are without job input, or are superseded will be ignored. :param jobs: The job models to put on the queue :type jobs: list :param requeue: Whether this is a re-queue (True) or a first queue (False) :type requeue: bool :param priority: An optional argument to reset the jobs' priority when they are queued :type priority: int :returns: The list of job IDs that were successfully QUEUED :rtype: list """ when_queued = timezone.now() # Set job models to QUEUED queued_job_ids = Job.objects.update_jobs_to_queued(jobs, when_queued, requeue=requeue) if not queued_job_ids: return queued_job_ids # Done if nothing was queued # Retrieve the related job_type, job_type_rev, and batch models for the queued jobs queued_jobs = Job.objects.get_jobs_with_related(queued_job_ids) # Query for all input files of the queued jobs input_files = {} input_file_ids = set() for job in queued_jobs: input_file_ids.update(job.get_job_data().get_input_file_ids()) if input_file_ids: for input_file in ScaleFile.objects.get_files_for_queued_jobs( input_file_ids): input_files[input_file.id] = input_file # Bulk create queue models queues = [] configurator = QueuedExecutionConfigurator(input_files) for job in queued_jobs: config = configurator.configure_queued_job(job) manifest = None if JobInterfaceSunset.is_seed_dict(job.job_type.manifest): manifest = SeedManifest(job.job_type.manifest) if priority: queued_priority = priority elif job.priority: queued_priority = job.priority elif job.batch and self.batch.get_configuration().priority: queued_priority = self.batch.get_configuration().priority else: queued_priority = job.job_type.get_job_configuration().priority queue = Queue() queue.job_type_id = job.job_type_id queue.job_id = job.id queue.recipe_id = job.recipe_id queue.batch_id = job.batch_id queue.exe_num = job.num_exes queue.input_file_size = job.input_file_size if job.input_file_size else 0.0 queue.is_canceled = False queue.priority = queued_priority queue.timeout = manifest.get_timeout() if manifest else job.timeout queue.interface = job.get_job_interface().get_dict() queue.configuration = config.get_dict() queue.resources = job.get_resources().get_json().get_dict() queue.queued = when_queued queues.append(queue) if queues: self.bulk_create(queues) return queued_job_ids
def test_init_default_values(self): """Tests creating and validating a Seed manifest JSON and ensures the correct defaults are used""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'my-job', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'My Job', 'description': 'Processes my job', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 30, 'interface': { 'command': '${INPUT_IMAGE} ${JSON_FILES} ${OUTPUT_DIR}', 'inputs': { 'files': [{ 'name': 'INPUT_IMAGE' }, { 'name': 'JSON_FILES', 'mediaTypes': ['application/json'], 'multiple': True, 'partial': True, 'required': False }] }, 'outputs': { 'files': [{ 'name': 'OUTPUT_IMAGE_A', 'pattern': '*.tif' }, { 'name': 'OUTPUT_IMAGE_B', 'pattern': '*.tif', 'mediaType': 'image/tiff', 'multiple': True, 'required': False }] } }, 'resources': { 'scalar': [{ 'name': 'cpus', 'value': 1 }, { 'name': 'mem', 'value': 64 }] }, 'errors': [] } } manifest = SeedManifest(manifest_dict, do_validate=True) # Check input and output files for correct values input_files = manifest.get_input_files() input_image_dict = input_files[0] json_files_dict = input_files[1] self.assertDictEqual( input_image_dict, { 'name': 'INPUT_IMAGE', 'mediaTypes': [], 'multiple': False, 'partial': False, 'required': True }) self.assertDictEqual( json_files_dict, { 'name': 'JSON_FILES', 'mediaTypes': ['application/json'], 'multiple': True, 'partial': True, 'required': False }) output_files = manifest.get_output_files() output_image_a_dict = output_files[0] output_image_b_dict = output_files[1] self.assertDictEqual( output_image_a_dict, { 'name': 'OUTPUT_IMAGE_A', 'pattern': '*.tif', 'mediaType': UNKNOWN_MEDIA_TYPE, 'multiple': False, 'required': True }) self.assertDictEqual( output_image_b_dict, { 'name': 'OUTPUT_IMAGE_B', 'pattern': '*.tif', 'mediaType': 'image/tiff', 'multiple': True, 'required': False })
def convert_interface_to_manifest(apps, schema_editor): # Go through all of the JobType models and convert legacy interfaces to Seed manifests # Also inactivate/pause them JobType = apps.get_model('job', 'JobType') JobTypeRevision = apps.get_model('job', 'JobTypeRevision') RecipeTypeJobLink = apps.get_model('recipe', 'RecipeTypeJobLink') RecipeType = apps.get_model('recipe', 'RecipeType') unique = 0 for jt in JobType.objects.all().iterator(): if JobInterfaceSunset.is_seed_dict(jt.manifest): continue jt.is_active = False jt.is_paused = True old_name = jt.name old_name_version = jt.name + ' ' + jt.version jt.name = 'legacy-' + jt.name.replace('_', '-') if not jt.manifest: jt.manifest = {} input_files = [] input_json = [] output_files = [] global INTERFACE_NAME_COUNTER INTERFACE_NAME_COUNTER = 0 for input in jt.manifest.get('input_data', []): type = input.get('type', '') if 'file' not in type: json = {} json['name'] = get_unique_name(input.get('name')) json['type'] = 'string' json['required'] = input.get('required', True) input_json.append(json) continue file = {} file['name'] = get_unique_name(input.get('name')) file['required'] = input.get('required', True) file['partial'] = input.get('partial', False) file['mediaTypes'] = input.get('media_types', []) file['multiple'] = (type == 'files') input_files.append(file) for output in jt.manifest.get('output_data', []): type = output.get('type', '') file = {} file['name'] = get_unique_name(output.get('name')) file['required'] = output.get('required', True) file['mediaType'] = output.get('media_type', '') file['multiple'] = (type == 'files') file['pattern'] = "*.*" output_files.append(file) mounts = [] for mount in jt.manifest.get('mounts', []): mt = {} mt['name'] = get_unique_name(mount.get('name')) mt['path'] = mount.get('path') mt['mode'] = mount.get('mode', 'ro') mounts.append(mt) settings = [] for setting in jt.manifest.get('settings', []): s = {} s['name'] = get_unique_name(setting.get('name')) s['secret'] = setting.get('secret', False) settings.append(s) for var in jt.manifest.get('env_vars', []): s = {} name = get_unique_name(var.get('name')) name = 'ENV_' + name s['name'] = name settings.append(s) errors = [] ec = jt.error_mapping.get('exit_codes', {}) for exit_code, error_name in ec.items(): error = { 'code': int(exit_code), 'name': get_unique_name(error_name), 'title': 'Error Name', 'description': 'Error Description', 'category': 'algorithm' } errors.append(error) new_manifest = { 'seedVersion': '1.0.0', 'job': { 'name': jt.name, 'jobVersion': '0.0.0', 'packageVersion': '1.0.0', 'title': 'LEGACY ' + jt.title, 'description': jt.description, 'tags': [jt.category, old_name_version], 'maintainer': { 'name': jt.author_name, 'email': '*****@*****.**', 'url': jt.author_url }, 'timeout': jt.timeout, 'interface': { 'command': jt.manifest.get('command', ''), 'inputs': { 'files': input_files, 'json': input_json }, 'outputs': { 'files': output_files, 'json': [] }, 'mounts': mounts, 'settings': settings }, 'resources': { 'scalar': [{ 'name': 'cpus', 'value': jt.cpus_required }, { 'name': 'mem', 'value': jt.mem_const_required, 'inputMultiplier': jt.mem_mult_required }, { 'name': 'sharedMem', 'value': jt.shared_mem_required }, { 'name': 'disk', 'value': jt.disk_out_const_required, 'inputMultiplier': jt.disk_out_mult_required }] }, 'errors': errors } } jt.manifest = new_manifest SeedManifest(jt.manifest, do_validate=True) jt.save() for jtr in JobTypeRevision.objects.filter( job_type_id=jt.id).iterator(): jtr.manifest = jt.manifest jtr.save() # Update any recipe types that reference the updated job name for rtjl in RecipeTypeJobLink.objects.all().filter( job_type_id=jt.id).iterator(): recipe_type = RecipeType.objects.get(id=rtjl.id) definition = recipe_type.definition changed = False # v6 if 'nodes' in definition: for node in definition['nodes']: jt_node = node['node_type'] if jt_node['node_type'] == 'job' and jt_node[ 'job_type_name'].replace( '_', '-') == old_name and jt_node[ 'job_type_version'] == jt.version: node['node_type']['job_type_name'] = jt.name changed = True # v5 elif 'jobs' in definition: for job in definition['jobs']: jt_node = job['job_type'] if jt_node['name'].replace( '_', '-' ) == old_name and jt_node['version'] == jt.version: job['job_type']['name'] = jt.name changed = True if changed: recipe_type.definition = definition recipe_type.save()
def _perform_job_type_manifest_iteration(self): """Performs a single iteration of updating job type interfaces """ # Get job type ID jt_qry = JobType.objects.all() if self._current_job_type_id: jt_qry = jt_qry.filter(id__gt=self._current_job_type_id) for jt in jt_qry.order_by('id').only('id')[:1]: jt_id = jt.id break jt = JobType.objects.get(pk=jt_id) if not JobInterfaceSunset.is_seed_dict(jt.manifest): jt.is_active = False jt.is_paused = True old_name_version = jt.name + ' ' + jt.version jt.name = 'legacy-' + jt.name.replace('_', '-') if not jt.manifest: jt.manifest = {} input_files = [] input_json = [] output_files = [] global INTERFACE_NAME_COUNTER INTERFACE_NAME_COUNTER = 0 for input in jt.manifest.get('input_data', []): type = input.get('type', '') if 'file' not in type: json = {} json['name'] = get_unique_name(input.get('name')) json['type'] = 'string' json['required'] = input.get('required', True) input_json.append(json) continue file = {} file['name'] = get_unique_name(input.get('name')) file['required'] = input.get('required', True) file['partial'] = input.get('partial', False) file['mediaTypes'] = input.get('media_types', []) file['multiple'] = (type == 'files') input_files.append(file) for output in jt.manifest.get('output_data', []): type = output.get('type', '') file = {} file['name'] = get_unique_name(output.get('name')) file['required'] = output.get('required', True) file['mediaType'] = output.get('media_type', '') file['multiple'] = (type == 'files') file['pattern'] = "*.*" output_files.append(file) mounts = [] for mount in jt.manifest.get('mounts', []): mt = {} mt['name'] = get_unique_name(mount.get('name')) mt['path'] = mount.get('path') mt['mode'] = mount.get('mode', 'ro') mounts.append(mt) settings = [] for setting in jt.manifest.get('settings', []): s = {} s['name'] = get_unique_name(setting.get('name')) s['secret'] = setting.get('secret', False) settings.append(s) for var in jt.manifest.get('env_vars', []): s = {} name = get_unique_name(var.get('name')) name = 'ENV_' + name s['name'] = name settings.append(s) new_manifest = { 'seedVersion': '1.0.0', 'job': { 'name': jt.name, 'jobVersion': '0.0.0', 'packageVersion': '1.0.0', 'title': 'Legacy Title', 'description': 'legacy job type: ' + old_name_version, 'tags': [], 'maintainer': { 'name': 'Legacy', 'email': '*****@*****.**' }, 'timeout': 3600, 'interface': { 'command': jt.manifest.get('command', ''), 'inputs': { 'files': input_files, 'json': input_json }, 'outputs': { 'files': output_files, 'json': [] }, 'mounts': mounts, 'settings': settings }, 'resources': { 'scalar': [ { 'name': 'cpus', 'value': 1.0 }, { 'name': 'mem', 'value': 1024.0 }, { 'name': 'disk', 'value': 1000.0, 'inputMultiplier': 4.0 } ] }, 'errors': [] } } jt.manifest = new_manifest SeedManifest(jt.manifest, do_validate=True) jt.save() for jtr in JobTypeRevision.objects.filter(job_type_id=jt.id).iterator(): jtr.manifest = jt.manifest jtr.save() self._current_job_type_id = jt_id self._updated_job_type += 1 if self._updated_job_type > self._total_job_type: self._updated_job_type = self._total_job_type percent = (float(self._updated_job_type) / float(self._total_job_type)) * 100.00 logger.info('Completed %s of %s job types (%.1f%%)', self._updated_job_type, self._total_job_type, percent)
def patch(self, request, name, version): """Edits an existing seed job type and returns the updated details :param request: the HTTP PATCH request :type request: :class:`rest_framework.request.Request` :param job_type_id: The ID for the job type. :type job_type_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ auto_update = rest_util.parse_bool(request, 'auto_update', required=False, default_value=True) icon_code = rest_util.parse_string(request, 'icon_code', required=False) is_published = rest_util.parse_string(request, 'is_published', required=False) is_active = rest_util.parse_bool(request, 'is_active', required=False) is_paused = rest_util.parse_bool(request, 'is_paused', required=False) max_scheduled = rest_util.parse_int(request, 'max_scheduled', required=False) docker_image = rest_util.parse_string(request, 'docker_image', required=False) # Validate the manifest manifest_dict = rest_util.parse_dict(request, 'manifest', required=False) manifest = None if manifest_dict: try: manifest = SeedManifest(manifest_dict, do_validate=True) except InvalidSeedManifestDefinition as ex: message = 'Seed Manifest invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) # validate manifest name/version matches job type manifest_name = manifest.get_name() if name != manifest_name: raise BadParameter('Manifest name %s does not match current Job Type name %s.' % (manifest_name, name)) manifest_version = manifest.get_job_version() if manifest_version != version: raise BadParameter('Manifest version %s does not match current Job Type version %s.' % (manifest_version, version)) # Validate the job configuration and pull out secrets configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) configuration = None try: if configuration_dict: configuration = JobConfigurationV6(configuration_dict).get_configuration() except InvalidJobConfiguration as ex: raise BadParameter('Job type configuration invalid: %s' % unicode(ex)) # Fetch the current job type model try: job_type = JobType.objects.get(name=name, version=version) except JobType.DoesNotExist: raise Http404 # Check for invalid fields fields = {'icon_code', 'is_published', 'is_active', 'is_paused', 'max_scheduled', 'configuration', 'manifest', 'docker_image', 'auto_update'} for key, value in request.data.iteritems(): if key not in fields: raise BadParameter('%s is not a valid field. Valid fields are: %s' % (key, fields)) try: with transaction.atomic(): # Edit the job type validation = JobType.objects.edit_job_type_v6(job_type_id=job_type.id, manifest=manifest, is_published=is_published, docker_image=docker_image, icon_code=icon_code, is_active=is_active, is_paused=is_paused, max_scheduled=max_scheduled, configuration=configuration, auto_update=auto_update) except (InvalidJobField, InvalidSecretsConfiguration, ValueError, InvalidJobConfiguration, InvalidInterfaceDefinition) as ex: logger.exception('Unable to update job type: %i', job_type.id) raise BadParameter(unicode(ex)) resp_dict = {'is_valid': validation.is_valid, 'errors': [e.to_dict() for e in validation.errors], 'warnings': [w.to_dict() for w in validation.warnings]} return Response(resp_dict)
def create_v6(self, request): """Creates or edits a Seed job type and returns a link to the detail URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ # Optional icon code value icon_code = rest_util.parse_string(request, 'icon_code', required=False) # Optional is published value is_published = rest_util.parse_string(request, 'is_published', required=False) # Optional max scheduled value max_scheduled = rest_util.parse_int(request, 'max_scheduled', required=False) # Require docker image value docker_image = rest_util.parse_string(request, 'docker_image', required=True) # Validate the job interface / manifest manifest_dict = rest_util.parse_dict(request, 'manifest', required=True) # If editing an existing job type, automatically update recipes containing said job type auto_update = rest_util.parse_bool(request, 'auto_update', required=False, default_value=True) # Optional setting job type active if editing existing job is_active = rest_util.parse_bool(request, 'is_active', required=False) # Optional setting job type to paused if editing an existing job is_paused = rest_util.parse_bool(request, 'is_paused', required=False) manifest = None try: manifest = SeedManifest(manifest_dict, do_validate=True) except InvalidSeedManifestDefinition as ex: message = 'Seed Manifest invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) # Validate the job configuration and pull out secrets configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) configuration = None if configuration_dict: try: configuration = JobConfigurationV6(configuration_dict, do_validate=True).get_configuration() except InvalidJobConfiguration as ex: message = 'Job type configuration invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) # Check for invalid fields fields = {'icon_code', 'is_published', 'max_scheduled', 'docker_image', 'configuration', 'manifest', 'auto_update', 'is_active', 'is_paused'} for key, value in request.data.iteritems(): if key not in fields: raise BadParameter('%s is not a valid field. Valid fields are: %s' % (key, fields)) name = manifest_dict['job']['name'] version = manifest_dict['job']['jobVersion'] if name == 'validation': logger.exception('Unable to create job type named "validation"') raise BadParameter(unicode('Unable to create job type named "validation"')) existing_job_type = JobType.objects.filter(name=name, version=version).first() if not existing_job_type: try: # Create the job type job_type = JobType.objects.create_job_type_v6(icon_code=icon_code, is_published=is_published, max_scheduled=max_scheduled, docker_image=docker_image, manifest=manifest, configuration=configuration) except (InvalidJobField, InvalidSecretsConfiguration, ValueError) as ex: message = 'Unable to create new job type' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) except InvalidSeedManifestDefinition as ex: message = 'Job type manifest invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) except InvalidJobConfiguration as ex: message = 'Job type configuration invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) # Fetch the full job type with details try: job_type = JobType.objects.get_details_v6(name=name, version=version) except JobType.DoesNotExist: raise Http404 url = reverse('job_type_details_view', args=[job_type.name, job_type.version], request=request) serializer = JobTypeDetailsSerializerV6(job_type) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url)) else: try: validation = JobType.objects.edit_job_type_v6(job_type_id=existing_job_type.id, manifest=manifest, docker_image=docker_image, icon_code=icon_code, is_active=is_active, is_paused=is_paused, max_scheduled=max_scheduled, is_published=is_published, configuration=configuration, auto_update=auto_update) except (InvalidJobField, InvalidSecretsConfiguration, ValueError, InvalidInterfaceDefinition) as ex: logger.exception('Unable to update job type: %i', existing_job_type.id) raise BadParameter(unicode(ex)) except InvalidSeedManifestDefinition as ex: message = 'Job type manifest invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) except InvalidJobConfiguration as ex: message = 'Job type configuration invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) resp_dict = {'is_valid': validation.is_valid, 'errors': [e.to_dict() for e in validation.errors], 'warnings': [w.to_dict() for w in validation.warnings]} return Response(resp_dict)
def test_validate_settings(self): """Tests calling JobConfiguration.validate() to validate settings configuration""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'settings': [{ 'name': 'setting_a' }, { 'name': 'secret_setting_a', 'secret': True }, { 'name': 'secret_setting_b', 'secret': True }, { 'name': 'secret_setting_c', 'secret': True }] } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() configuration.add_setting('setting_a', 'value_1') configuration.add_setting('secret_setting_a', 'secret_value_1') configuration.add_setting('secret_setting_b', 'secret_value_2') configuration.add_setting('secret_setting_c', 'secret_value_3') configuration.add_setting('setting_4', 'value_4') warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].name, 'UNKNOWN_SETTING') manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'settings': [{ 'name': 'setting_a' }, { 'name': 'secret_setting_a', 'secret': True }, { 'name': 'secret_setting_b', 'secret': True }, { 'name': 'secret_setting_c', 'secret': True }] } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() configuration.add_setting('setting_a', 'value_1') configuration.add_setting('secret_setting_a', 'secret_value_1') configuration.add_setting('secret_setting_b', 'secret_value_2') warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].name, 'MISSING_SETTING')
def configure_queued_job(self, job): """Creates and returns an execution configuration for the given queued job. The given job model should have its related job_type, job_type_rev, and batch models populated. :param job: The queued job model :type job: :class:`job.models.Job` :returns: The execution configuration for the queued job :rtype: :class:`job.execution.configuration.json.exe_config.ExecutionConfiguration` """ config = ExecutionConfiguration() data = job.get_job_data() # Add input file meta-data input_files_dict = self._create_input_file_dict(data) config.set_input_files(input_files_dict) # Set up env vars for job's input data input_values = data.get_injected_input_values(input_files_dict) interface = job.job_type_rev.get_input_interface() env_vars = {} if isinstance(data, JobData): # call job.data.job_data.JobData.get_injected_env_vars env_vars = data.get_injected_env_vars(input_files_dict, interface) else: # call old job.configuration.data.job_data.get_injected_env_vars # TODO: remove once old JobData class is no longer used env_vars = data.get_injected_env_vars(input_files_dict) task_workspaces = {} if job.job_type.is_system: # Add any workspaces needed for this system job task_workspaces = QueuedExecutionConfigurator._system_job_workspaces( job) else: # Set any output workspaces needed output_workspaces = {} if job.input and 'version' in job.input and job.input[ 'version'] == '1.0': # Set output workspaces using legacy job data self._cache_workspace_names(data.get_output_workspace_ids()) output_workspaces = {} for output, workspace_id in data.get_output_workspaces().items( ): output_workspaces[output] = self._cached_workspace_names[ workspace_id] config.set_output_workspaces(output_workspaces) if not output_workspaces: # Set output workspaces from job configuration output_workspaces = {} job_config = job.get_job_configuration() interface = SeedManifest(job.job_type_rev.manifest, do_validate=False) for output_name in interface.get_file_output_names(): output_workspace = job_config.get_output_workspace( output_name) if output_workspace: output_workspaces[output_name] = output_workspace config.set_output_workspaces(output_workspaces) # Create main task with fields populated from input data args = job.get_job_interface().get_injected_command_args( input_values, env_vars) config.create_tasks(['main']) config.add_to_task('main', args=args, env_vars=env_vars, workspaces=task_workspaces) return config
def test_validate_output_workspaces(self): """Tests calling JobConfiguration.validate() to validate output workspaces""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'outputs': { 'files': [{ 'name': 'output_a', 'mediaType': 'image/gif', 'pattern': '*_a.gif' }, { 'name': 'output_b', 'mediaType': 'image/gif', 'pattern': '*_a.gif' }] } } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() # No workspaces defined warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 2) self.assertEqual(warnings[0].name, 'MISSING_WORKSPACE') self.assertEqual(warnings[1].name, 'MISSING_WORKSPACE') # Workspace does not exist configuration.default_output_workspace = 'workspace_1' with self.assertRaises(InvalidJobConfiguration) as context: configuration.validate(manifest) self.assertEqual(context.exception.error.name, 'INVALID_WORKSPACE') # Default workspace defined with valid workspace workspace_1 = storage_test_utils.create_workspace(name='workspace_1') warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 0) # Workspace is only defined for output_a configuration.default_output_workspace = None configuration.add_output_workspace('output_a', 'workspace_1') warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].name, 'MISSING_WORKSPACE') # Workspace defined for both outputs storage_test_utils.create_workspace(name='workspace_2') configuration.add_output_workspace('output_b', 'workspace_2') warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 0) # Workspace is deprecated workspace_1.is_active = False workspace_1.save() warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].name, 'DEPRECATED_WORKSPACE')
def test_validate_mounts(self): """Tests calling JobConfiguration.validate() to validate mount configuration""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'mounts': [{ 'name': 'mount_a', 'path': '/the/a/path' }, { 'name': 'mount_b', 'path': '/the/b/path' }, { 'name': 'mount_c', 'path': '/the/c/path' }] } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() configuration.add_mount(HostMountConfig('mount_a', '/the/host/a/path')) configuration.add_mount(HostMountConfig('mount_b', '/the/host/b/path')) configuration.add_mount(HostMountConfig('mount_c', '/the/host/c/path')) configuration.add_mount(HostMountConfig('mount_d', '/the/host/d/path')) warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].name, 'UNKNOWN_MOUNT') manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'mounts': [{ 'name': 'mount_a', 'path': '/the/a/path' }, { 'name': 'mount_b', 'path': '/the/b/path' }, { 'name': 'mount_c', 'path': '/the/c/path' }] } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() configuration.add_mount(HostMountConfig('mount_a', '/the/host/a/path')) configuration.add_mount(HostMountConfig('mount_b', '/the/host/b/path')) warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].name, 'MISSING_MOUNT')