def edit_job_type_v6(job_type,
manifest_dict=None,
docker_image=None,
icon_code=None,
is_active=None,
is_paused=None,
max_scheduled=None,
configuration_dict=None):
"""Updates a job type, including creating a new revision for unit testing
"""
manifest = SeedManifest(manifest_dict, do_validate=True)
configuration = None
if configuration_dict:
configuration = JobConfigurationV6(
configuration_dict, do_validate=True).get_configuration()
JobType.objects.edit_job_type_v6(job_type.id,
manifest=manifest,
docker_image=docker_image,
icon_code=icon_code,
is_active=is_active,
is_paused=is_paused,
max_scheduled=max_scheduled,
configuration=configuration)
def test_init_validation(self):
"""Tests creating and validating a Seed manifest JSON"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name':
'image-watermark',
'jobVersion':
'0.1.0',
'packageVersion':
'0.1.0',
'title':
'Image Watermarker',
'description':
'Processes an input PNG and outputs watermarked PNG.',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout':
30,
'interface': {
'command': '${INPUT_IMAGE} ${OUTPUT_DIR}',
'inputs': {
'files': [{
'name': 'INPUT_IMAGE'
}]
},
'outputs': {
'files': [{
'name': 'OUTPUT_IMAGE',
'pattern': '*_watermark.png'
}]
}
},
'resources': {
'scalar': [{
'name': 'cpus',
'value': 1
}, {
'name': 'mem',
'value': 64
}]
},
'errors': [{
'code': 1,
'name': 'image-Corrupt-1',
'description':
'Image input is not recognized as a valid PNG.',
'category': 'data'
}, {
'code': 2,
'name': 'algorithm-failure'
}]
}
}
# No exception is success
SeedManifest(manifest_dict, do_validate=True)
def get_job_interface(self):
"""Returns the interface for this queued job
:returns: The job interface
:rtype: :class:`job.configuration.interface.job_interface.JobInterface`
"""
return SeedManifest(self.interface, do_validate=False)
def test_remove_secret_settings(self):
"""Tests calling JobConfiguration.remove_secret_settings()"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'settings': [{
'name': 'setting_a'
}, {
'name': 'secret_setting_a',
'secret': True
}, {
'name': 'secret_setting_b',
'secret': True
}, {
'name': 'secret_setting_c',
'secret': True
}]
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
configuration.add_setting('setting_a', 'value_1')
configuration.add_setting('secret_setting_a', 'secret_value_1')
configuration.add_setting('secret_setting_b', 'secret_value_2')
configuration.add_setting('setting_d', 'value_4')
secret_settings = configuration.remove_secret_settings(manifest)
self.assertDictEqual(
secret_settings, {
'secret_setting_a': 'secret_value_1',
'secret_setting_b': 'secret_value_2'
})
self.assertDictEqual(configuration.settings, {
'setting_a': 'value_1',
'setting_d': 'value_4'
})
def test_validation(self):
"""Tests creating and validating the Seed manifest in delete_files_job_type.json"""
json_file_name = 'delete_files_job_type.json'
storage_dir = os.path.abspath(os.path.join(
__file__, "../../..")) # storage/test/fixtures
json_file = os.path.join(storage_dir, 'fixtures', json_file_name)
with open(json_file) as json_data:
d = json.load(json_data)
manifest_dict = d[0]['fields']['manifest']
# No exception is success
SeedManifest(manifest_dict, do_validate=True)
def create(interface_dict, do_validate=True):
"""Instantiate an instance of the JobInterface based on inferred type
:param interface_dict: deserialized JSON interface
:type interface_dict: dict
:param do_validate: whether schema validation should be applied
:type do_validate: bool
:return: instance of the job interface appropriate for input data
:rtype: :class:`job.configuration.interface.job_interface.JobInterface` or
:class:`job.seed.manifest.SeedManifest`
"""
if JobInterfaceSunset.is_seed_dict(interface_dict):
return SeedManifest(interface_dict, do_validate=do_validate)
else:
return JobInterface(interface_dict, do_validate=do_validate)
def test_no_default_workspace(self):
"""Tests calling JobConfiguration.validate() to validate output workspaces"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'outputs': {
'files': [{
'name': 'output_a',
'mediaType': 'image/gif',
'pattern': '*_a.gif'
}, {
'name': 'output_b',
'mediaType': 'image/gif',
'pattern': '*_a.gif'
}]
}
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
# No workspaces defined for outputs
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 2)
self.assertEqual(warnings[0].name, 'MISSING_WORKSPACE')
self.assertEqual(warnings[1].name, 'MISSING_WORKSPACE')
def test_validate_priority(self):
"""Tests calling JobConfiguration.validate() to validate priority"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
configuration.priority = 100
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 0)
configuration.priority = 0
with self.assertRaises(InvalidJobConfiguration) as context:
configuration.validate(manifest)
self.assertEqual(context.exception.error.name, 'INVALID_PRIORITY')
configuration.priority = -1
with self.assertRaises(InvalidJobConfiguration) as context:
configuration.validate(manifest)
self.assertEqual(context.exception.error.name, 'INVALID_PRIORITY')
def queue_jobs(self, jobs, requeue=False, priority=None):
"""Queues the given jobs. The caller must have obtained model locks on the job models in an atomic transaction.
Any jobs that are not in a valid status for being queued, are without job input, or are superseded will be
ignored.
:param jobs: The job models to put on the queue
:type jobs: list
:param requeue: Whether this is a re-queue (True) or a first queue (False)
:type requeue: bool
:param priority: An optional argument to reset the jobs' priority when they are queued
:type priority: int
:returns: The list of job IDs that were successfully QUEUED
:rtype: list
"""
when_queued = timezone.now()
# Set job models to QUEUED
queued_job_ids = Job.objects.update_jobs_to_queued(jobs,
when_queued,
requeue=requeue)
if not queued_job_ids:
return queued_job_ids # Done if nothing was queued
# Retrieve the related job_type, job_type_rev, and batch models for the queued jobs
queued_jobs = Job.objects.get_jobs_with_related(queued_job_ids)
# Query for all input files of the queued jobs
input_files = {}
input_file_ids = set()
for job in queued_jobs:
input_file_ids.update(job.get_job_data().get_input_file_ids())
if input_file_ids:
for input_file in ScaleFile.objects.get_files_for_queued_jobs(
input_file_ids):
input_files[input_file.id] = input_file
# Bulk create queue models
queues = []
configurator = QueuedExecutionConfigurator(input_files)
for job in queued_jobs:
config = configurator.configure_queued_job(job)
manifest = None
if JobInterfaceSunset.is_seed_dict(job.job_type.manifest):
manifest = SeedManifest(job.job_type.manifest)
if priority:
queued_priority = priority
elif job.priority:
queued_priority = job.priority
elif job.batch and self.batch.get_configuration().priority:
queued_priority = self.batch.get_configuration().priority
else:
queued_priority = job.job_type.get_job_configuration().priority
queue = Queue()
queue.job_type_id = job.job_type_id
queue.job_id = job.id
queue.recipe_id = job.recipe_id
queue.batch_id = job.batch_id
queue.exe_num = job.num_exes
queue.input_file_size = job.input_file_size if job.input_file_size else 0.0
queue.is_canceled = False
queue.priority = queued_priority
queue.timeout = manifest.get_timeout() if manifest else job.timeout
queue.interface = job.get_job_interface().get_dict()
queue.configuration = config.get_dict()
queue.resources = job.get_resources().get_json().get_dict()
queue.queued = when_queued
queues.append(queue)
if queues:
self.bulk_create(queues)
return queued_job_ids
def test_init_default_values(self):
"""Tests creating and validating a Seed manifest JSON and ensures the correct defaults are used"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'my-job',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'My Job',
'description': 'Processes my job',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 30,
'interface': {
'command': '${INPUT_IMAGE} ${JSON_FILES} ${OUTPUT_DIR}',
'inputs': {
'files': [{
'name': 'INPUT_IMAGE'
}, {
'name': 'JSON_FILES',
'mediaTypes': ['application/json'],
'multiple': True,
'partial': True,
'required': False
}]
},
'outputs': {
'files': [{
'name': 'OUTPUT_IMAGE_A',
'pattern': '*.tif'
}, {
'name': 'OUTPUT_IMAGE_B',
'pattern': '*.tif',
'mediaType': 'image/tiff',
'multiple': True,
'required': False
}]
}
},
'resources': {
'scalar': [{
'name': 'cpus',
'value': 1
}, {
'name': 'mem',
'value': 64
}]
},
'errors': []
}
}
manifest = SeedManifest(manifest_dict, do_validate=True)
# Check input and output files for correct values
input_files = manifest.get_input_files()
input_image_dict = input_files[0]
json_files_dict = input_files[1]
self.assertDictEqual(
input_image_dict, {
'name': 'INPUT_IMAGE',
'mediaTypes': [],
'multiple': False,
'partial': False,
'required': True
})
self.assertDictEqual(
json_files_dict, {
'name': 'JSON_FILES',
'mediaTypes': ['application/json'],
'multiple': True,
'partial': True,
'required': False
})
output_files = manifest.get_output_files()
output_image_a_dict = output_files[0]
output_image_b_dict = output_files[1]
self.assertDictEqual(
output_image_a_dict, {
'name': 'OUTPUT_IMAGE_A',
'pattern': '*.tif',
'mediaType': UNKNOWN_MEDIA_TYPE,
'multiple': False,
'required': True
})
self.assertDictEqual(
output_image_b_dict, {
'name': 'OUTPUT_IMAGE_B',
'pattern': '*.tif',
'mediaType': 'image/tiff',
'multiple': True,
'required': False
})
def convert_interface_to_manifest(apps, schema_editor):
# Go through all of the JobType models and convert legacy interfaces to Seed manifests
# Also inactivate/pause them
JobType = apps.get_model('job', 'JobType')
JobTypeRevision = apps.get_model('job', 'JobTypeRevision')
RecipeTypeJobLink = apps.get_model('recipe', 'RecipeTypeJobLink')
RecipeType = apps.get_model('recipe', 'RecipeType')
unique = 0
for jt in JobType.objects.all().iterator():
if JobInterfaceSunset.is_seed_dict(jt.manifest):
continue
jt.is_active = False
jt.is_paused = True
old_name = jt.name
old_name_version = jt.name + ' ' + jt.version
jt.name = 'legacy-' + jt.name.replace('_', '-')
if not jt.manifest:
jt.manifest = {}
input_files = []
input_json = []
output_files = []
global INTERFACE_NAME_COUNTER
INTERFACE_NAME_COUNTER = 0
for input in jt.manifest.get('input_data', []):
type = input.get('type', '')
if 'file' not in type:
json = {}
json['name'] = get_unique_name(input.get('name'))
json['type'] = 'string'
json['required'] = input.get('required', True)
input_json.append(json)
continue
file = {}
file['name'] = get_unique_name(input.get('name'))
file['required'] = input.get('required', True)
file['partial'] = input.get('partial', False)
file['mediaTypes'] = input.get('media_types', [])
file['multiple'] = (type == 'files')
input_files.append(file)
for output in jt.manifest.get('output_data', []):
type = output.get('type', '')
file = {}
file['name'] = get_unique_name(output.get('name'))
file['required'] = output.get('required', True)
file['mediaType'] = output.get('media_type', '')
file['multiple'] = (type == 'files')
file['pattern'] = "*.*"
output_files.append(file)
mounts = []
for mount in jt.manifest.get('mounts', []):
mt = {}
mt['name'] = get_unique_name(mount.get('name'))
mt['path'] = mount.get('path')
mt['mode'] = mount.get('mode', 'ro')
mounts.append(mt)
settings = []
for setting in jt.manifest.get('settings', []):
s = {}
s['name'] = get_unique_name(setting.get('name'))
s['secret'] = setting.get('secret', False)
settings.append(s)
for var in jt.manifest.get('env_vars', []):
s = {}
name = get_unique_name(var.get('name'))
name = 'ENV_' + name
s['name'] = name
settings.append(s)
errors = []
ec = jt.error_mapping.get('exit_codes', {})
for exit_code, error_name in ec.items():
error = {
'code': int(exit_code),
'name': get_unique_name(error_name),
'title': 'Error Name',
'description': 'Error Description',
'category': 'algorithm'
}
errors.append(error)
new_manifest = {
'seedVersion': '1.0.0',
'job': {
'name': jt.name,
'jobVersion': '0.0.0',
'packageVersion': '1.0.0',
'title': 'LEGACY ' + jt.title,
'description': jt.description,
'tags': [jt.category, old_name_version],
'maintainer': {
'name': jt.author_name,
'email': '*****@*****.**',
'url': jt.author_url
},
'timeout': jt.timeout,
'interface': {
'command': jt.manifest.get('command', ''),
'inputs': {
'files': input_files,
'json': input_json
},
'outputs': {
'files': output_files,
'json': []
},
'mounts': mounts,
'settings': settings
},
'resources': {
'scalar': [{
'name': 'cpus',
'value': jt.cpus_required
}, {
'name': 'mem',
'value': jt.mem_const_required,
'inputMultiplier': jt.mem_mult_required
}, {
'name': 'sharedMem',
'value': jt.shared_mem_required
}, {
'name': 'disk',
'value': jt.disk_out_const_required,
'inputMultiplier': jt.disk_out_mult_required
}]
},
'errors': errors
}
}
jt.manifest = new_manifest
SeedManifest(jt.manifest, do_validate=True)
jt.save()
for jtr in JobTypeRevision.objects.filter(
job_type_id=jt.id).iterator():
jtr.manifest = jt.manifest
jtr.save()
# Update any recipe types that reference the updated job name
for rtjl in RecipeTypeJobLink.objects.all().filter(
job_type_id=jt.id).iterator():
recipe_type = RecipeType.objects.get(id=rtjl.id)
definition = recipe_type.definition
changed = False
# v6
if 'nodes' in definition:
for node in definition['nodes']:
jt_node = node['node_type']
if jt_node['node_type'] == 'job' and jt_node[
'job_type_name'].replace(
'_', '-') == old_name and jt_node[
'job_type_version'] == jt.version:
node['node_type']['job_type_name'] = jt.name
changed = True
# v5
elif 'jobs' in definition:
for job in definition['jobs']:
jt_node = job['job_type']
if jt_node['name'].replace(
'_', '-'
) == old_name and jt_node['version'] == jt.version:
job['job_type']['name'] = jt.name
changed = True
if changed:
recipe_type.definition = definition
recipe_type.save()
def _perform_job_type_manifest_iteration(self):
"""Performs a single iteration of updating job type interfaces
"""
# Get job type ID
jt_qry = JobType.objects.all()
if self._current_job_type_id:
jt_qry = jt_qry.filter(id__gt=self._current_job_type_id)
for jt in jt_qry.order_by('id').only('id')[:1]:
jt_id = jt.id
break
jt = JobType.objects.get(pk=jt_id)
if not JobInterfaceSunset.is_seed_dict(jt.manifest):
jt.is_active = False
jt.is_paused = True
old_name_version = jt.name + ' ' + jt.version
jt.name = 'legacy-' + jt.name.replace('_', '-')
if not jt.manifest:
jt.manifest = {}
input_files = []
input_json = []
output_files = []
global INTERFACE_NAME_COUNTER
INTERFACE_NAME_COUNTER = 0
for input in jt.manifest.get('input_data', []):
type = input.get('type', '')
if 'file' not in type:
json = {}
json['name'] = get_unique_name(input.get('name'))
json['type'] = 'string'
json['required'] = input.get('required', True)
input_json.append(json)
continue
file = {}
file['name'] = get_unique_name(input.get('name'))
file['required'] = input.get('required', True)
file['partial'] = input.get('partial', False)
file['mediaTypes'] = input.get('media_types', [])
file['multiple'] = (type == 'files')
input_files.append(file)
for output in jt.manifest.get('output_data', []):
type = output.get('type', '')
file = {}
file['name'] = get_unique_name(output.get('name'))
file['required'] = output.get('required', True)
file['mediaType'] = output.get('media_type', '')
file['multiple'] = (type == 'files')
file['pattern'] = "*.*"
output_files.append(file)
mounts = []
for mount in jt.manifest.get('mounts', []):
mt = {}
mt['name'] = get_unique_name(mount.get('name'))
mt['path'] = mount.get('path')
mt['mode'] = mount.get('mode', 'ro')
mounts.append(mt)
settings = []
for setting in jt.manifest.get('settings', []):
s = {}
s['name'] = get_unique_name(setting.get('name'))
s['secret'] = setting.get('secret', False)
settings.append(s)
for var in jt.manifest.get('env_vars', []):
s = {}
name = get_unique_name(var.get('name'))
name = 'ENV_' + name
s['name'] = name
settings.append(s)
new_manifest = {
'seedVersion': '1.0.0',
'job': {
'name': jt.name,
'jobVersion': '0.0.0',
'packageVersion': '1.0.0',
'title': 'Legacy Title',
'description': 'legacy job type: ' + old_name_version,
'tags': [],
'maintainer': {
'name': 'Legacy',
'email': '*****@*****.**'
},
'timeout': 3600,
'interface': {
'command': jt.manifest.get('command', ''),
'inputs': {
'files': input_files,
'json': input_json
},
'outputs': {
'files': output_files,
'json': []
},
'mounts': mounts,
'settings': settings
},
'resources': {
'scalar': [
{ 'name': 'cpus', 'value': 1.0 },
{ 'name': 'mem', 'value': 1024.0 },
{ 'name': 'disk', 'value': 1000.0, 'inputMultiplier': 4.0 }
]
},
'errors': []
}
}
jt.manifest = new_manifest
SeedManifest(jt.manifest, do_validate=True)
jt.save()
for jtr in JobTypeRevision.objects.filter(job_type_id=jt.id).iterator():
jtr.manifest = jt.manifest
jtr.save()
self._current_job_type_id = jt_id
self._updated_job_type += 1
if self._updated_job_type > self._total_job_type:
self._updated_job_type = self._total_job_type
percent = (float(self._updated_job_type) / float(self._total_job_type)) * 100.00
logger.info('Completed %s of %s job types (%.1f%%)', self._updated_job_type, self._total_job_type, percent)
def patch(self, request, name, version):
"""Edits an existing seed job type and returns the updated details
:param request: the HTTP PATCH request
:type request: :class:`rest_framework.request.Request`
:param job_type_id: The ID for the job type.
:type job_type_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
auto_update = rest_util.parse_bool(request, 'auto_update', required=False, default_value=True)
icon_code = rest_util.parse_string(request, 'icon_code', required=False)
is_published = rest_util.parse_string(request, 'is_published', required=False)
is_active = rest_util.parse_bool(request, 'is_active', required=False)
is_paused = rest_util.parse_bool(request, 'is_paused', required=False)
max_scheduled = rest_util.parse_int(request, 'max_scheduled', required=False)
docker_image = rest_util.parse_string(request, 'docker_image', required=False)
# Validate the manifest
manifest_dict = rest_util.parse_dict(request, 'manifest', required=False)
manifest = None
if manifest_dict:
try:
manifest = SeedManifest(manifest_dict, do_validate=True)
except InvalidSeedManifestDefinition as ex:
message = 'Seed Manifest invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
# validate manifest name/version matches job type
manifest_name = manifest.get_name()
if name != manifest_name:
raise BadParameter('Manifest name %s does not match current Job Type name %s.' % (manifest_name, name))
manifest_version = manifest.get_job_version()
if manifest_version != version:
raise BadParameter('Manifest version %s does not match current Job Type version %s.' % (manifest_version, version))
# Validate the job configuration and pull out secrets
configuration_dict = rest_util.parse_dict(request, 'configuration', required=False)
configuration = None
try:
if configuration_dict:
configuration = JobConfigurationV6(configuration_dict).get_configuration()
except InvalidJobConfiguration as ex:
raise BadParameter('Job type configuration invalid: %s' % unicode(ex))
# Fetch the current job type model
try:
job_type = JobType.objects.get(name=name, version=version)
except JobType.DoesNotExist:
raise Http404
# Check for invalid fields
fields = {'icon_code', 'is_published', 'is_active', 'is_paused', 'max_scheduled', 'configuration', 'manifest',
'docker_image', 'auto_update'}
for key, value in request.data.iteritems():
if key not in fields:
raise BadParameter('%s is not a valid field. Valid fields are: %s' % (key, fields))
try:
with transaction.atomic():
# Edit the job type
validation = JobType.objects.edit_job_type_v6(job_type_id=job_type.id, manifest=manifest, is_published=is_published,
docker_image=docker_image, icon_code=icon_code, is_active=is_active,
is_paused=is_paused, max_scheduled=max_scheduled,
configuration=configuration, auto_update=auto_update)
except (InvalidJobField, InvalidSecretsConfiguration, ValueError,
InvalidJobConfiguration, InvalidInterfaceDefinition) as ex:
logger.exception('Unable to update job type: %i', job_type.id)
raise BadParameter(unicode(ex))
resp_dict = {'is_valid': validation.is_valid, 'errors': [e.to_dict() for e in validation.errors],
'warnings': [w.to_dict() for w in validation.warnings]}
return Response(resp_dict)
def create_v6(self, request):
"""Creates or edits a Seed job type and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
# Optional icon code value
icon_code = rest_util.parse_string(request, 'icon_code', required=False)
# Optional is published value
is_published = rest_util.parse_string(request, 'is_published', required=False)
# Optional max scheduled value
max_scheduled = rest_util.parse_int(request, 'max_scheduled', required=False)
# Require docker image value
docker_image = rest_util.parse_string(request, 'docker_image', required=True)
# Validate the job interface / manifest
manifest_dict = rest_util.parse_dict(request, 'manifest', required=True)
# If editing an existing job type, automatically update recipes containing said job type
auto_update = rest_util.parse_bool(request, 'auto_update', required=False, default_value=True)
# Optional setting job type active if editing existing job
is_active = rest_util.parse_bool(request, 'is_active', required=False)
# Optional setting job type to paused if editing an existing job
is_paused = rest_util.parse_bool(request, 'is_paused', required=False)
manifest = None
try:
manifest = SeedManifest(manifest_dict, do_validate=True)
except InvalidSeedManifestDefinition as ex:
message = 'Seed Manifest invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
# Validate the job configuration and pull out secrets
configuration_dict = rest_util.parse_dict(request, 'configuration', required=False)
configuration = None
if configuration_dict:
try:
configuration = JobConfigurationV6(configuration_dict, do_validate=True).get_configuration()
except InvalidJobConfiguration as ex:
message = 'Job type configuration invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
# Check for invalid fields
fields = {'icon_code', 'is_published', 'max_scheduled', 'docker_image', 'configuration', 'manifest',
'auto_update', 'is_active', 'is_paused'}
for key, value in request.data.iteritems():
if key not in fields:
raise BadParameter('%s is not a valid field. Valid fields are: %s' % (key, fields))
name = manifest_dict['job']['name']
version = manifest_dict['job']['jobVersion']
if name == 'validation':
logger.exception('Unable to create job type named "validation"')
raise BadParameter(unicode('Unable to create job type named "validation"'))
existing_job_type = JobType.objects.filter(name=name, version=version).first()
if not existing_job_type:
try:
# Create the job type
job_type = JobType.objects.create_job_type_v6(icon_code=icon_code,
is_published=is_published,
max_scheduled=max_scheduled,
docker_image=docker_image,
manifest=manifest,
configuration=configuration)
except (InvalidJobField, InvalidSecretsConfiguration, ValueError) as ex:
message = 'Unable to create new job type'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
except InvalidSeedManifestDefinition as ex:
message = 'Job type manifest invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
except InvalidJobConfiguration as ex:
message = 'Job type configuration invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
# Fetch the full job type with details
try:
job_type = JobType.objects.get_details_v6(name=name, version=version)
except JobType.DoesNotExist:
raise Http404
url = reverse('job_type_details_view', args=[job_type.name, job_type.version], request=request)
serializer = JobTypeDetailsSerializerV6(job_type)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
else:
try:
validation = JobType.objects.edit_job_type_v6(job_type_id=existing_job_type.id, manifest=manifest,
docker_image=docker_image, icon_code=icon_code, is_active=is_active,
is_paused=is_paused, max_scheduled=max_scheduled,
is_published=is_published, configuration=configuration,
auto_update=auto_update)
except (InvalidJobField, InvalidSecretsConfiguration, ValueError, InvalidInterfaceDefinition) as ex:
logger.exception('Unable to update job type: %i', existing_job_type.id)
raise BadParameter(unicode(ex))
except InvalidSeedManifestDefinition as ex:
message = 'Job type manifest invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
except InvalidJobConfiguration as ex:
message = 'Job type configuration invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
resp_dict = {'is_valid': validation.is_valid, 'errors': [e.to_dict() for e in validation.errors],
'warnings': [w.to_dict() for w in validation.warnings]}
return Response(resp_dict)
def test_validate_settings(self):
"""Tests calling JobConfiguration.validate() to validate settings configuration"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'settings': [{
'name': 'setting_a'
}, {
'name': 'secret_setting_a',
'secret': True
}, {
'name': 'secret_setting_b',
'secret': True
}, {
'name': 'secret_setting_c',
'secret': True
}]
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
configuration.add_setting('setting_a', 'value_1')
configuration.add_setting('secret_setting_a', 'secret_value_1')
configuration.add_setting('secret_setting_b', 'secret_value_2')
configuration.add_setting('secret_setting_c', 'secret_value_3')
configuration.add_setting('setting_4', 'value_4')
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].name, 'UNKNOWN_SETTING')
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'settings': [{
'name': 'setting_a'
}, {
'name': 'secret_setting_a',
'secret': True
}, {
'name': 'secret_setting_b',
'secret': True
}, {
'name': 'secret_setting_c',
'secret': True
}]
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
configuration.add_setting('setting_a', 'value_1')
configuration.add_setting('secret_setting_a', 'secret_value_1')
configuration.add_setting('secret_setting_b', 'secret_value_2')
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].name, 'MISSING_SETTING')
def configure_queued_job(self, job):
"""Creates and returns an execution configuration for the given queued job. The given job model should have its
related job_type, job_type_rev, and batch models populated.
:param job: The queued job model
:type job: :class:`job.models.Job`
:returns: The execution configuration for the queued job
:rtype: :class:`job.execution.configuration.json.exe_config.ExecutionConfiguration`
"""
config = ExecutionConfiguration()
data = job.get_job_data()
# Add input file meta-data
input_files_dict = self._create_input_file_dict(data)
config.set_input_files(input_files_dict)
# Set up env vars for job's input data
input_values = data.get_injected_input_values(input_files_dict)
interface = job.job_type_rev.get_input_interface()
env_vars = {}
if isinstance(data, JobData):
# call job.data.job_data.JobData.get_injected_env_vars
env_vars = data.get_injected_env_vars(input_files_dict, interface)
else:
# call old job.configuration.data.job_data.get_injected_env_vars
# TODO: remove once old JobData class is no longer used
env_vars = data.get_injected_env_vars(input_files_dict)
task_workspaces = {}
if job.job_type.is_system:
# Add any workspaces needed for this system job
task_workspaces = QueuedExecutionConfigurator._system_job_workspaces(
job)
else:
# Set any output workspaces needed
output_workspaces = {}
if job.input and 'version' in job.input and job.input[
'version'] == '1.0':
# Set output workspaces using legacy job data
self._cache_workspace_names(data.get_output_workspace_ids())
output_workspaces = {}
for output, workspace_id in data.get_output_workspaces().items(
):
output_workspaces[output] = self._cached_workspace_names[
workspace_id]
config.set_output_workspaces(output_workspaces)
if not output_workspaces:
# Set output workspaces from job configuration
output_workspaces = {}
job_config = job.get_job_configuration()
interface = SeedManifest(job.job_type_rev.manifest,
do_validate=False)
for output_name in interface.get_file_output_names():
output_workspace = job_config.get_output_workspace(
output_name)
if output_workspace:
output_workspaces[output_name] = output_workspace
config.set_output_workspaces(output_workspaces)
# Create main task with fields populated from input data
args = job.get_job_interface().get_injected_command_args(
input_values, env_vars)
config.create_tasks(['main'])
config.add_to_task('main',
args=args,
env_vars=env_vars,
workspaces=task_workspaces)
return config
def test_validate_output_workspaces(self):
"""Tests calling JobConfiguration.validate() to validate output workspaces"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'outputs': {
'files': [{
'name': 'output_a',
'mediaType': 'image/gif',
'pattern': '*_a.gif'
}, {
'name': 'output_b',
'mediaType': 'image/gif',
'pattern': '*_a.gif'
}]
}
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
# No workspaces defined
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 2)
self.assertEqual(warnings[0].name, 'MISSING_WORKSPACE')
self.assertEqual(warnings[1].name, 'MISSING_WORKSPACE')
# Workspace does not exist
configuration.default_output_workspace = 'workspace_1'
with self.assertRaises(InvalidJobConfiguration) as context:
configuration.validate(manifest)
self.assertEqual(context.exception.error.name, 'INVALID_WORKSPACE')
# Default workspace defined with valid workspace
workspace_1 = storage_test_utils.create_workspace(name='workspace_1')
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 0)
# Workspace is only defined for output_a
configuration.default_output_workspace = None
configuration.add_output_workspace('output_a', 'workspace_1')
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].name, 'MISSING_WORKSPACE')
# Workspace defined for both outputs
storage_test_utils.create_workspace(name='workspace_2')
configuration.add_output_workspace('output_b', 'workspace_2')
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 0)
# Workspace is deprecated
workspace_1.is_active = False
workspace_1.save()
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].name, 'DEPRECATED_WORKSPACE')
def test_validate_mounts(self):
"""Tests calling JobConfiguration.validate() to validate mount configuration"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'mounts': [{
'name': 'mount_a',
'path': '/the/a/path'
}, {
'name': 'mount_b',
'path': '/the/b/path'
}, {
'name': 'mount_c',
'path': '/the/c/path'
}]
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
configuration.add_mount(HostMountConfig('mount_a', '/the/host/a/path'))
configuration.add_mount(HostMountConfig('mount_b', '/the/host/b/path'))
configuration.add_mount(HostMountConfig('mount_c', '/the/host/c/path'))
configuration.add_mount(HostMountConfig('mount_d', '/the/host/d/path'))
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].name, 'UNKNOWN_MOUNT')
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'mounts': [{
'name': 'mount_a',
'path': '/the/a/path'
}, {
'name': 'mount_b',
'path': '/the/b/path'
}, {
'name': 'mount_c',
'path': '/the/c/path'
}]
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
configuration.add_mount(HostMountConfig('mount_a', '/the/host/a/path'))
configuration.add_mount(HostMountConfig('mount_b', '/the/host/b/path'))
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].name, 'MISSING_MOUNT')