def test_merge_recipe_config(self):
"""Tests calling JobConfiguration.merge_recipe_config()"""
configuration = JobConfiguration()
host_mount = HostMountConfig('mount_1', '/the/host/path')
configuration.add_mount(host_mount)
configuration.add_output_workspace('output_1', 'workspace_1')
configuration.add_setting('setting_1', 'value_1')
recipe_config = {
'version': '6',
'mounts': {
'mount_1': {
'type': 'host',
'host_path': '/the/host/path2'
},
'mount_2': {
'type': 'volume',
'driver': 'driver',
'driver_opts': {
'opt_1': 'foo',
'opt_2': 'bar'
}
}
},
'output_workspaces': {
'default': 'workspace_1',
'outputs': {
'output': 'workspace_2'
}
},
'priority': 999,
'settings': {}
}
configuration.merge_recipe_config(recipe_config)
self.assertEqual(configuration.default_output_workspace, 'workspace_1')
self.assertEqual(configuration.priority, 999)
self.assertDictEqual(configuration.output_workspaces, {
'output': 'workspace_2',
'output_1': 'workspace_1'
})
self.assertItemsEqual(configuration.mounts.keys(),
['mount_1', 'mount_2'])
self.assertEqual(configuration.mounts['mount_1'].host_path,
'/the/host/path2')
self.assertDictEqual(configuration.mounts['mount_2'].driver_opts, {
'opt_1': 'foo',
'opt_2': 'bar'
})
self.assertDictEqual(configuration.settings, {'setting_1': 'value_1'})
def test_store_output_files(self, dummy_store, isfile):
workspace = storage_test_utils.create_workspace()
files = {'OUTPUT_TIFFS': [ProductFileMetadata('OUTPUT_TIFFS', 'outfile0.tif', media_type='image/tiff')]}
job_data = JobData({})
job_config = JobConfiguration()
job_config.add_output_workspace('OUTPUT_TIFFS', workspace.name)
job_exe = Mock()
job_exe.job_type.get_job_configuration.return_value = job_config
results = JobResults()._store_output_data_files(files, job_data, job_exe)
self.assertEqual({'OUTPUT_TIFFS': [1]}, results.files)
def test_remove_secret_settings(self):
"""Tests calling JobConfiguration.remove_secret_settings()"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'settings': [{
'name': 'setting_a'
}, {
'name': 'secret_setting_a',
'secret': True
}, {
'name': 'secret_setting_b',
'secret': True
}, {
'name': 'secret_setting_c',
'secret': True
}]
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
configuration.add_setting('setting_a', 'value_1')
configuration.add_setting('secret_setting_a', 'secret_value_1')
configuration.add_setting('secret_setting_b', 'secret_value_2')
configuration.add_setting('setting_d', 'value_4')
secret_settings = configuration.remove_secret_settings(manifest)
self.assertDictEqual(
secret_settings, {
'secret_setting_a': 'secret_value_1',
'secret_setting_b': 'secret_value_2'
})
self.assertDictEqual(configuration.settings, {
'setting_a': 'value_1',
'setting_d': 'value_4'
})
def test_add_setting(self):
"""Tests calling JobConfiguration.add_setting()"""
configuration = JobConfiguration()
configuration.add_setting('setting_1', 'value_1')
with self.assertRaises(InvalidJobConfiguration) as context:
configuration.add_setting('setting_1', 'value_2')
self.assertEqual(context.exception.error.name, 'DUPLICATE_SETTING')
with self.assertRaises(InvalidJobConfiguration) as context:
configuration.add_setting('setting_2', None)
self.assertEqual(context.exception.error.name, 'INVALID_SETTING')
def test_no_default_workspace(self):
"""Tests calling JobConfiguration.validate() to validate output workspaces"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'outputs': {
'files': [{
'name': 'output_a',
'mediaType': 'image/gif',
'pattern': '*_a.gif'
}, {
'name': 'output_b',
'mediaType': 'image/gif',
'pattern': '*_a.gif'
}]
}
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
# No workspaces defined for outputs
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 2)
self.assertEqual(warnings[0].name, 'MISSING_WORKSPACE')
self.assertEqual(warnings[1].name, 'MISSING_WORKSPACE')
def test_add_output_workspace(self):
"""Tests calling JobConfiguration.add_output_workspace()"""
configuration = JobConfiguration()
configuration.add_output_workspace('output_1', 'workspace_1')
with self.assertRaises(InvalidJobConfiguration) as context:
configuration.add_output_workspace('output_1', 'workspace_2')
self.assertEqual(context.exception.error.name, 'DUPLICATE_WORKSPACE')
def test_validate_priority(self):
"""Tests calling JobConfiguration.validate() to validate priority"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
configuration.priority = 100
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 0)
configuration.priority = 0
with self.assertRaises(InvalidJobConfiguration) as context:
configuration.validate(manifest)
self.assertEqual(context.exception.error.name, 'INVALID_PRIORITY')
configuration.priority = -1
with self.assertRaises(InvalidJobConfiguration) as context:
configuration.validate(manifest)
self.assertEqual(context.exception.error.name, 'INVALID_PRIORITY')
def test_add_mount(self):
"""Tests calling JobConfiguration.add_mount()"""
configuration = JobConfiguration()
host_mount = HostMountConfig('mount_1', '/the/host/path')
configuration.add_mount(host_mount)
vol_mount = VolumeMountConfig('mount_1',
driver='driver',
driver_opts={})
with self.assertRaises(InvalidJobConfiguration) as context:
configuration.add_mount(vol_mount)
self.assertEqual(context.exception.error.name, 'DUPLICATE_MOUNT')
def test_validate_settings(self):
"""Tests calling JobConfiguration.validate() to validate settings configuration"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'settings': [{
'name': 'setting_a'
}, {
'name': 'secret_setting_a',
'secret': True
}, {
'name': 'secret_setting_b',
'secret': True
}, {
'name': 'secret_setting_c',
'secret': True
}]
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
configuration.add_setting('setting_a', 'value_1')
configuration.add_setting('secret_setting_a', 'secret_value_1')
configuration.add_setting('secret_setting_b', 'secret_value_2')
configuration.add_setting('secret_setting_c', 'secret_value_3')
configuration.add_setting('setting_4', 'value_4')
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].name, 'UNKNOWN_SETTING')
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'settings': [{
'name': 'setting_a'
}, {
'name': 'secret_setting_a',
'secret': True
}, {
'name': 'secret_setting_b',
'secret': True
}, {
'name': 'secret_setting_c',
'secret': True
}]
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
configuration.add_setting('setting_a', 'value_1')
configuration.add_setting('secret_setting_a', 'secret_value_1')
configuration.add_setting('secret_setting_b', 'secret_value_2')
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].name, 'MISSING_SETTING')
def test_validate_output_workspaces(self):
"""Tests calling JobConfiguration.validate() to validate output workspaces"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'outputs': {
'files': [{
'name': 'output_a',
'mediaType': 'image/gif',
'pattern': '*_a.gif'
}, {
'name': 'output_b',
'mediaType': 'image/gif',
'pattern': '*_a.gif'
}]
}
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
# No workspaces defined
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 2)
self.assertEqual(warnings[0].name, 'MISSING_WORKSPACE')
self.assertEqual(warnings[1].name, 'MISSING_WORKSPACE')
# Workspace does not exist
configuration.default_output_workspace = 'workspace_1'
with self.assertRaises(InvalidJobConfiguration) as context:
configuration.validate(manifest)
self.assertEqual(context.exception.error.name, 'INVALID_WORKSPACE')
# Default workspace defined with valid workspace
workspace_1 = storage_test_utils.create_workspace(name='workspace_1')
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 0)
# Workspace is only defined for output_a
configuration.default_output_workspace = None
configuration.add_output_workspace('output_a', 'workspace_1')
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].name, 'MISSING_WORKSPACE')
# Workspace defined for both outputs
storage_test_utils.create_workspace(name='workspace_2')
configuration.add_output_workspace('output_b', 'workspace_2')
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 0)
# Workspace is deprecated
workspace_1.is_active = False
workspace_1.save()
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].name, 'DEPRECATED_WORKSPACE')
def test_validate_mounts(self):
"""Tests calling JobConfiguration.validate() to validate mount configuration"""
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'mounts': [{
'name': 'mount_a',
'path': '/the/a/path'
}, {
'name': 'mount_b',
'path': '/the/b/path'
}, {
'name': 'mount_c',
'path': '/the/c/path'
}]
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
configuration.add_mount(HostMountConfig('mount_a', '/the/host/a/path'))
configuration.add_mount(HostMountConfig('mount_b', '/the/host/b/path'))
configuration.add_mount(HostMountConfig('mount_c', '/the/host/c/path'))
configuration.add_mount(HostMountConfig('mount_d', '/the/host/d/path'))
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].name, 'UNKNOWN_MOUNT')
manifest_dict = {
'seedVersion': '1.0.0',
'job': {
'name': 'random-number-gen',
'jobVersion': '0.1.0',
'packageVersion': '0.1.0',
'title': 'Random Number Generator',
'description':
'Generates a random number and outputs on stdout',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'mounts': [{
'name': 'mount_a',
'path': '/the/a/path'
}, {
'name': 'mount_b',
'path': '/the/b/path'
}, {
'name': 'mount_c',
'path': '/the/c/path'
}]
}
}
}
manifest = SeedManifest(manifest_dict)
configuration = JobConfiguration()
configuration.add_mount(HostMountConfig('mount_a', '/the/host/a/path'))
configuration.add_mount(HostMountConfig('mount_b', '/the/host/b/path'))
warnings = configuration.validate(manifest)
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].name, 'MISSING_MOUNT')
def get_configuration(self):
"""Returns the job configuration represented by this JSON
:returns: The job configuration
:rtype: :class:`job.configuration.configuration.JobConfiguration`:
"""
config = JobConfiguration()
for name, mount_dict in self._config['mounts'].items():
if mount_dict['type'] == 'host':
config.add_mount(HostMountConfig(name,
mount_dict['host_path']))
elif mount_dict['type'] == 'volume':
config.add_mount(
VolumeMountConfig(name, mount_dict['driver'],
mount_dict['driver_opts']))
default_workspace = self._config['output_workspaces']['default']
if default_workspace:
config.default_output_workspace = default_workspace
for output, workspace in self._config['output_workspaces'][
'outputs'].items():
config.add_output_workspace(output, workspace)
config.priority = self._config['priority']
for name, value in self._config['settings'].items():
config.add_setting(name, value)
return config
def test_convert_config_to_v2_json(self):
"""Tests calling convert_config_to_v2_json()"""
# Try configuration with nothing set
config = JobConfiguration()
json = convert_config_to_v2_json(config)
JobConfigurationV2(configuration=json.get_dict(),
do_validate=True) # Revalidate
# Try configuration with a variety of values
config = JobConfiguration()
config.add_mount(HostMountConfig('mount_1', '/the/host/path'))
config.add_mount(
VolumeMountConfig('mount_2', 'driver', {
'opt_1': 'foo',
'opt_2': 'bar'
}))
config.default_output_workspace = 'workspace_1'
config.add_output_workspace('output_2', 'workspace_2')
config.priority = 999
config.add_setting('setting_1', 'Hello')
config.add_setting('setting_2', 'Scale!')
json = convert_config_to_v2_json(config)
JobConfigurationV2(configuration=json.get_dict(),
do_validate=True) # Revalidate
self.assertSetEqual(set(json.get_dict()['mounts'].keys()),
{'mount_1', 'mount_2'})
self.assertSetEqual(set(json.get_dict()['settings'].keys()),
{'setting_1', 'setting_2'})
