def test_merge_recipe_config(self): """Tests calling JobConfiguration.merge_recipe_config()""" configuration = JobConfiguration() host_mount = HostMountConfig('mount_1', '/the/host/path') configuration.add_mount(host_mount) configuration.add_output_workspace('output_1', 'workspace_1') configuration.add_setting('setting_1', 'value_1') recipe_config = { 'version': '6', 'mounts': { 'mount_1': { 'type': 'host', 'host_path': '/the/host/path2' }, 'mount_2': { 'type': 'volume', 'driver': 'driver', 'driver_opts': { 'opt_1': 'foo', 'opt_2': 'bar' } } }, 'output_workspaces': { 'default': 'workspace_1', 'outputs': { 'output': 'workspace_2' } }, 'priority': 999, 'settings': {} } configuration.merge_recipe_config(recipe_config) self.assertEqual(configuration.default_output_workspace, 'workspace_1') self.assertEqual(configuration.priority, 999) self.assertDictEqual(configuration.output_workspaces, { 'output': 'workspace_2', 'output_1': 'workspace_1' }) self.assertItemsEqual(configuration.mounts.keys(), ['mount_1', 'mount_2']) self.assertEqual(configuration.mounts['mount_1'].host_path, '/the/host/path2') self.assertDictEqual(configuration.mounts['mount_2'].driver_opts, { 'opt_1': 'foo', 'opt_2': 'bar' }) self.assertDictEqual(configuration.settings, {'setting_1': 'value_1'})
def test_store_output_files(self, dummy_store, isfile): workspace = storage_test_utils.create_workspace() files = {'OUTPUT_TIFFS': [ProductFileMetadata('OUTPUT_TIFFS', 'outfile0.tif', media_type='image/tiff')]} job_data = JobData({}) job_config = JobConfiguration() job_config.add_output_workspace('OUTPUT_TIFFS', workspace.name) job_exe = Mock() job_exe.job_type.get_job_configuration.return_value = job_config results = JobResults()._store_output_data_files(files, job_data, job_exe) self.assertEqual({'OUTPUT_TIFFS': [1]}, results.files)
def test_remove_secret_settings(self): """Tests calling JobConfiguration.remove_secret_settings()""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'settings': [{ 'name': 'setting_a' }, { 'name': 'secret_setting_a', 'secret': True }, { 'name': 'secret_setting_b', 'secret': True }, { 'name': 'secret_setting_c', 'secret': True }] } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() configuration.add_setting('setting_a', 'value_1') configuration.add_setting('secret_setting_a', 'secret_value_1') configuration.add_setting('secret_setting_b', 'secret_value_2') configuration.add_setting('setting_d', 'value_4') secret_settings = configuration.remove_secret_settings(manifest) self.assertDictEqual( secret_settings, { 'secret_setting_a': 'secret_value_1', 'secret_setting_b': 'secret_value_2' }) self.assertDictEqual(configuration.settings, { 'setting_a': 'value_1', 'setting_d': 'value_4' })
def test_add_setting(self): """Tests calling JobConfiguration.add_setting()""" configuration = JobConfiguration() configuration.add_setting('setting_1', 'value_1') with self.assertRaises(InvalidJobConfiguration) as context: configuration.add_setting('setting_1', 'value_2') self.assertEqual(context.exception.error.name, 'DUPLICATE_SETTING') with self.assertRaises(InvalidJobConfiguration) as context: configuration.add_setting('setting_2', None) self.assertEqual(context.exception.error.name, 'INVALID_SETTING')
def test_no_default_workspace(self): """Tests calling JobConfiguration.validate() to validate output workspaces""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'outputs': { 'files': [{ 'name': 'output_a', 'mediaType': 'image/gif', 'pattern': '*_a.gif' }, { 'name': 'output_b', 'mediaType': 'image/gif', 'pattern': '*_a.gif' }] } } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() # No workspaces defined for outputs warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 2) self.assertEqual(warnings[0].name, 'MISSING_WORKSPACE') self.assertEqual(warnings[1].name, 'MISSING_WORKSPACE')
def test_add_output_workspace(self): """Tests calling JobConfiguration.add_output_workspace()""" configuration = JobConfiguration() configuration.add_output_workspace('output_1', 'workspace_1') with self.assertRaises(InvalidJobConfiguration) as context: configuration.add_output_workspace('output_1', 'workspace_2') self.assertEqual(context.exception.error.name, 'DUPLICATE_WORKSPACE')
def test_validate_priority(self): """Tests calling JobConfiguration.validate() to validate priority""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10 } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() configuration.priority = 100 warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 0) configuration.priority = 0 with self.assertRaises(InvalidJobConfiguration) as context: configuration.validate(manifest) self.assertEqual(context.exception.error.name, 'INVALID_PRIORITY') configuration.priority = -1 with self.assertRaises(InvalidJobConfiguration) as context: configuration.validate(manifest) self.assertEqual(context.exception.error.name, 'INVALID_PRIORITY')
def test_add_mount(self): """Tests calling JobConfiguration.add_mount()""" configuration = JobConfiguration() host_mount = HostMountConfig('mount_1', '/the/host/path') configuration.add_mount(host_mount) vol_mount = VolumeMountConfig('mount_1', driver='driver', driver_opts={}) with self.assertRaises(InvalidJobConfiguration) as context: configuration.add_mount(vol_mount) self.assertEqual(context.exception.error.name, 'DUPLICATE_MOUNT')
def test_validate_settings(self): """Tests calling JobConfiguration.validate() to validate settings configuration""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'settings': [{ 'name': 'setting_a' }, { 'name': 'secret_setting_a', 'secret': True }, { 'name': 'secret_setting_b', 'secret': True }, { 'name': 'secret_setting_c', 'secret': True }] } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() configuration.add_setting('setting_a', 'value_1') configuration.add_setting('secret_setting_a', 'secret_value_1') configuration.add_setting('secret_setting_b', 'secret_value_2') configuration.add_setting('secret_setting_c', 'secret_value_3') configuration.add_setting('setting_4', 'value_4') warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].name, 'UNKNOWN_SETTING') manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'settings': [{ 'name': 'setting_a' }, { 'name': 'secret_setting_a', 'secret': True }, { 'name': 'secret_setting_b', 'secret': True }, { 'name': 'secret_setting_c', 'secret': True }] } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() configuration.add_setting('setting_a', 'value_1') configuration.add_setting('secret_setting_a', 'secret_value_1') configuration.add_setting('secret_setting_b', 'secret_value_2') warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].name, 'MISSING_SETTING')
def test_validate_output_workspaces(self): """Tests calling JobConfiguration.validate() to validate output workspaces""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'outputs': { 'files': [{ 'name': 'output_a', 'mediaType': 'image/gif', 'pattern': '*_a.gif' }, { 'name': 'output_b', 'mediaType': 'image/gif', 'pattern': '*_a.gif' }] } } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() # No workspaces defined warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 2) self.assertEqual(warnings[0].name, 'MISSING_WORKSPACE') self.assertEqual(warnings[1].name, 'MISSING_WORKSPACE') # Workspace does not exist configuration.default_output_workspace = 'workspace_1' with self.assertRaises(InvalidJobConfiguration) as context: configuration.validate(manifest) self.assertEqual(context.exception.error.name, 'INVALID_WORKSPACE') # Default workspace defined with valid workspace workspace_1 = storage_test_utils.create_workspace(name='workspace_1') warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 0) # Workspace is only defined for output_a configuration.default_output_workspace = None configuration.add_output_workspace('output_a', 'workspace_1') warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].name, 'MISSING_WORKSPACE') # Workspace defined for both outputs storage_test_utils.create_workspace(name='workspace_2') configuration.add_output_workspace('output_b', 'workspace_2') warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 0) # Workspace is deprecated workspace_1.is_active = False workspace_1.save() warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].name, 'DEPRECATED_WORKSPACE')
def test_validate_mounts(self): """Tests calling JobConfiguration.validate() to validate mount configuration""" manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'mounts': [{ 'name': 'mount_a', 'path': '/the/a/path' }, { 'name': 'mount_b', 'path': '/the/b/path' }, { 'name': 'mount_c', 'path': '/the/c/path' }] } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() configuration.add_mount(HostMountConfig('mount_a', '/the/host/a/path')) configuration.add_mount(HostMountConfig('mount_b', '/the/host/b/path')) configuration.add_mount(HostMountConfig('mount_c', '/the/host/c/path')) configuration.add_mount(HostMountConfig('mount_d', '/the/host/d/path')) warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].name, 'UNKNOWN_MOUNT') manifest_dict = { 'seedVersion': '1.0.0', 'job': { 'name': 'random-number-gen', 'jobVersion': '0.1.0', 'packageVersion': '0.1.0', 'title': 'Random Number Generator', 'description': 'Generates a random number and outputs on stdout', 'maintainer': { 'name': 'John Doe', 'email': '*****@*****.**' }, 'timeout': 10, 'interface': { 'mounts': [{ 'name': 'mount_a', 'path': '/the/a/path' }, { 'name': 'mount_b', 'path': '/the/b/path' }, { 'name': 'mount_c', 'path': '/the/c/path' }] } } } manifest = SeedManifest(manifest_dict) configuration = JobConfiguration() configuration.add_mount(HostMountConfig('mount_a', '/the/host/a/path')) configuration.add_mount(HostMountConfig('mount_b', '/the/host/b/path')) warnings = configuration.validate(manifest) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0].name, 'MISSING_MOUNT')
def get_configuration(self): """Returns the job configuration represented by this JSON :returns: The job configuration :rtype: :class:`job.configuration.configuration.JobConfiguration`: """ config = JobConfiguration() for name, mount_dict in self._config['mounts'].items(): if mount_dict['type'] == 'host': config.add_mount(HostMountConfig(name, mount_dict['host_path'])) elif mount_dict['type'] == 'volume': config.add_mount( VolumeMountConfig(name, mount_dict['driver'], mount_dict['driver_opts'])) default_workspace = self._config['output_workspaces']['default'] if default_workspace: config.default_output_workspace = default_workspace for output, workspace in self._config['output_workspaces'][ 'outputs'].items(): config.add_output_workspace(output, workspace) config.priority = self._config['priority'] for name, value in self._config['settings'].items(): config.add_setting(name, value) return config
def test_convert_config_to_v2_json(self): """Tests calling convert_config_to_v2_json()""" # Try configuration with nothing set config = JobConfiguration() json = convert_config_to_v2_json(config) JobConfigurationV2(configuration=json.get_dict(), do_validate=True) # Revalidate # Try configuration with a variety of values config = JobConfiguration() config.add_mount(HostMountConfig('mount_1', '/the/host/path')) config.add_mount( VolumeMountConfig('mount_2', 'driver', { 'opt_1': 'foo', 'opt_2': 'bar' })) config.default_output_workspace = 'workspace_1' config.add_output_workspace('output_2', 'workspace_2') config.priority = 999 config.add_setting('setting_1', 'Hello') config.add_setting('setting_2', 'Scale!') json = convert_config_to_v2_json(config) JobConfigurationV2(configuration=json.get_dict(), do_validate=True) # Revalidate self.assertSetEqual(set(json.get_dict()['mounts'].keys()), {'mount_1', 'mount_2'}) self.assertSetEqual(set(json.get_dict()['settings'].keys()), {'setting_1', 'setting_2'})