def test_pass_two_env(self): """ Test that it pass two env variables correctly with undefined runner """ _tool_label = '_unittest_tool' _tool_spec = { _tool_label: { 'env': { 'name1': 'value1', 'name2': "value2" } } } self.td[_tool_label] = _tool_spec[_tool_label] result = [{'name': 'name1', 'value': 'value1'}, {'name': 'name2', 'value': 'value2'}] tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, self.td, []) env, params, runner, tags = build_spec(tool_spec, dest_spec=SPECIFICATIONS) d1 = {n['name']: n['value'] for n in env if n in ['name1', 'name2']} d2 = {n['name']: n['value'] for n in result} for k, v in d1.items(): self.assertEqual(d1[k], d2[k])
def test_runner_undefined(self): """ Test no runner specification, should reply with the DEFAULT_DESTINATION """ _tool_label = '_unittest_tool' _dest_label = 'condor_unittest_destination' _tool_spec = { _tool_label: {} } _dest_spec = { _dest_label: { 'env': {}, 'params': { 'request_cpus': '{PARALLELISATION}', 'request_memory': '{MEMORY}' } } } self.td[_tool_label] = _tool_spec[_tool_label] self.sp[_dest_label] = _dest_spec[_dest_label] result = DEFAULT_DESTINATION tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, self.td, []) _, _, runner, _ = build_spec(tool_spec, dest_spec=self.sp) self.assertEqual(runner, result)
def test_pass_param_cores_with_default_destination(self): """ Test tool with defined cores and destination by default 'request_cpus' has to be in params with the same value provided by the tool """ _tool_label = '_unittest_tool' _tool_spec = { _tool_label: { 'cores': 3 } } self.td[_tool_label] = _tool_spec[_tool_label] result = { 'params': {'request_cpus': '3'} } tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, self.td, []) _, params, _, _ = build_spec(tool_spec, dest_spec=self.sp) self.assertEqual(params['request_cpus'], result['params']['request_cpus'])
def test_default_values_setup(self): """ Test if default values will be assigned in case of a tool without specifications :return: """ _tool_label = '_unittest_tool' _tool_spec = {_tool_label: {}} TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] result = { 'cores': DEFAULT_TOOL_SPEC['cores'], 'mem': DEFAULT_TOOL_SPEC['mem'], 'gpus': DEFAULT_TOOL_SPEC['gpus'], 'force_destination_id': DEFAULT_TOOL_SPEC['force_destination_id'], 'runner': DEFAULT_TOOL_SPEC['runner'] } tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) for i in result: if i in ['cores', 'gpus']: self.assertIsInstance(tool_spec[i], int) elif i == 'mem': self.assertIsInstance(tool_spec[i], float) elif i == 'force_destination_id': self.assertIsInstance(tool_spec[i], bool) elif i == 'runner': self.assertIsInstance(tool_spec[i], str) self.assertEqual(tool_spec[i], result[i], msg="for {}, actual {} value ({}) is different form expected " "result ({}) ".format(tool_id, i, tool_spec[i], result[i]))
def test_pass_docker_params(self): """ Test if the tool's docker requirements are handled correctly """ _tool_label = '_unittest_tool' for dest in SPECIFICATIONS: with self.subTest(dest=dest): _tool_spec = { _tool_label: { 'runner': dest, 'docker_set_user': 1000, 'docker_run_extra_arguments': 'extra argument', } } self.td[_tool_label] = _tool_spec[_tool_label] result = { 'params': { 'docker_set_user': '******', 'docker_run_extra_arguments': 'extra argument', }, } tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, self.td, []) _, params, _, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS) if 'docker_enabled' in params and params['docker_enabled'] in ('true', 'True'): for i in ['docker_set_user', 'docker_run_extra_arguments']: self.assertEqual(params[i], result['params'][i])
def test_force_destination_id_false_with_runner(self): """ Test that it use the id from destination_id """ _tool_label = '_unittest_tool' _dest_label = '_unittest_destination' _tool_spec = { _tool_label: { 'force_destination_id': False, 'runner': _dest_label } } TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] result = '1cores_4.0G' tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) name = name_it(tool_spec) self.assertEqual(name, result)
def test_pass_param_gpus_and_destination(self): """ Test tool with undefined gpu and destination against all destinations. If destination starts with remote_cluster_mq, 'submit_request_gpus' hasn't to be present in params. If destination starts with condor, 'request_gpus' hasn't to be present in params. """ _tool_label = '_unittest_tool' for dest in SPECIFICATIONS: _tool_spec = { _tool_label: { 'runner': dest, } } self.td[_tool_label] = _tool_spec[_tool_label] tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, self.td, []) _, params, _, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS) if dest.startswith('remote_cluster_mq'): self.assertFalse('submit_request_gpus' in params) if dest.startswith('condor'): self.assertFalse('request_gpus' in params)
def test_pass_param_cores(self): """ Test cores request for each tools. '*request_cpus' has to be in params with the same value provided by the tool. """ for tool_label in TOOL_DESTINATIONS: if 'cores' in TOOL_DESTINATIONS[tool_label]: cores = str(TOOL_DESTINATIONS[tool_label]['cores']) else: cores = '1' result = {'params': {'request_cpus': cores}} tool_id = tool_label tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) _, params, dest, _, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS) if dest.startswith('remote_cluster_mq'): self.assertEqual(params['submit_request_cpus'], result['params']['submit_request_cpus']) if dest.startswith('condor'): self.assertEqual(params['request_cpus'], result['params']['request_cpus'])
def test_default_keys(self): """ Test default keys from TOOL_DESTINATIONS """ for tool_id in TOOL_DESTINATIONS: result = TOOL_DESTINATIONS[tool_id] tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) for i in result: if i in ['cores', 'gpus']: self.assertIsInstance(tool_spec[i], int) elif i == 'mem': self.assertIsInstance(tool_spec[i], float) elif i == 'force_destination_id': self.assertIsInstance(tool_spec[i], bool) print(tool_spec[i]) elif i == 'runner': self.assertIsInstance(tool_spec[i], str) self.assertEqual( tool_spec[i], result[i], msg= "for {}, actual {} value ({}) is different form expected " "result ({}) ".format(tool_id, i, tool_spec[i], result[i]))
def test_runner_pulsar(self): """ Test Pulsar destination """ _tool_label = '_unittest_tool' _dest_label = 'remote_cluster_mq_unittest_destination' _tool_spec = {_tool_label: {'runner': _dest_label}} _dest_spec = { _dest_label: { 'env': {}, 'params': { 'submit_request_cpus': '{PARALLELISATION}', 'request_memory': '{MEMORY}', 'request_gpus': '{GPUS}' } } } TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label] result = 'pulsar_eu_unittest_destination' tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) env, params, runner, _, tags = build_spec(tool_spec, dest_spec=SPECIFICATIONS) self.assertEqual(runner, result)
def test_runner_undefined(self): """ Test no runner specification, should reply with the DEFAULT_DESTINATION """ _tool_label = '_unittest_tool' _dest_label = 'condor_unittest_destination' _tool_spec = {_tool_label: {}} _dest_spec = { _dest_label: { 'env': {}, 'params': { 'request_cpus': '{PARALLELISATION}', 'request_memory': '{MEMORY}' } } } TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label] result = DEFAULT_DESTINATION tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) env, params, runner, _, tags = build_spec(tool_spec, dest_spec=SPECIFICATIONS) self.assertEqual(runner, result)
def test_runner_from_joint_destination(self): """ Test runner retrieved from joint destinations """ _tool_label = '_unittest_tool' _dest_label = 'remote_condor_cluster_gpu_docker' _tool_spec = {_tool_label: {'runner': _dest_label}} _dest_spec = { _dest_label: { 'env': {}, 'params': { 'submit_request_cpus': '{PARALLELISATION}', 'request_memory': '{MEMORY}', 'request_gpus': '{GPUS}' } } } TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] result = ['pulsar_eu_de03', 'pulsar_eu_uk01'] tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) _, _, runner, _, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS) self.assertIn(runner, result)
def test_runner_pulsar(self): """ Test Pulsar destination """ _tool_label = '_unittest_tool' _dest_label = 'remote_cluster_mq_unittest_destination' _tool_spec = { _tool_label: { 'runner': _dest_label } } _dest_spec = { _dest_label: { 'env': {}, 'params': { 'submit_request_cpus': '{PARALLELISATION}', 'request_memory': '{MEMORY}', 'request_gpus': '{GPUS}' } } } self.td[_tool_label] = _tool_spec[_tool_label] self.sp[_dest_label] = _dest_spec[_dest_label] result = 'pulsar_eu_destination' tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, self.td, []) _, _, runner, _ = build_spec(tool_spec, dest_spec=self.sp) self.assertEqual(runner, result)
def test_pass_three_env(self): """ Test that it pass three env variables correctly. Two from the destination and one from the runner, all different. """ _tool_label = '_unittest_tool' _dest_label = '_unittest_destination' _tool_spec = { _tool_label: { 'runner': _dest_label, 'env': { 'name1': 'value1', 'name2': "value2" } } } _dest_spec = { _dest_label: { 'env': { 'name3': "value3" }, 'params': { 'request_cpus': '{PARALLELISATION}', 'request_memory': '{MEMORY}' } } } TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label] result = [{ 'name': 'name3', 'value': 'value3' }, { 'name': 'name1', 'value': 'value1' }, { 'name': 'name2', 'value': 'value2' }] tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) env, params, runner, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS) self.assertEqual(len(env), len(result)) d1 = {n['name']: n['value'] for n in env} d2 = {n['name']: n['value'] for n in result} for k, v in d1.items(): self.assertEqual(d1[k], d2[k])
def test_ft_disabled_requirements(self): """ Check if disabling FastTurnaround, requirements are not updated """ self.ft['enabled'] = False self.ft['mode'] = 'all_jobs' tool_spec = _finalize_tool_spec(self.tool_id, self.td, []) _, params_b, _, _ = build_spec(tool_spec, dest_spec=self.sp) _, params_g, _, _, _ = _gateway(self.tool_id, '', '', '', '', ft=self.ft, dest_spec=self.sp, tools_spec=self.td) self.assertEqual(params_b['requirements'], params_g['requirements'])
def test_boundaries2(self): """ Test that it pass boundary values if ones from the destination are too high Add gpus """ _tool_label = '_unittest_tool' _dest_label = '_unittest_destination' _tool_spec = { _tool_label: { 'runner': _dest_label, 'cores': 100, 'mem': 2000, 'gpus': 100 } } _dest_label = '_unittest_destination' _dest_spec = { _dest_label: { 'limits': { 'cores': 10, 'mem': 10, 'gpus': 10 }, 'env': {}, 'params': { 'request_cpus': '{PARALLELISATION}', 'request_memory': '{MEMORY}', 'request_gpus': '{GPUS}' } } } TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label] result = { 'request_cpus': '10', 'request_memory': '10G', 'request_gpus': '10' } tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) env, params, runner, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS) for k, v in result.items(): self.assertIn(k, params) self.assertEqual(params[k], result[k])
def test_pass_two_env_plus_one_identical_from_runner(self): """ Runner has defined the same env variable as tool, but the tool's one has a higher priority """ _tool_label = '_unittest_tool' _dest_label = '_unittest_destination' _tool_spec = { _tool_label: { 'runner': _dest_label, 'env': { 'name1': 'value1', 'name2': "value2" } } } _dest_spec = { _dest_label: { 'env': { 'name2': "value3" }, 'params': { 'request_cpus': '{PARALLELISATION}', 'request_memory': '{MEMORY}' } } } TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label] result = [{ 'name': 'name2', 'value': 'value2' }, { 'name': 'name1', 'value': 'value1' }] tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) env, params, runner, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS) d1 = {n['name']: n['value'] for n in env} d2 = {n['name']: n['value'] for n in result} for k, v in d1.items(): self.assertEqual(d1[k], d2[k])
def test_pass_two_env_plus_runner(self): """ Test that it pass two env variables correctly with defined runner """ _tool_label = '_unittest_tool' _dest_label = '_unittest_destination' _tool_spec = { _tool_label: { 'runner': _dest_label, 'env': { 'name1': 'value1', 'name2': "value2" } } } _dest_spec = { _dest_label: { 'env': {}, 'params': { 'request_cpus': '{PARALLELISATION}', 'request_memory': '{MEMORY}' } } } TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label] result = [{ 'name': 'name1', 'value': 'value1' }, { 'name': 'name2', 'value': 'value2' }] tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) env, params, runner, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS) d1 = {n['name']: n['value'] for n in env} d2 = {n['name']: n['value'] for n in result} for k, v in d1.items(): self.assertEqual(d1[k], d2[k])
def test_pass_subparam_params(self): """ Test if build_spec handles properly a destination with the params array containing a list of array. 'subparam' has to be in params with the same value provided by the destination """ _tool_label = '_unittest_tool' _dest_label = '_unittest_destination' _tool_spec = { _tool_label: { 'runner': _dest_label } } _dest_spec = { _dest_label: { 'env': {}, 'params': { 'subparam': [ { 'name1': 'value1', 'name2': "value2" } ] } } } self.td[_tool_label] = _tool_spec[_tool_label] self.sp[_dest_label] = _dest_spec[_dest_label] result = { 'params': { 'subparam': [ { 'name1': 'value1', 'name2': "value2" } ] } } tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, self.td, []) _, params, _, _ = build_spec(tool_spec, dest_spec=self.sp) self.assertIn('subparam', params) self.assertIsInstance(params['subparam'], list) self.assertEqual(result['params'], params)
def test_force_destination_id_true(self): """ Test that it use the id from destination_id """ _tool_label = '_unittest_tool' _tool_spec = {_tool_label: {'force_destination_id': True}} self.td[_tool_label] = _tool_spec[_tool_label] result = FDID_PREFIX + DEFAULT_DESTINATION tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, self.td, []) name = name_it(tool_spec) self.assertEqual(name, result)
def test_force_destination_id_false(self): """ Test """ _tool_label = '_unittest_tool' _tool_spec = {_tool_label: {'force_destination_id': False}} self.td[_tool_label] = _tool_spec[_tool_label] result = '1cores_4.0G' tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, self.td, []) name = name_it(tool_spec) self.assertEqual(name, result)
def test_force_destination_id_default(self): """ Test """ _tool_label = '_unittest_tool' _tool_spec = {_tool_label: {}} TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] result = '1cores_4.0G' tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, self.td, []) name = name_it(tool_spec) self.assertEqual(name, result)
def test_runner_unknown(self): """ Test unknown runner value, should reply with the DEFAULT_DESTINATION """ _tool_label = '_unittest_tool' _tool_spec = {_tool_label: {'runner': 'unknown'}} TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] result = DEFAULT_DESTINATION tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) _, _, runner, _, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS) self.assertEqual(runner, result)
def test_force_destination_id_false(self): """ Test """ _tool_label = '_unittest_tool' _tool_spec = {_tool_label: {'force_destination_id': False}} TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] result = '1cores_4.0G' tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) name = name_it(tool_spec) self.assertEqual(name, result)
def test_boundaries(self): """ Test that it pass boundary values if ones from the destination are too high """ _tool_label = '_unittest_tool' _dest_label = '_unittest_destination' _tool_spec = { _tool_label: { 'runner': _dest_label, 'cores': 100, 'mem': 2000, 'gpus': 100 } } _dest_spec = { _dest_label: { 'limits': { 'cores': 10, 'mem': 10 }, 'env': {}, 'params': { 'request_cpus': '{PARALLELISATION}', 'request_memory': '{MEMORY}' } } } self.td[_tool_label] = _tool_spec[_tool_label] self.sp[_dest_label] = _dest_spec[_dest_label] result = {'request_cpus': '10', 'request_memory': '10G'} tool_id = '_unittest_tool' tool_spec = _finalize_tool_spec(tool_id, self.td, []) env, params, runner, tags = build_spec(tool_spec, dest_spec=self.sp) for k, v in result.items(): self.assertIn(k, params) self.assertEqual(params[k], result[k])
def test_pass_param_mem(self): """ Test tool with defined mem """ _tool_label = '_unittest_tool' _tool_spec = {_tool_label: {'mem': 32}} TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] result = {'params': {'request_memory': '32.0G'}} tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS) _, params, _, _, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS) self.assertEqual(params['request_memory'], result['params']['request_memory'])
def test_runner_from_joint_destination(self): """ Test runner retrieved from joint destinations """ _tool_label = '_unittest_tool' _dest_label = 'remote_condor_cluster_gpu_docker' _tool_spec = { _tool_label: { 'runner': _dest_label } } self.td[_tool_label] = _tool_spec[_tool_label] result = ['pulsar_eu_de03', 'pulsar_eu_uk01'] tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, self.td, []) _, _, runner, _ = build_spec(tool_spec, dest_spec=self.sp) self.assertIn(runner, result)
def test_force_destination_id_true_with_runner(self): """ Test that it use the id from destination_id """ _tool_label = '_unittest_tool' _dest_label = '_unittest_destination' _tool_spec = { _tool_label: { 'force_destination_id': True, 'runner': _dest_label } } self.td[_tool_label] = _tool_spec[_tool_label] result = FDID_PREFIX + _dest_label tool_id = _tool_label tool_spec = _finalize_tool_spec(tool_id, self.td, []) name = name_it(tool_spec) self.assertEqual(name, result)