def test_pass_two_env(self):
"""
Test that it pass two env variables correctly with undefined runner
"""
_tool_label = '_unittest_tool'
_tool_spec = {
_tool_label: {
'env': {
'name1': 'value1',
'name2': "value2"
}
}
}
self.td[_tool_label] = _tool_spec[_tool_label]
result = [{'name': 'name1', 'value': 'value1'}, {'name': 'name2', 'value': 'value2'}]
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id, self.td, [])
env, params, runner, tags = build_spec(tool_spec, dest_spec=SPECIFICATIONS)
d1 = {n['name']: n['value'] for n in env if n in ['name1', 'name2']}
d2 = {n['name']: n['value'] for n in result}
for k, v in d1.items():
self.assertEqual(d1[k], d2[k])
def test_runner_undefined(self):
"""
Test no runner specification, should reply with the DEFAULT_DESTINATION
"""
_tool_label = '_unittest_tool'
_dest_label = 'condor_unittest_destination'
_tool_spec = {
_tool_label: {}
}
_dest_spec = {
_dest_label: {
'env': {},
'params': {
'request_cpus': '{PARALLELISATION}',
'request_memory': '{MEMORY}'
}
}
}
self.td[_tool_label] = _tool_spec[_tool_label]
self.sp[_dest_label] = _dest_spec[_dest_label]
result = DEFAULT_DESTINATION
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id, self.td, [])
_, _, runner, _ = build_spec(tool_spec, dest_spec=self.sp)
self.assertEqual(runner, result)
def test_pass_param_cores_with_default_destination(self):
"""
Test tool with defined cores and destination by default
'request_cpus' has to be in params with the same value provided by the tool
"""
_tool_label = '_unittest_tool'
_tool_spec = {
_tool_label: {
'cores': 3
}
}
self.td[_tool_label] = _tool_spec[_tool_label]
result = {
'params': {'request_cpus': '3'}
}
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id, self.td, [])
_, params, _, _ = build_spec(tool_spec, dest_spec=self.sp)
self.assertEqual(params['request_cpus'], result['params']['request_cpus'])
def test_default_values_setup(self):
"""
Test if default values will be assigned in case of a tool without specifications
:return:
"""
_tool_label = '_unittest_tool'
_tool_spec = {_tool_label: {}}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
result = {
'cores': DEFAULT_TOOL_SPEC['cores'],
'mem': DEFAULT_TOOL_SPEC['mem'],
'gpus': DEFAULT_TOOL_SPEC['gpus'],
'force_destination_id': DEFAULT_TOOL_SPEC['force_destination_id'],
'runner': DEFAULT_TOOL_SPEC['runner']
}
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id, '', tools_spec=TOOL_DESTINATIONS)
for i in result:
if i in ['cores', 'gpus']:
self.assertIsInstance(tool_spec[i], int)
elif i == 'mem':
self.assertIsInstance(tool_spec[i], float)
elif i == 'force_destination_id':
self.assertIsInstance(tool_spec[i], bool)
elif i == 'runner':
self.assertIsInstance(tool_spec[i], str)
self.assertEqual(tool_spec[i], result[i],
msg="for {}, actual {} value ({}) is different form expected "
"result ({}) ".format(tool_id, i, tool_spec[i], result[i]))
def test_pass_docker_params(self):
"""
Test if the tool's docker requirements are handled correctly
"""
_tool_label = '_unittest_tool'
for dest in SPECIFICATIONS:
with self.subTest(dest=dest):
_tool_spec = {
_tool_label: {
'runner': dest,
'docker_set_user': 1000,
'docker_run_extra_arguments': 'extra argument',
}
}
self.td[_tool_label] = _tool_spec[_tool_label]
result = {
'params': {
'docker_set_user': '******',
'docker_run_extra_arguments': 'extra argument',
},
}
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id, self.td, [])
_, params, _, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS)
if 'docker_enabled' in params and params['docker_enabled'] in ('true', 'True'):
for i in ['docker_set_user', 'docker_run_extra_arguments']:
self.assertEqual(params[i], result['params'][i])
def test_force_destination_id_false_with_runner(self):
"""
Test that it use the id from destination_id
"""
_tool_label = '_unittest_tool'
_dest_label = '_unittest_destination'
_tool_spec = {
_tool_label: {
'force_destination_id': False,
'runner': _dest_label
}
}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
result = '1cores_4.0G'
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id,
'',
tools_spec=TOOL_DESTINATIONS)
name = name_it(tool_spec)
self.assertEqual(name, result)
def test_pass_param_gpus_and_destination(self):
"""
Test tool with undefined gpu and destination against all destinations.
If destination starts with remote_cluster_mq, 'submit_request_gpus' hasn't
to be present in params.
If destination starts with condor, 'request_gpus' hasn't
to be present in params.
"""
_tool_label = '_unittest_tool'
for dest in SPECIFICATIONS:
_tool_spec = {
_tool_label: {
'runner': dest,
}
}
self.td[_tool_label] = _tool_spec[_tool_label]
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id, self.td, [])
_, params, _, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS)
if dest.startswith('remote_cluster_mq'):
self.assertFalse('submit_request_gpus' in params)
if dest.startswith('condor'):
self.assertFalse('request_gpus' in params)
def test_pass_param_cores(self):
"""
Test cores request for each tools.
'*request_cpus' has to be in params with the same value provided by the tool.
"""
for tool_label in TOOL_DESTINATIONS:
if 'cores' in TOOL_DESTINATIONS[tool_label]:
cores = str(TOOL_DESTINATIONS[tool_label]['cores'])
else:
cores = '1'
result = {'params': {'request_cpus': cores}}
tool_id = tool_label
tool_spec = _finalize_tool_spec(tool_id,
'',
tools_spec=TOOL_DESTINATIONS)
_, params, dest, _, _ = build_spec(tool_spec,
dest_spec=SPECIFICATIONS)
if dest.startswith('remote_cluster_mq'):
self.assertEqual(params['submit_request_cpus'],
result['params']['submit_request_cpus'])
if dest.startswith('condor'):
self.assertEqual(params['request_cpus'],
result['params']['request_cpus'])
def test_default_keys(self):
"""
Test default keys from TOOL_DESTINATIONS
"""
for tool_id in TOOL_DESTINATIONS:
result = TOOL_DESTINATIONS[tool_id]
tool_spec = _finalize_tool_spec(tool_id,
'',
tools_spec=TOOL_DESTINATIONS)
for i in result:
if i in ['cores', 'gpus']:
self.assertIsInstance(tool_spec[i], int)
elif i == 'mem':
self.assertIsInstance(tool_spec[i], float)
elif i == 'force_destination_id':
self.assertIsInstance(tool_spec[i], bool)
print(tool_spec[i])
elif i == 'runner':
self.assertIsInstance(tool_spec[i], str)
self.assertEqual(
tool_spec[i],
result[i],
msg=
"for {}, actual {} value ({}) is different form expected "
"result ({}) ".format(tool_id, i, tool_spec[i], result[i]))
def test_runner_pulsar(self):
"""
Test Pulsar destination
"""
_tool_label = '_unittest_tool'
_dest_label = 'remote_cluster_mq_unittest_destination'
_tool_spec = {_tool_label: {'runner': _dest_label}}
_dest_spec = {
_dest_label: {
'env': {},
'params': {
'submit_request_cpus': '{PARALLELISATION}',
'request_memory': '{MEMORY}',
'request_gpus': '{GPUS}'
}
}
}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label]
result = 'pulsar_eu_unittest_destination'
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id,
'',
tools_spec=TOOL_DESTINATIONS)
env, params, runner, _, tags = build_spec(tool_spec,
dest_spec=SPECIFICATIONS)
self.assertEqual(runner, result)
def test_runner_undefined(self):
"""
Test no runner specification, should reply with the DEFAULT_DESTINATION
"""
_tool_label = '_unittest_tool'
_dest_label = 'condor_unittest_destination'
_tool_spec = {_tool_label: {}}
_dest_spec = {
_dest_label: {
'env': {},
'params': {
'request_cpus': '{PARALLELISATION}',
'request_memory': '{MEMORY}'
}
}
}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label]
result = DEFAULT_DESTINATION
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id,
'',
tools_spec=TOOL_DESTINATIONS)
env, params, runner, _, tags = build_spec(tool_spec,
dest_spec=SPECIFICATIONS)
self.assertEqual(runner, result)
def test_runner_from_joint_destination(self):
"""
Test runner retrieved from joint destinations
"""
_tool_label = '_unittest_tool'
_dest_label = 'remote_condor_cluster_gpu_docker'
_tool_spec = {_tool_label: {'runner': _dest_label}}
_dest_spec = {
_dest_label: {
'env': {},
'params': {
'submit_request_cpus': '{PARALLELISATION}',
'request_memory': '{MEMORY}',
'request_gpus': '{GPUS}'
}
}
}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
result = ['pulsar_eu_de03', 'pulsar_eu_uk01']
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id,
'',
tools_spec=TOOL_DESTINATIONS)
_, _, runner, _, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS)
self.assertIn(runner, result)
def test_runner_pulsar(self):
"""
Test Pulsar destination
"""
_tool_label = '_unittest_tool'
_dest_label = 'remote_cluster_mq_unittest_destination'
_tool_spec = {
_tool_label: {
'runner': _dest_label
}
}
_dest_spec = {
_dest_label: {
'env': {},
'params': {
'submit_request_cpus': '{PARALLELISATION}',
'request_memory': '{MEMORY}',
'request_gpus': '{GPUS}'
}
}
}
self.td[_tool_label] = _tool_spec[_tool_label]
self.sp[_dest_label] = _dest_spec[_dest_label]
result = 'pulsar_eu_destination'
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id, self.td, [])
_, _, runner, _ = build_spec(tool_spec, dest_spec=self.sp)
self.assertEqual(runner, result)
def test_pass_three_env(self):
"""
Test that it pass three env variables correctly.
Two from the destination and one from the runner, all different.
"""
_tool_label = '_unittest_tool'
_dest_label = '_unittest_destination'
_tool_spec = {
_tool_label: {
'runner': _dest_label,
'env': {
'name1': 'value1',
'name2': "value2"
}
}
}
_dest_spec = {
_dest_label: {
'env': {
'name3': "value3"
},
'params': {
'request_cpus': '{PARALLELISATION}',
'request_memory': '{MEMORY}'
}
}
}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label]
result = [{
'name': 'name3',
'value': 'value3'
}, {
'name': 'name1',
'value': 'value1'
}, {
'name': 'name2',
'value': 'value2'
}]
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id,
'',
tools_spec=TOOL_DESTINATIONS)
env, params, runner, _ = build_spec(tool_spec,
dest_spec=SPECIFICATIONS)
self.assertEqual(len(env), len(result))
d1 = {n['name']: n['value'] for n in env}
d2 = {n['name']: n['value'] for n in result}
for k, v in d1.items():
self.assertEqual(d1[k], d2[k])
def test_ft_disabled_requirements(self):
"""
Check if disabling FastTurnaround, requirements are not updated
"""
self.ft['enabled'] = False
self.ft['mode'] = 'all_jobs'
tool_spec = _finalize_tool_spec(self.tool_id, self.td, [])
_, params_b, _, _ = build_spec(tool_spec, dest_spec=self.sp)
_, params_g, _, _, _ = _gateway(self.tool_id, '', '', '', '', ft=self.ft, dest_spec=self.sp, tools_spec=self.td)
self.assertEqual(params_b['requirements'], params_g['requirements'])
def test_boundaries2(self):
"""
Test that it pass boundary values if ones from the destination are too high
Add gpus
"""
_tool_label = '_unittest_tool'
_dest_label = '_unittest_destination'
_tool_spec = {
_tool_label: {
'runner': _dest_label,
'cores': 100,
'mem': 2000,
'gpus': 100
}
}
_dest_label = '_unittest_destination'
_dest_spec = {
_dest_label: {
'limits': {
'cores': 10,
'mem': 10,
'gpus': 10
},
'env': {},
'params': {
'request_cpus': '{PARALLELISATION}',
'request_memory': '{MEMORY}',
'request_gpus': '{GPUS}'
}
}
}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label]
result = {
'request_cpus': '10',
'request_memory': '10G',
'request_gpus': '10'
}
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id,
'',
tools_spec=TOOL_DESTINATIONS)
env, params, runner, _ = build_spec(tool_spec,
dest_spec=SPECIFICATIONS)
for k, v in result.items():
self.assertIn(k, params)
self.assertEqual(params[k], result[k])
def test_pass_two_env_plus_one_identical_from_runner(self):
"""
Runner has defined the same env variable as tool, but the tool's one has a higher priority
"""
_tool_label = '_unittest_tool'
_dest_label = '_unittest_destination'
_tool_spec = {
_tool_label: {
'runner': _dest_label,
'env': {
'name1': 'value1',
'name2': "value2"
}
}
}
_dest_spec = {
_dest_label: {
'env': {
'name2': "value3"
},
'params': {
'request_cpus': '{PARALLELISATION}',
'request_memory': '{MEMORY}'
}
}
}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label]
result = [{
'name': 'name2',
'value': 'value2'
}, {
'name': 'name1',
'value': 'value1'
}]
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id,
'',
tools_spec=TOOL_DESTINATIONS)
env, params, runner, _ = build_spec(tool_spec,
dest_spec=SPECIFICATIONS)
d1 = {n['name']: n['value'] for n in env}
d2 = {n['name']: n['value'] for n in result}
for k, v in d1.items():
self.assertEqual(d1[k], d2[k])
def test_pass_two_env_plus_runner(self):
"""
Test that it pass two env variables correctly with defined runner
"""
_tool_label = '_unittest_tool'
_dest_label = '_unittest_destination'
_tool_spec = {
_tool_label: {
'runner': _dest_label,
'env': {
'name1': 'value1',
'name2': "value2"
}
}
}
_dest_spec = {
_dest_label: {
'env': {},
'params': {
'request_cpus': '{PARALLELISATION}',
'request_memory': '{MEMORY}'
}
}
}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label]
result = [{
'name': 'name1',
'value': 'value1'
}, {
'name': 'name2',
'value': 'value2'
}]
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id,
'',
tools_spec=TOOL_DESTINATIONS)
env, params, runner, _ = build_spec(tool_spec,
dest_spec=SPECIFICATIONS)
d1 = {n['name']: n['value'] for n in env}
d2 = {n['name']: n['value'] for n in result}
for k, v in d1.items():
self.assertEqual(d1[k], d2[k])
def test_pass_subparam_params(self):
"""
Test if build_spec handles properly a destination with the params array containing
a list of array.
'subparam' has to be in params with the same value provided by the destination
"""
_tool_label = '_unittest_tool'
_dest_label = '_unittest_destination'
_tool_spec = {
_tool_label: {
'runner': _dest_label
}
}
_dest_spec = {
_dest_label: {
'env': {},
'params': {
'subparam': [
{
'name1': 'value1',
'name2': "value2"
}
]
}
}
}
self.td[_tool_label] = _tool_spec[_tool_label]
self.sp[_dest_label] = _dest_spec[_dest_label]
result = {
'params': {
'subparam': [
{
'name1': 'value1',
'name2': "value2"
}
]
}
}
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id, self.td, [])
_, params, _, _ = build_spec(tool_spec, dest_spec=self.sp)
self.assertIn('subparam', params)
self.assertIsInstance(params['subparam'], list)
self.assertEqual(result['params'], params)
def test_force_destination_id_true(self):
"""
Test that it use the id from destination_id
"""
_tool_label = '_unittest_tool'
_tool_spec = {_tool_label: {'force_destination_id': True}}
self.td[_tool_label] = _tool_spec[_tool_label]
result = FDID_PREFIX + DEFAULT_DESTINATION
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id, self.td, [])
name = name_it(tool_spec)
self.assertEqual(name, result)
def test_force_destination_id_false(self):
"""
Test
"""
_tool_label = '_unittest_tool'
_tool_spec = {_tool_label: {'force_destination_id': False}}
self.td[_tool_label] = _tool_spec[_tool_label]
result = '1cores_4.0G'
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id, self.td, [])
name = name_it(tool_spec)
self.assertEqual(name, result)
def test_force_destination_id_default(self):
"""
Test
"""
_tool_label = '_unittest_tool'
_tool_spec = {_tool_label: {}}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
result = '1cores_4.0G'
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id, self.td, [])
name = name_it(tool_spec)
self.assertEqual(name, result)
def test_runner_unknown(self):
"""
Test unknown runner value, should reply with the DEFAULT_DESTINATION
"""
_tool_label = '_unittest_tool'
_tool_spec = {_tool_label: {'runner': 'unknown'}}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
result = DEFAULT_DESTINATION
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id,
'',
tools_spec=TOOL_DESTINATIONS)
_, _, runner, _, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS)
self.assertEqual(runner, result)
def test_force_destination_id_false(self):
"""
Test
"""
_tool_label = '_unittest_tool'
_tool_spec = {_tool_label: {'force_destination_id': False}}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
result = '1cores_4.0G'
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id,
'',
tools_spec=TOOL_DESTINATIONS)
name = name_it(tool_spec)
self.assertEqual(name, result)
def test_boundaries(self):
"""
Test that it pass boundary values if ones from the destination are too high
"""
_tool_label = '_unittest_tool'
_dest_label = '_unittest_destination'
_tool_spec = {
_tool_label: {
'runner': _dest_label,
'cores': 100,
'mem': 2000,
'gpus': 100
}
}
_dest_spec = {
_dest_label: {
'limits': {
'cores': 10,
'mem': 10
},
'env': {},
'params': {
'request_cpus': '{PARALLELISATION}',
'request_memory': '{MEMORY}'
}
}
}
self.td[_tool_label] = _tool_spec[_tool_label]
self.sp[_dest_label] = _dest_spec[_dest_label]
result = {'request_cpus': '10', 'request_memory': '10G'}
tool_id = '_unittest_tool'
tool_spec = _finalize_tool_spec(tool_id, self.td, [])
env, params, runner, tags = build_spec(tool_spec, dest_spec=self.sp)
for k, v in result.items():
self.assertIn(k, params)
self.assertEqual(params[k], result[k])
def test_pass_param_mem(self):
"""
Test tool with defined mem
"""
_tool_label = '_unittest_tool'
_tool_spec = {_tool_label: {'mem': 32}}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
result = {'params': {'request_memory': '32.0G'}}
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id,
'',
tools_spec=TOOL_DESTINATIONS)
_, params, _, _, _ = build_spec(tool_spec, dest_spec=SPECIFICATIONS)
self.assertEqual(params['request_memory'],
result['params']['request_memory'])
def test_runner_from_joint_destination(self):
"""
Test runner retrieved from joint destinations
"""
_tool_label = '_unittest_tool'
_dest_label = 'remote_condor_cluster_gpu_docker'
_tool_spec = {
_tool_label: {
'runner': _dest_label
}
}
self.td[_tool_label] = _tool_spec[_tool_label]
result = ['pulsar_eu_de03', 'pulsar_eu_uk01']
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id, self.td, [])
_, _, runner, _ = build_spec(tool_spec, dest_spec=self.sp)
self.assertIn(runner, result)
def test_force_destination_id_true_with_runner(self):
"""
Test that it use the id from destination_id
"""
_tool_label = '_unittest_tool'
_dest_label = '_unittest_destination'
_tool_spec = {
_tool_label: {
'force_destination_id': True,
'runner': _dest_label
}
}
self.td[_tool_label] = _tool_spec[_tool_label]
result = FDID_PREFIX + _dest_label
tool_id = _tool_label
tool_spec = _finalize_tool_spec(tool_id, self.td, [])
name = name_it(tool_spec)
self.assertEqual(name, result)
