def test_get_tags_from_tool_and_destination(self):
"""
Test if tags string is collected from tool and destination specifications
"""
_tool_label = '_unittest_tool'
_tool_tags = 'unittest_tool_tag'
_dest_label = '_unittest_destination'
_dest_tags = 'unittest_destination_tag'
_tool_spec = {_tool_label: {'runner': _dest_label, 'tags': _tool_tags}}
_dest_spec = {
_dest_label: {
'env': {},
'params': {},
'tags': _dest_tags
}
}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label]
tool_id = _tool_label
env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '')
self.assertIn(_tool_tags, tags)
self.assertIn(_dest_tags, tags)
def test_get_tags_from_destination(self):
"""
Test if tags string is collected from destination's specification
"""
_tool_label = '_unittest_tool'
_dest_label = '_unittest_destination'
_dest_tags = 'unittest_destination_tag'
_tool_spec = {_tool_label: {'runner': _dest_label}}
_dest_spec = {
_dest_label: {
'env': {},
'params': {},
'tags': _dest_tags
}
}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label]
result = _dest_tags
tool_id = _tool_label
env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '')
self.assertEqual(tags, result)
def test_reroute_aliasing(self):
"""
Some particular events can have contemporary trainings (like GCC) and we want to collect
them under the same label (e.g. to assign the the same computing cluster)
Given a user's roles with several training like "training-gcc-*" label,
the gateway function has to append the "training-gcc" label to the others
"""
_user_roles = [
'training-gcc-unittest_training1',
'training-gcc-unittest_training2'
]
result = {
'params': {
'requirements':
'(GalaxyGroup == "compute") || ((GalaxyGroup == "{}") || (GalaxyGroup == "{}") || (GalaxyGroup == "{}"))'
.format(_user_roles[0], _user_roles[1], "training-gcc"),
'+Group':
'"{}, {}, {}"'.format(_user_roles[0], _user_roles[1],
"training-gcc"),
},
}
_, params, _, _, _ = _gateway(self.tool_id,
'',
_user_roles,
'',
'',
tools_spec=self.td)
self.assertEqual(params['requirements'],
result['params']['requirements'])
self.assertEqual(params['+Group'], result['params']['+Group'])
def test_tool_set_metadata_(self):
"""
Test that it pass default values if no specifications
"""
result = {
'env': [],
'params': {
'priority': '-128',
'request_memory': '0.3G',
'tmp_dir': 'True',
'requirements': 'GalaxyTraining == false',
'rank': 'GalaxyGroup == "metadata"',
'accounting_group_user': '',
'description': '__SET_METADATA__'
},
'runner': DEFAULT_DESTINATION,
'tool_spec': {
'mem': 0.3,
'runner': 'condor',
'rank': 'GalaxyGroup == "metadata"',
'requirements': 'GalaxyTraining == false'
}
}
tool_id = '__SET_METADATA__'
env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '')
self.assertEqual(env, result['env'])
self.assertEqual(params, result['params'])
self.assertEqual(runner, result['runner'])
self.assertEqual(tool_spec, result['tool_spec'])
def test_reroute(self):
"""
Given a user's role starting with "training-" label, the function has to return two
parameters, requirements and +Group, updated with the training label
"""
_user_roles = [
'training-unittest_training1', 'training-unittest_training2'
]
result = {
'params': {
'requirements':
'(GalaxyGroup == "compute") || ((GalaxyGroup == "{}") || (GalaxyGroup == "{}"))'
.format(_user_roles[0], _user_roles[1]),
'+Group':
'"{}, {}"'.format(_user_roles[0], _user_roles[1]),
},
}
_, params, _, _, _ = _gateway(self.tool_id,
'',
_user_roles,
'',
'',
tools_spec=self.td)
self.assertEqual(params['requirements'],
result['params']['requirements'])
self.assertEqual(params['+Group'], result['params']['+Group'])
def test_ft_disabled_requirements(self):
"""
Check if disabling FastTurnaround, requirements are not updated
"""
self.ft['enabled'] = False
self.ft['mode'] = 'all_jobs'
tool_spec = _finalize_tool_spec(self.tool_id, self.td, [])
_, params_b, _, _ = build_spec(tool_spec, dest_spec=self.sp)
_, params_g, _, _, _ = _gateway(self.tool_id, '', '', '', '', ft=self.ft, dest_spec=self.sp, tools_spec=self.td)
self.assertEqual(params_b['requirements'], params_g['requirements'])
def test_ft_enabled_requirements(self):
"""
Check if enabling FastTurnaround, requirements are properly updated
"""
self.ft['enabled'] = True
self.ft['mode'] = 'all_jobs'
result = {'requirements': self.ft.get('requirements')}
_, params, _, _, _ = _gateway(self.tool_id, '', '', '', '', ft=self.ft)
self.assertEqual(params['requirements'], result['requirements'])
def test_tool_data_fetch_(self):
"""
Test that it pass default values if no specifications
"""
result = {
'env': [{
'name': 'NUMBA_CACHE_DIR',
'value': '/data/2/galaxy_db/tmp'
}, {
'name': 'GALAXY_MEMORY_MB',
'value': '307'
}, {
'name': 'GALAXY_SLOTS',
'value': '1'
}, {
'name': 'TEMP',
'value': '/data/1/galaxy_db/tmp'
}],
'params': {
'priority': '-128',
'request_memory': '0.3G',
'request_cpus': '1',
'requirements': 'GalaxyTraining == false',
'rank': 'GalaxyGroup == "upload"',
'accounting_group_user': '',
'tmp_dir': 'True',
'description': '__DATA_FETCH__'
},
'runner':
DEFAULT_DESTINATION,
'tool_spec': {
'cores': 1,
'mem': 0.3,
'gpus': 0,
'runner': 'condor',
'env': {
'TEMP': '/data/1/galaxy_db/tmp'
},
'params': {
'rank': 'GalaxyGroup == "upload"',
'requirements': 'GalaxyTraining == false'
},
}
}
tool_id = '__DATA_FETCH__'
env, params, runner, tool_spec, tags = _gateway(
tool_id, '', '', '', '')
self.assertEqual(env, result['env'])
self.assertEqual(params, result['params'])
self.assertEqual(runner, result['runner'])
self.assertEqual(tool_spec, result['tool_spec'])
def test_tool_interactive_tool_(self):
"""
Test that it pass default values if called without specifications
"""
result = {
'params': {
'request_cpus': '1',
'request_memory': '4.0G',
'docker_enabled': 'True',
'requirements':
'GalaxyDockerHack == True && GalaxyGroup == "compute"',
'accounting_group_user': '',
'description': 'interactive_tool_unittest_tool'
}
}
_tool_label = 'interactive_tool_unittest_tool'
_dest_label = 'docker_unittest_destination'
_tool_spec = {_tool_label: {'runner': _dest_label}}
_dest_spec = {
_dest_label: {
'env': {},
'params': {
'request_cpus':
'{PARALLELISATION}',
'request_memory':
'{MEMORY}',
'docker_enabled':
True,
'requirements':
'GalaxyDockerHack == True && GalaxyGroup == "compute"'
}
}
}
self.td[_tool_label] = _tool_spec[_tool_label]
self.sp[_dest_label] = _dest_spec[_dest_label]
tool_id = _tool_label
_, params, _, _, _ = _gateway(tool_id,
'',
'',
'',
'',
tools_spec=self.td,
dest_spec=self.sp)
self.assertEqual(params, result['params'])
def test_get_tags_from_tool(self):
"""
Test if tags string is collected from tool's specification
"""
_tool_label = '_unittest_tool'
_tool_tags = 'unittest_tool_tag'
_tool_spec = {_tool_label: {'tags': _tool_tags}}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
result = _tool_tags
tool_id = _tool_label
env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '')
self.assertEqual(tags, result)
def test_skip_user_preferences(self):
"""
Check if tools associated with the "skip_user_preferences" label are really skipped
DEFAULT_DESTINATION is assumed to be the proper runner for these tools
"""
up_labels = ['remote_cluster_mq_ut01', 'condor_unittest_destination', 'None']
tools_to_skip = [k for k, v in SPECIAL_TOOLS.items() if 'skip_user_preferences' in v]
for t in tools_to_skip:
for u in up_labels:
_user_preferences = {
'distributed_compute|remote_resources|unittest_remote_resource': u
}
tool_id = t
_, _, runner, _, _ = _gateway(tool_id, _user_preferences, '', '', '', dest_spec=self.sp)
self.assertEqual(runner, DEFAULT_DESTINATION)
def test_destination_in_user_role(self):
"""
Test if the runner hint works without runner in tool_spec
"""
_tool_label = '_unittest_tool'
_user_roles = ['destination-pulsar-de01']
_tool_spec = {_tool_label: {}}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
result = _user_roles[0].replace('destination-pulsar-', 'pulsar_eu_')
tool_id = _tool_label
env, params, runner, tool_spec, tags = _gateway(
tool_id, _user_roles, '', '')
self.assertEqual(runner, result)
def test_tool_pulsar_incompatible(self):
"""
For a pulsar incompatible tool, _gateway function should return the default runner
even if the tool is requesting a different destination
"""
result = {
'runner': DEFAULT_DESTINATION,
}
up_labels = {
'remote_unittest_destination': 'True',
'local_unittest_destination': 'False',
'None': 'False'
}
pulsar_incompatible_tools = [k for k, v in SPECIAL_TOOLS.items() if 'pulsar_incompatible' in v]
for t in pulsar_incompatible_tools:
for d, v in up_labels.items():
_tool_label = t
_dest_label = d
_tool_spec = {
_tool_label: {
'runner': _dest_label
}
}
_dest_spec = {
_dest_label:
{
'info': {
'remote': v
},
'env': {},
'params': {
'request_cpus': '{PARALLELISATION}',
'request_memory': '{MEMORY}'
}
}
}
self.td[_tool_label] = _tool_spec[_tool_label]
self.sp[_dest_label] = _dest_spec[_dest_label]
tool_id = _tool_label
_, params, runner, _, _ = _gateway(tool_id, '', '', '', '', tools_spec=self.td, dest_spec=self.sp)
with self.subTest(t=t, d=d):
self.assertEqual(runner, result['runner'])
def test_no_tags(self):
"""
Test tags is None if not present in tool or destination
see: https://docs.galaxyproject.org/en/master/_modules/galaxy/jobs.html#JobDestination
"""
_tool_label = '_unittest_tool'
_dest_label = '_unittest_destination'
_tool_spec = {_tool_label: {'runner': _dest_label}}
_dest_spec = {_dest_label: {'env': {}, 'params': {}}}
TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label]
SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label]
result = None
tool_id = _tool_label
env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '')
self.assertEqual(tags, result)
def test_tool_upload1(self):
"""
Test that it pass default values if no specifications
"""
result = {
'env': [{'name': 'TEMP', 'value': '/data/1/galaxy_db/tmp/'}],
'params': {'priority': '-128', 'request_memory': '0.3G', 'tmp_dir': 'True', 'requirements':
'GalaxyTraining == false', 'rank': 'GalaxyGroup == "upload"',
'accounting_group_user': '', 'description': 'upload1'},
'runner': DEFAULT_DESTINATION,
'tool_spec': {'mem': 0.3, 'runner': 'condor', 'rank': 'GalaxyGroup == "upload"',
'requirements': 'GalaxyTraining == false', 'env': {'TEMP': '/data/1/galaxy_db/tmp/'}
}
}
tool_id = 'upload1'
env, params, runner, tool_spec = _gateway(tool_id, '', '', '')
self.assertEqual(env, result['env'])
self.assertEqual(params, result['params'])
self.assertEqual(runner, result['runner'])
self.assertEqual(tool_spec, result['tool_spec'])
def test_docker_tools(self):
"""
Test tools using local docker destination
These tools need to have requirements allowing them to run on docker nodes only
"""
result = {
'params': {'requirements': 'GalaxyDockerHack == True && GalaxyGroup == "compute"'}
}
for tool_id, spec in TOOL_DESTINATIONS.items():
with self.subTest(tool_id=tool_id):
# remove permission to test admin tools also
if 'permissions' in spec:
del spec['permissions']
ts = {tool_id: spec}
destination = spec['runner'] if 'runner' in spec else DEFAULT_DESTINATION
env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '', '', tools_spec=ts)
if 'condor' in destination:
if 'docker_enabled' in params and params['docker_enabled'] in ('true', 'True'):
self.assertEqual(params['requirements'], result['params']['requirements'])
def test_tool_scanpy_suite(self):
"""
Test that it pass default values if no specifications
"""
tools_id = ['scanpy_inspect', 'scanpy_filter', 'scanpy_cluster_reduce_dimension',
'scanpy_normalize', 'scanpy_remove_confounders', 'scanpy_plot']
for tool_id in tools_id:
result = {
'env': [{'name': 'NUMBA_CACHE_DIR', 'value': '/data/2/galaxy_db/tmp'},
{'name': 'OMP_NUM_THREADS', 'value': '4'},
{'name': 'OPENBLAS_NUM_THREADS', 'value': '4'},
{'name': 'MKL_NUM_THREADS', 'value': '4'},
{'name': 'VECLIB_MAXIMUM_THREADS', 'value': '4'},
{'name': 'NUMEXPR_NUM_THREADS', 'value': '4'},
{'name': 'NUMBA_NUM_THREADS', 'value': '4'}],
'params': {'priority': '-128', 'request_cpus': '4', 'request_memory': '4.0G',
'tmp_dir': 'True', 'requirements': 'GalaxyGroup == "compute"',
'accounting_group_user': '', 'description': tool_id},
'runner': DEFAULT_DESTINATION,
'tool_spec': {'cores': 4,
'env': {'NUMBA_CACHE_DIR': '/data/2/galaxy_db/tmp', 'OMP_NUM_THREADS': 4,
'OPENBLAS_NUM_THREADS': 4, 'MKL_NUM_THREADS': 4,
'VECLIB_MAXIMUM_THREADS': 4, 'NUMEXPR_NUM_THREADS': 4,
'NUMBA_NUM_THREADS': 4},
'requirements': 'GalaxyGroup == "compute"',
'mem': 4.0,
'force_destination_id': False,
'gpus': 0,
'runner': DEFAULT_DESTINATION}
}
env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '')
d1 = {n['name']: n['value'] for n in env}
d2 = {n['name']: n['value'] for n in result['env']}
for k, v in d1.items():
self.assertEqual(d1[k], d2[k])
self.assertEqual(params, result['params'])
self.assertEqual(runner, result['runner'])
self.assertEqual(tool_spec, result['tool_spec'])
def test_runner_hint(self):
"""
A destination can be forced through the the user's preferences.
Check if the _gateway function, fed with a proper user_roles dict, return the runner associated
remote_cluster_mq_* > pulsar_eu_*
condor_* > condor
If None, return the expected runner from the tool's specification
"""
up_labels = ['remote_cluster_mq_ut01', 'condor_unittest_destination', 'None']
results = {
up_labels[0]: 'pulsar_eu_ut01',
up_labels[1]: 'condor',
'condor': 'condor'
}
t_runner_labels = [DEFAULT_DESTINATION, 'condor_unittest_destination', 'remote_cluster_mq_ut01']
for u in up_labels:
for r in t_runner_labels:
_tool_label = '_unittest_tool'
_tool_spec = {
_tool_label: {
'runner': r
}
}
self.td[_tool_label] = _tool_spec[_tool_label]
tool_id = _tool_label
_user_preferences = {
'distributed_compute|remote_resources|unittest_remote_resource': u
}
_, _, runner, _, _ = _gateway(tool_id, _user_preferences, '', '', '', tools_spec=self.td, dest_spec=self.sp)
if u == 'None':
self.assertEqual(runner, results[r])
else:
self.assertEqual(runner, results[u])
