def test_get_tags_from_tool_and_destination(self): """ Test if tags string is collected from tool and destination specifications """ _tool_label = '_unittest_tool' _tool_tags = 'unittest_tool_tag' _dest_label = '_unittest_destination' _dest_tags = 'unittest_destination_tag' _tool_spec = {_tool_label: {'runner': _dest_label, 'tags': _tool_tags}} _dest_spec = { _dest_label: { 'env': {}, 'params': {}, 'tags': _dest_tags } } TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label] tool_id = _tool_label env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '') self.assertIn(_tool_tags, tags) self.assertIn(_dest_tags, tags)
def test_get_tags_from_destination(self): """ Test if tags string is collected from destination's specification """ _tool_label = '_unittest_tool' _dest_label = '_unittest_destination' _dest_tags = 'unittest_destination_tag' _tool_spec = {_tool_label: {'runner': _dest_label}} _dest_spec = { _dest_label: { 'env': {}, 'params': {}, 'tags': _dest_tags } } TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label] result = _dest_tags tool_id = _tool_label env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '') self.assertEqual(tags, result)
def test_reroute_aliasing(self): """ Some particular events can have contemporary trainings (like GCC) and we want to collect them under the same label (e.g. to assign the the same computing cluster) Given a user's roles with several training like "training-gcc-*" label, the gateway function has to append the "training-gcc" label to the others """ _user_roles = [ 'training-gcc-unittest_training1', 'training-gcc-unittest_training2' ] result = { 'params': { 'requirements': '(GalaxyGroup == "compute") || ((GalaxyGroup == "{}") || (GalaxyGroup == "{}") || (GalaxyGroup == "{}"))' .format(_user_roles[0], _user_roles[1], "training-gcc"), '+Group': '"{}, {}, {}"'.format(_user_roles[0], _user_roles[1], "training-gcc"), }, } _, params, _, _, _ = _gateway(self.tool_id, '', _user_roles, '', '', tools_spec=self.td) self.assertEqual(params['requirements'], result['params']['requirements']) self.assertEqual(params['+Group'], result['params']['+Group'])
def test_tool_set_metadata_(self): """ Test that it pass default values if no specifications """ result = { 'env': [], 'params': { 'priority': '-128', 'request_memory': '0.3G', 'tmp_dir': 'True', 'requirements': 'GalaxyTraining == false', 'rank': 'GalaxyGroup == "metadata"', 'accounting_group_user': '', 'description': '__SET_METADATA__' }, 'runner': DEFAULT_DESTINATION, 'tool_spec': { 'mem': 0.3, 'runner': 'condor', 'rank': 'GalaxyGroup == "metadata"', 'requirements': 'GalaxyTraining == false' } } tool_id = '__SET_METADATA__' env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '') self.assertEqual(env, result['env']) self.assertEqual(params, result['params']) self.assertEqual(runner, result['runner']) self.assertEqual(tool_spec, result['tool_spec'])
def test_reroute(self): """ Given a user's role starting with "training-" label, the function has to return two parameters, requirements and +Group, updated with the training label """ _user_roles = [ 'training-unittest_training1', 'training-unittest_training2' ] result = { 'params': { 'requirements': '(GalaxyGroup == "compute") || ((GalaxyGroup == "{}") || (GalaxyGroup == "{}"))' .format(_user_roles[0], _user_roles[1]), '+Group': '"{}, {}"'.format(_user_roles[0], _user_roles[1]), }, } _, params, _, _, _ = _gateway(self.tool_id, '', _user_roles, '', '', tools_spec=self.td) self.assertEqual(params['requirements'], result['params']['requirements']) self.assertEqual(params['+Group'], result['params']['+Group'])
def test_ft_disabled_requirements(self): """ Check if disabling FastTurnaround, requirements are not updated """ self.ft['enabled'] = False self.ft['mode'] = 'all_jobs' tool_spec = _finalize_tool_spec(self.tool_id, self.td, []) _, params_b, _, _ = build_spec(tool_spec, dest_spec=self.sp) _, params_g, _, _, _ = _gateway(self.tool_id, '', '', '', '', ft=self.ft, dest_spec=self.sp, tools_spec=self.td) self.assertEqual(params_b['requirements'], params_g['requirements'])
def test_ft_enabled_requirements(self): """ Check if enabling FastTurnaround, requirements are properly updated """ self.ft['enabled'] = True self.ft['mode'] = 'all_jobs' result = {'requirements': self.ft.get('requirements')} _, params, _, _, _ = _gateway(self.tool_id, '', '', '', '', ft=self.ft) self.assertEqual(params['requirements'], result['requirements'])
def test_tool_data_fetch_(self): """ Test that it pass default values if no specifications """ result = { 'env': [{ 'name': 'NUMBA_CACHE_DIR', 'value': '/data/2/galaxy_db/tmp' }, { 'name': 'GALAXY_MEMORY_MB', 'value': '307' }, { 'name': 'GALAXY_SLOTS', 'value': '1' }, { 'name': 'TEMP', 'value': '/data/1/galaxy_db/tmp' }], 'params': { 'priority': '-128', 'request_memory': '0.3G', 'request_cpus': '1', 'requirements': 'GalaxyTraining == false', 'rank': 'GalaxyGroup == "upload"', 'accounting_group_user': '', 'tmp_dir': 'True', 'description': '__DATA_FETCH__' }, 'runner': DEFAULT_DESTINATION, 'tool_spec': { 'cores': 1, 'mem': 0.3, 'gpus': 0, 'runner': 'condor', 'env': { 'TEMP': '/data/1/galaxy_db/tmp' }, 'params': { 'rank': 'GalaxyGroup == "upload"', 'requirements': 'GalaxyTraining == false' }, } } tool_id = '__DATA_FETCH__' env, params, runner, tool_spec, tags = _gateway( tool_id, '', '', '', '') self.assertEqual(env, result['env']) self.assertEqual(params, result['params']) self.assertEqual(runner, result['runner']) self.assertEqual(tool_spec, result['tool_spec'])
def test_tool_interactive_tool_(self): """ Test that it pass default values if called without specifications """ result = { 'params': { 'request_cpus': '1', 'request_memory': '4.0G', 'docker_enabled': 'True', 'requirements': 'GalaxyDockerHack == True && GalaxyGroup == "compute"', 'accounting_group_user': '', 'description': 'interactive_tool_unittest_tool' } } _tool_label = 'interactive_tool_unittest_tool' _dest_label = 'docker_unittest_destination' _tool_spec = {_tool_label: {'runner': _dest_label}} _dest_spec = { _dest_label: { 'env': {}, 'params': { 'request_cpus': '{PARALLELISATION}', 'request_memory': '{MEMORY}', 'docker_enabled': True, 'requirements': 'GalaxyDockerHack == True && GalaxyGroup == "compute"' } } } self.td[_tool_label] = _tool_spec[_tool_label] self.sp[_dest_label] = _dest_spec[_dest_label] tool_id = _tool_label _, params, _, _, _ = _gateway(tool_id, '', '', '', '', tools_spec=self.td, dest_spec=self.sp) self.assertEqual(params, result['params'])
def test_get_tags_from_tool(self): """ Test if tags string is collected from tool's specification """ _tool_label = '_unittest_tool' _tool_tags = 'unittest_tool_tag' _tool_spec = {_tool_label: {'tags': _tool_tags}} TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] result = _tool_tags tool_id = _tool_label env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '') self.assertEqual(tags, result)
def test_skip_user_preferences(self): """ Check if tools associated with the "skip_user_preferences" label are really skipped DEFAULT_DESTINATION is assumed to be the proper runner for these tools """ up_labels = ['remote_cluster_mq_ut01', 'condor_unittest_destination', 'None'] tools_to_skip = [k for k, v in SPECIAL_TOOLS.items() if 'skip_user_preferences' in v] for t in tools_to_skip: for u in up_labels: _user_preferences = { 'distributed_compute|remote_resources|unittest_remote_resource': u } tool_id = t _, _, runner, _, _ = _gateway(tool_id, _user_preferences, '', '', '', dest_spec=self.sp) self.assertEqual(runner, DEFAULT_DESTINATION)
def test_destination_in_user_role(self): """ Test if the runner hint works without runner in tool_spec """ _tool_label = '_unittest_tool' _user_roles = ['destination-pulsar-de01'] _tool_spec = {_tool_label: {}} TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] result = _user_roles[0].replace('destination-pulsar-', 'pulsar_eu_') tool_id = _tool_label env, params, runner, tool_spec, tags = _gateway( tool_id, _user_roles, '', '') self.assertEqual(runner, result)
def test_tool_pulsar_incompatible(self): """ For a pulsar incompatible tool, _gateway function should return the default runner even if the tool is requesting a different destination """ result = { 'runner': DEFAULT_DESTINATION, } up_labels = { 'remote_unittest_destination': 'True', 'local_unittest_destination': 'False', 'None': 'False' } pulsar_incompatible_tools = [k for k, v in SPECIAL_TOOLS.items() if 'pulsar_incompatible' in v] for t in pulsar_incompatible_tools: for d, v in up_labels.items(): _tool_label = t _dest_label = d _tool_spec = { _tool_label: { 'runner': _dest_label } } _dest_spec = { _dest_label: { 'info': { 'remote': v }, 'env': {}, 'params': { 'request_cpus': '{PARALLELISATION}', 'request_memory': '{MEMORY}' } } } self.td[_tool_label] = _tool_spec[_tool_label] self.sp[_dest_label] = _dest_spec[_dest_label] tool_id = _tool_label _, params, runner, _, _ = _gateway(tool_id, '', '', '', '', tools_spec=self.td, dest_spec=self.sp) with self.subTest(t=t, d=d): self.assertEqual(runner, result['runner'])
def test_no_tags(self): """ Test tags is None if not present in tool or destination see: https://docs.galaxyproject.org/en/master/_modules/galaxy/jobs.html#JobDestination """ _tool_label = '_unittest_tool' _dest_label = '_unittest_destination' _tool_spec = {_tool_label: {'runner': _dest_label}} _dest_spec = {_dest_label: {'env': {}, 'params': {}}} TOOL_DESTINATIONS[_tool_label] = _tool_spec[_tool_label] SPECIFICATIONS[_dest_label] = _dest_spec[_dest_label] result = None tool_id = _tool_label env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '') self.assertEqual(tags, result)
def test_tool_upload1(self): """ Test that it pass default values if no specifications """ result = { 'env': [{'name': 'TEMP', 'value': '/data/1/galaxy_db/tmp/'}], 'params': {'priority': '-128', 'request_memory': '0.3G', 'tmp_dir': 'True', 'requirements': 'GalaxyTraining == false', 'rank': 'GalaxyGroup == "upload"', 'accounting_group_user': '', 'description': 'upload1'}, 'runner': DEFAULT_DESTINATION, 'tool_spec': {'mem': 0.3, 'runner': 'condor', 'rank': 'GalaxyGroup == "upload"', 'requirements': 'GalaxyTraining == false', 'env': {'TEMP': '/data/1/galaxy_db/tmp/'} } } tool_id = 'upload1' env, params, runner, tool_spec = _gateway(tool_id, '', '', '') self.assertEqual(env, result['env']) self.assertEqual(params, result['params']) self.assertEqual(runner, result['runner']) self.assertEqual(tool_spec, result['tool_spec'])
def test_docker_tools(self): """ Test tools using local docker destination These tools need to have requirements allowing them to run on docker nodes only """ result = { 'params': {'requirements': 'GalaxyDockerHack == True && GalaxyGroup == "compute"'} } for tool_id, spec in TOOL_DESTINATIONS.items(): with self.subTest(tool_id=tool_id): # remove permission to test admin tools also if 'permissions' in spec: del spec['permissions'] ts = {tool_id: spec} destination = spec['runner'] if 'runner' in spec else DEFAULT_DESTINATION env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '', '', tools_spec=ts) if 'condor' in destination: if 'docker_enabled' in params and params['docker_enabled'] in ('true', 'True'): self.assertEqual(params['requirements'], result['params']['requirements'])
def test_tool_scanpy_suite(self): """ Test that it pass default values if no specifications """ tools_id = ['scanpy_inspect', 'scanpy_filter', 'scanpy_cluster_reduce_dimension', 'scanpy_normalize', 'scanpy_remove_confounders', 'scanpy_plot'] for tool_id in tools_id: result = { 'env': [{'name': 'NUMBA_CACHE_DIR', 'value': '/data/2/galaxy_db/tmp'}, {'name': 'OMP_NUM_THREADS', 'value': '4'}, {'name': 'OPENBLAS_NUM_THREADS', 'value': '4'}, {'name': 'MKL_NUM_THREADS', 'value': '4'}, {'name': 'VECLIB_MAXIMUM_THREADS', 'value': '4'}, {'name': 'NUMEXPR_NUM_THREADS', 'value': '4'}, {'name': 'NUMBA_NUM_THREADS', 'value': '4'}], 'params': {'priority': '-128', 'request_cpus': '4', 'request_memory': '4.0G', 'tmp_dir': 'True', 'requirements': 'GalaxyGroup == "compute"', 'accounting_group_user': '', 'description': tool_id}, 'runner': DEFAULT_DESTINATION, 'tool_spec': {'cores': 4, 'env': {'NUMBA_CACHE_DIR': '/data/2/galaxy_db/tmp', 'OMP_NUM_THREADS': 4, 'OPENBLAS_NUM_THREADS': 4, 'MKL_NUM_THREADS': 4, 'VECLIB_MAXIMUM_THREADS': 4, 'NUMEXPR_NUM_THREADS': 4, 'NUMBA_NUM_THREADS': 4}, 'requirements': 'GalaxyGroup == "compute"', 'mem': 4.0, 'force_destination_id': False, 'gpus': 0, 'runner': DEFAULT_DESTINATION} } env, params, runner, tool_spec, tags = _gateway(tool_id, '', '', '') d1 = {n['name']: n['value'] for n in env} d2 = {n['name']: n['value'] for n in result['env']} for k, v in d1.items(): self.assertEqual(d1[k], d2[k]) self.assertEqual(params, result['params']) self.assertEqual(runner, result['runner']) self.assertEqual(tool_spec, result['tool_spec'])
def test_runner_hint(self): """ A destination can be forced through the the user's preferences. Check if the _gateway function, fed with a proper user_roles dict, return the runner associated remote_cluster_mq_* > pulsar_eu_* condor_* > condor If None, return the expected runner from the tool's specification """ up_labels = ['remote_cluster_mq_ut01', 'condor_unittest_destination', 'None'] results = { up_labels[0]: 'pulsar_eu_ut01', up_labels[1]: 'condor', 'condor': 'condor' } t_runner_labels = [DEFAULT_DESTINATION, 'condor_unittest_destination', 'remote_cluster_mq_ut01'] for u in up_labels: for r in t_runner_labels: _tool_label = '_unittest_tool' _tool_spec = { _tool_label: { 'runner': r } } self.td[_tool_label] = _tool_spec[_tool_label] tool_id = _tool_label _user_preferences = { 'distributed_compute|remote_resources|unittest_remote_resource': u } _, _, runner, _, _ = _gateway(tool_id, _user_preferences, '', '', '', tools_spec=self.td, dest_spec=self.sp) if u == 'None': self.assertEqual(runner, results[r]) else: self.assertEqual(runner, results[u])