def assert_is_valid_xml(output):
""" Simple assertion that just verifies the specified output
is valid XML."""
try:
parse_xml_string(output)
except XMLSyntaxError as e:
raise AssertionError(
f"Expected valid XML, but could not parse output. {unicodify(e)}")
def test_parse_citation():
xml_text = EXAMPLE_BIBTEX_CITATION
citation_elem = parse_xml_string(xml_text)
with temp_directory() as test_directory:
citation = parse_citation(citation_elem, test_directory, None)
bibtex = citation.to_bibtex()
assert "title={Galaxy" in bibtex
def _replace_output_collectors(self, xml_str):
# Rewrite tool as if it had been created with output containing
# supplied dataset_collector elem.
elem = util.parse_xml_string(xml_str)
self.tool.outputs[
DEFAULT_TOOL_OUTPUT].dataset_collector_descriptions = output_collection_def.dataset_collector_descriptions_from_elem(
elem)
def test_get_env_shell_file_paths_from_setup_environment_elem():
xml = """<action name="setup_r_environment">
<repository name="package_r_3_0_1" owner="bgruening" toolshed="toolshed.g2.bx.psu.edu" changeset_revision="1234567">
<package name="R" version="3.0.1" />
</repository>
</action>
"""
mock_app = MockApp()
action_elem = parse_xml_string(xml)
required_for_install_env_sh = "/path/to/existing.sh"
all_env_paths = [required_for_install_env_sh]
action_dict = {}
r_env_sh = "/path/to/go/env.sh"
def mock_get_env_shell_file_paths(app, elem):
assert app == mock_app
assert elem.get("name") == "package_r_3_0_1"
return [r_env_sh]
with __mock_common_util_method("get_env_shell_file_paths", mock_get_env_shell_file_paths):
td_common_util.get_env_shell_file_paths_from_setup_environment_elem(
mock_app, all_env_paths, action_elem, action_dict
)
## Verify old env files weren't deleted.
assert required_for_install_env_sh in all_env_paths
## Verify new ones added.
assert r_env_sh in all_env_paths
## env_shell_file_paths includes everything
assert all([env in action_dict["env_shell_file_paths"] for env in all_env_paths])
## action_shell_file_paths includes only env files defined in
## inside the setup_ action element.
assert required_for_install_env_sh not in action_dict["action_shell_file_paths"]
assert r_env_sh in action_dict["action_shell_file_paths"]
def test_parse_citation():
xml_text = EXAMPLE_BIBTEX_CITATION
citation_elem = parse_xml_string(xml_text)
with temp_directory() as test_directory:
citation = parse_citation(citation_elem, test_directory, None)
bibtex = citation.to_bibtex()
assert "title={Galaxy" in bibtex
def __init__(self, config=None):
self.tool_sheds = {}
self.tool_sheds_auth = {}
if config:
# Parse tool_sheds_conf.xml
tree, error_message = parse_xml(config)
if tree is None:
log.warning(
"Unable to load references to tool sheds defined in file %s"
% str(config))
return
root = tree.getroot()
else:
root = parse_xml_string(DEFAULT_TOOL_SHEDS_CONF_XML)
config = "internal default config"
log.debug('Loading references to tool sheds from %s' % config)
for elem in root.findall('tool_shed'):
try:
name = elem.get('name', None)
url = elem.get('url', None)
username = elem.get('user', None)
password = elem.get('pass', None)
if name and url:
self.tool_sheds[name] = url
self.tool_sheds_auth[name] = None
log.debug('Loaded reference to tool shed: %s' % name)
if name and url and username and password:
self.tool_sheds_auth[name] = AUTH_TUPLE(username, password)
except Exception as e:
log.warning(
'Error loading reference to tool shed "{}", problem: {}'.
format(name, str(e)))
def test_xml_to_string_pretty():
section = util.parse_xml_string(SECTION_XML)
s = util.xml_to_string(section, pretty=True)
PRETTY = """<?xml version="1.0" ?>
<section id="fasta_fastq_manipulation" name="Fasta Fastq Manipulation" version="">
<tool file="toolshed.g2.bx.psu.edu/repos/peterjc/seq_filter_by_id/fb1313d79396/seq_filter_by_id/tools/seq_filter_by_id/seq_filter_by_id.xml" guid="toolshed.g2.bx.psu.edu/repos/peterjc/seq_filter_by_id/seq_filter_by_id/0.2.5">
<tool_shed>toolshed.g2.bx.psu.edu</tool_shed>
</tool>
</section>"""
assert s == PRETTY
def to_xml_file(self,
shed_tool_data_table_config,
new_elems=None,
remove_elems=None):
"""
Write the current in-memory version of the shed_tool_data_table_conf.xml file to disk.
remove_elems are removed before new_elems are added.
"""
if not (new_elems or remove_elems):
log.debug(
'ToolDataTableManager.to_xml_file called without any elements to add or remove.'
)
return # no changes provided, no need to persist any changes
if not new_elems:
new_elems = []
if not remove_elems:
remove_elems = []
full_path = os.path.abspath(shed_tool_data_table_config)
# FIXME: we should lock changing this file by other threads / head nodes
try:
try:
tree = util.parse_xml(full_path)
except OSError as e:
if e.errno == errno.ENOENT:
with open(full_path, 'w') as fh:
fh.write(TOOL_DATA_TABLE_CONF_XML)
tree = util.parse_xml(full_path)
else:
raise
root = tree.getroot()
out_elems = [elem for elem in root]
except Exception as e:
out_elems = []
log.debug(
'Could not parse existing tool data table config, assume no existing elements: %s',
e)
for elem in remove_elems:
# handle multiple occurrences of remove elem in existing elems
while elem in out_elems:
remove_elems.remove(elem)
# add new elems
out_elems.extend(new_elems)
out_path_is_new = not os.path.exists(full_path)
root = util.parse_xml_string(
'<?xml version="1.0"?>\n<tables></tables>')
for elem in out_elems:
root.append(elem)
with RenamedTemporaryFile(full_path, mode='w') as out:
out.write(util.xml_to_string(root, pretty=True))
os.chmod(full_path, RW_R__R__)
if out_path_is_new:
self.tool_data_path_files.update_files()
def assert_xml_element(output,
path,
verify_assertions_function=None,
children=None,
attribute=None,
all=False,
n: int = None,
delta: int = 0,
min: int = None,
max: int = None,
negate: bool = False):
"""
Check if path occurs in the xml. If n and delta or min and max are given
also the number of occurences is checked.
If there are any sub assertions then check them against
- the element's text if attribute is None
- the content of the attribute
If all is True then the sub assertions are checked for all occurences.
"""
children = children or []
all = asbool(all)
# assert that path is in output (the specified number of times)
xml = parse_xml_string(output)
asserts._util._assert_presence_number(
xml, path, n, delta, min, max, negate,
lambda x, p: x.find(p) is not None, lambda x, p: len(x.findall(p)),
"{expected} path '{text}' in xml",
"{expected} {n}+-{delta} occurrences of path '{text}' in xml",
"{expected} that the number of occurences of path '{text}' in xml is in [{min}:{max}]"
)
# check sub-assertions
if len(children) == 0 or verify_assertions_function is None:
return
for occ in xml.findall(path):
if attribute is None or attribute == "":
content = occ.text
else:
content = occ.attrib[attribute]
try:
verify_assertions_function(content, children)
except AssertionError as e:
if attribute is not None and attribute != "":
raise AssertionError(
f"Attribute '{attribute}' on element with path '{path}': {str(e)}"
)
else:
raise AssertionError(
f"Text of element with path '{path}': {str(e)}")
if not all:
break
def config_elems_to_xml_file(self, config_elems, config_filename, tool_path):
"""
Persist the current in-memory list of config_elems to a file named by the
value of config_filename.
"""
try:
root = parse_xml_string('<?xml version="1.0"?>\n<toolbox tool_path="%s"></toolbox>' % str(tool_path))
for elem in config_elems:
root.append(elem)
with RenamedTemporaryFile(config_filename, mode='w') as fh:
fh.write(xml_to_string(root, pretty=True))
except Exception:
log.exception("Exception in ToolPanelManager.config_elems_to_xml_file")
def config_elems_to_xml_file(self, config_elems, config_filename, tool_path, tool_cache_data_dir=None):
"""
Persist the current in-memory list of config_elems to a file named by the
value of config_filename.
"""
try:
tool_cache_data_dir = f' tool_cache_data_dir="{tool_cache_data_dir}"' if tool_cache_data_dir else ''
root = parse_xml_string(f'<?xml version="1.0"?>\n<toolbox tool_path="{tool_path}"{tool_cache_data_dir}></toolbox>')
for elem in config_elems:
root.append(elem)
with RenamedTemporaryFile(config_filename, mode='w') as fh:
fh.write(xml_to_string(root, pretty=True))
except Exception:
log.exception("Exception in ToolPanelManager.config_elems_to_xml_file")
def get_authenticators(auth_config_file, auth_config_file_set):
__plugins_dict = plugin_config.plugins_dict(galaxy.auth.providers,
'plugin_type')
# parse XML
try:
ct = parse_xml(auth_config_file)
conf_root = ct.getroot()
except (OSError, IOError) as exc:
if exc.errno == errno.ENOENT and not auth_config_file_set:
conf_root = parse_xml_string(AUTH_CONF_XML)
else:
raise
authenticators = []
# process authenticators
for auth_elem in conf_root:
type_elem_text = auth_elem.find('type').text
plugin_class = __plugins_dict.get(type_elem_text)
if not plugin_class:
raise Exception(
"Authenticator type '%s' not recognized, should be one of %s" %
(type_elem_text, ', '.join(__plugins_dict)))
plugin = plugin_class()
# check filterelem
filter_elem = auth_elem.find('filter')
if filter_elem is not None:
filter_template = str(filter_elem.text)
else:
filter_template = None
# extract options
options_elem = auth_elem.find('options')
options = {}
if options_elem is not None:
for opt in options_elem:
options[opt.tag] = opt.text
authenticator = Authenticator(
plugin=plugin,
filter_template=filter_template,
options=options,
)
authenticators.append(authenticator)
return authenticators
def test_get_env_shell_file_paths_from_setup_environment_elem():
xml = """<action name="setup_r_environment">
<repository name="package_r_3_0_1" owner="bgruening" toolshed="toolshed.g2.bx.psu.edu" changeset_revision="1234567">
<package name="R" version="3.0.1" />
</repository>
<repository name="package_zlib_1_2_8" owner="iuc" toolshed="toolshed.g2.bx.psu.edu" changeset_revision="7654321">
<package name="zlib" version="1.2.8" />
</repository>
</action>
"""
mock_app = MockApp()
action_elem = parse_xml_string(xml)
required_for_install_env_sh = '/path/to/existing.sh'
all_env_paths = [required_for_install_env_sh]
action_dict = {}
env_manager = EnvManager(mock_app)
r_env_sh = '/path/to/go/env.sh'
def mock_get_env_shell_file_paths(elem):
assert elem.get('name') in ["package_r_3_0_1", "package_zlib_1_2_8"]
return [r_env_sh]
with __mock_common_util_method(env_manager, "get_env_shell_file_paths",
mock_get_env_shell_file_paths):
env_manager.get_env_shell_file_paths_from_setup_environment_elem(
all_env_paths, action_elem, action_dict)
# Verify old env files weren't deleted.
assert required_for_install_env_sh in all_env_paths
# Verify new ones added.
assert r_env_sh in all_env_paths
# env_shell_file_paths includes everything
assert all([
env in action_dict['env_shell_file_paths'] for env in all_env_paths
])
# for every given repository there should be one env
# file + the required_for_install_env_sh file
assert len(action_dict['env_shell_file_paths']) == 3
# action_shell_file_paths includes only env files defined
# inside the setup_ action element.
assert required_for_install_env_sh in action_dict[
'action_shell_file_paths']
assert r_env_sh in action_dict['action_shell_file_paths']
def data_manager_config_elems_to_xml_file(self, config_elems,
config_filename):
"""
Persist the current in-memory list of config_elems to a file named by the value
of config_filename.
"""
data_managers_path = self.data_managers_path
if data_managers_path:
root_str = '<?xml version="1.0"?><data_managers tool_path="%s"></data_managers>' % data_managers_path
else:
root_str = '<?xml version="1.0"?><data_managers></data_managers>'
root = parse_xml_string(root_str)
for elem in config_elems:
root.append(elem)
try:
with RenamedTemporaryFile(config_filename, mode='w') as fh:
fh.write(xml_to_string(root))
except Exception:
log.exception(
"Exception in DataManagerHandler.data_manager_config_elems_to_xml_file"
)
def parse_status(self, status, job_ids):
# in case there's noise in the output, find the big blob 'o xml
tree = None
rval = {}
for line in status.strip().splitlines():
try:
tree = parse_xml_string(line.strip())
assert tree.tag == 'Data'
break
except Exception:
tree = None
if tree is None:
log.warning(f'No valid qstat XML return from `qstat -x`, got the following: {status}')
return None
else:
for job in tree.findall('Job'):
id = job.find('Job_Id').text
if id in job_ids:
state = job.find('job_state').text
# map PBS job states to Galaxy job states.
rval[id] = self._get_job_state(state)
return rval
def test_get_env_shell_file_paths_from_setup_environment_elem():
xml = """<action name="setup_r_environment">
<repository name="package_r_3_0_1" owner="bgruening" toolshed="toolshed.g2.bx.psu.edu" changeset_revision="1234567">
<package name="R" version="3.0.1" />
</repository>
<repository name="package_zlib_1_2_8" owner="iuc" toolshed="toolshed.g2.bx.psu.edu" changeset_revision="7654321">
<package name="zlib" version="1.2.8" />
</repository>
</action>
"""
mock_app = MockApp()
action_elem = parse_xml_string( xml )
required_for_install_env_sh = '/path/to/existing.sh'
all_env_paths = [ required_for_install_env_sh ]
action_dict = {}
env_manager = EnvManager( mock_app )
r_env_sh = '/path/to/go/env.sh'
def mock_get_env_shell_file_paths( elem ):
assert elem.get( 'name' ) in ["package_r_3_0_1", "package_zlib_1_2_8"]
return [ r_env_sh ]
with __mock_common_util_method( env_manager, "get_env_shell_file_paths", mock_get_env_shell_file_paths ):
env_manager.get_env_shell_file_paths_from_setup_environment_elem( all_env_paths, action_elem, action_dict )
# Verify old env files weren't deleted.
assert required_for_install_env_sh in all_env_paths
# Verify new ones added.
assert r_env_sh in all_env_paths
# env_shell_file_paths includes everything
assert all( [ env in action_dict[ 'env_shell_file_paths' ] for env in all_env_paths ] )
# for every given repository there should be one env
# file + the required_for_install_env_sh file
assert len( action_dict[ 'env_shell_file_paths' ] ) == 3
# action_shell_file_paths includes only env files defined
# inside the setup_ action element.
assert required_for_install_env_sh in action_dict[ 'action_shell_file_paths' ]
assert r_env_sh in action_dict[ 'action_shell_file_paths' ]
def test_parse_xml_string():
section = util.parse_xml_string(SECTION_XML)
_verify_section(section)
def test_parse_citation():
xml_text = EXAMPLE_BIBTEX_CITATION
citation_elem = parse_xml_string(xml_text)
citation = parse_citation(citation_elem, None)
bibtex = citation.to_bibtex()
assert "title={Galaxy" in bibtex
def to_xml(output):
return parse_xml_string(output)
def _replace_output_collectors( self, xml_str ):
# Rewrite tool as if it had been created with output containing
# supplied dataset_collector elem.
elem = util.parse_xml_string( xml_str )
self.tool.outputs[ DEFAULT_TOOL_OUTPUT ].dataset_collector_descriptions = output_collection_def.dataset_collector_descriptions_from_elem( elem )
def test_xml_to_string():
section = util.parse_xml_string(SECTION_XML)
s = util.xml_to_string(section)
assert len(s.split('\n')) == 1
