def _assert_valid_xml(self, filename):
try:
parse_xml(filename)
except Exception:
message_template = "file %s does not contain valid XML, content %s"
message = message_template % (filename, open(filename, "r").read())
raise AssertionError(message)
def test_parse_xml_enoent():
fd, path = tempfile.mkstemp()
os.close(fd)
os.remove(path)
with pytest.raises(IOError) as excinfo:
util.parse_xml(path)
assert excinfo.value.errno == errno.ENOENT
def _assert_valid_xml(self, filename):
try:
parse_xml(filename)
except Exception:
message_template = "file %s does not contain valid XML, content %s"
message = message_template % (filename, open(filename, "r").read())
raise AssertionError(message)
def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number ):
# Get the 000x_tools.xml file associated with the current migrate_tools version number.
tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) )
# Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config.
migrated_tool_configs_dict = odict()
tree = util.parse_xml( tools_xml_file_path )
root = tree.getroot()
tool_shed = root.get( 'name' )
tool_shed_url = get_tool_shed_url_from_tools_xml_file_path( app, tool_shed )
# The default behavior is that the tool shed is down.
tool_shed_accessible = False
if tool_shed_url:
for elem in root:
if elem.tag == 'repository':
tool_dependencies = []
tool_dependencies_dict = {}
repository_name = elem.get( 'name' )
changeset_revision = elem.get( 'changeset_revision' )
url = '%s/repository/get_tool_dependencies?name=%s&owner=%s&changeset_revision=%s&from_install_manager=True' % \
( tool_shed_url, repository_name, REPOSITORY_OWNER, changeset_revision )
try:
response = urllib2.urlopen( url )
text = response.read()
response.close()
tool_shed_accessible = True
except Exception, e:
# Tool shed may be unavailable - we have to set tool_shed_accessible since we're looping.
tool_shed_accessible = False
print "The URL\n%s\nraised the exception:\n%s\n" % ( url, str( e ) )
if tool_shed_accessible:
if text:
tool_dependencies_dict = encoding_util.tool_shed_decode( text )
for dependency_key, requirements_dict in tool_dependencies_dict.items():
tool_dependency_name = requirements_dict[ 'name' ]
tool_dependency_version = requirements_dict[ 'version' ]
tool_dependency_type = requirements_dict[ 'type' ]
tool_dependency_readme = requirements_dict.get( 'readme', '' )
tool_dependencies.append( ( tool_dependency_name, tool_dependency_version, tool_dependency_type, tool_dependency_readme ) )
for tool_elem in elem.findall( 'tool' ):
migrated_tool_configs_dict[ tool_elem.get( 'file' ) ] = tool_dependencies
if tool_shed_accessible:
# Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names.
missing_tool_configs_dict = odict()
for tool_panel_config in tool_panel_configs:
tree = util.parse_xml( tool_panel_config )
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
missing_tool_configs_dict = check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict )
elif elem.tag == 'section':
for section_elem in elem:
if section_elem.tag == 'tool':
missing_tool_configs_dict = check_tool_tag_set( section_elem, migrated_tool_configs_dict, missing_tool_configs_dict )
def to_xml_file(self,
shed_tool_data_table_config,
new_elems=None,
remove_elems=None):
"""
Write the current in-memory version of the shed_tool_data_table_conf.xml file to disk.
remove_elems are removed before new_elems are added.
"""
if not (new_elems or remove_elems):
log.debug(
'ToolDataTableManager.to_xml_file called without any elements to add or remove.'
)
return # no changes provided, no need to persist any changes
if not new_elems:
new_elems = []
if not remove_elems:
remove_elems = []
full_path = os.path.abspath(shed_tool_data_table_config)
# FIXME: we should lock changing this file by other threads / head nodes
try:
try:
tree = util.parse_xml(full_path)
except OSError as e:
if e.errno == errno.ENOENT:
with open(full_path, 'w') as fh:
fh.write(TOOL_DATA_TABLE_CONF_XML)
tree = util.parse_xml(full_path)
else:
raise
root = tree.getroot()
out_elems = [elem for elem in root]
except Exception as e:
out_elems = []
log.debug(
'Could not parse existing tool data table config, assume no existing elements: %s',
e)
for elem in remove_elems:
# handle multiple occurrences of remove elem in existing elems
while elem in out_elems:
remove_elems.remove(elem)
# add new elems
out_elems.extend(new_elems)
out_path_is_new = not os.path.exists(full_path)
root = util.parse_xml_string(
'<?xml version="1.0"?>\n<tables></tables>')
for elem in out_elems:
root.append(elem)
with RenamedTemporaryFile(full_path, mode='w') as out:
out.write(util.xml_to_string(root, pretty=True))
os.chmod(full_path, RW_R__R__)
if out_path_is_new:
self.tool_data_path_files.update_files()
def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_script_number ):
# Get the 000x_tools.xml file associated with the current migrate_tools version number.
tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) )
# Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config.
migrated_tool_configs_dict = odict()
tree = util.parse_xml( tools_xml_file_path )
root = tree.getroot()
tool_shed = root.get( 'name' )
tool_shed_url = get_tool_shed_url_from_tools_xml_file_path( app, tool_shed )
if tool_shed_url:
for elem in root:
if elem.tag == 'repository':
tool_dependencies = []
tool_dependencies_dict = {}
repository_name = elem.get( 'name' )
changeset_revision = elem.get( 'changeset_revision' )
url = '%s/repository/get_tool_dependencies?name=%s&owner=%s&changeset_revision=%s&webapp=install_manager&no_reset=true' % \
( tool_shed_url, repository_name, REPOSITORY_OWNER, changeset_revision )
response = urllib2.urlopen( url )
text = response.read()
response.close()
if text:
tool_dependencies_dict = tool_shed_decode( text )
for dependency_key, requirements_dict in tool_dependencies_dict.items():
tool_dependency_name = requirements_dict[ 'name' ]
tool_dependency_version = requirements_dict[ 'version' ]
tool_dependency_type = requirements_dict[ 'type' ]
tool_dependency_readme = requirements_dict.get( 'readme', '' )
tool_dependencies.append( ( tool_dependency_name, tool_dependency_version, tool_dependency_type, tool_dependency_readme ) )
for tool_elem in elem.findall( 'tool' ):
migrated_tool_configs_dict[ tool_elem.get( 'file' ) ] = tool_dependencies
# Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names.
missing_tool_configs_dict = odict()
for tool_panel_config in tool_panel_configs:
tree = util.parse_xml( tool_panel_config )
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
missing_tool_configs_dict = check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict )
elif elem.tag == 'section':
for section_elem in elem:
if section_elem.tag == 'tool':
missing_tool_configs_dict = check_tool_tag_set( section_elem, migrated_tool_configs_dict, missing_tool_configs_dict )
else:
exception_msg = '\n\nThe entry for the main Galaxy tool shed at %s is missing from the %s file. ' % ( tool_shed, app.config.tool_sheds_config )
exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n'
raise Exception( exception_msg )
return missing_tool_configs_dict
def add_new_entries_from_config_file( self, config_filename ):
"""
We have 2 cases to handle, files whose root tag is <tables>, for example:
<tables>
<!-- Location of Tmap files -->
<table name="tmap_indexes" comment_char="#">
<columns>value, dbkey, name, path</columns>
<file path="tool-data/tmap_index.loc" />
</table>
</tables>
and files whose root tag is <table>, for example:
<!-- Location of Tmap files -->
<table name="tmap_indexes" comment_char="#">
<columns>value, dbkey, name, path</columns>
<file path="tool-data/tmap_index.loc" />
</table>
"""
tree = util.parse_xml( config_filename )
root = tree.getroot()
if root.tag == 'tables':
table_elems = self.load_from_config_file( config_filename )
else:
table_elems = []
type = root.get( 'type', 'tabular' )
assert type in tool_data_table_types, "Unknown data table type '%s'" % type
table_elems.append( root )
table = tool_data_table_types[ type ]( root )
if table.name not in self.data_tables:
self.data_tables[ table.name ] = table
log.debug( "Loaded tool data table '%s", table.name )
return table_elems
def add_new_entries_from_config_file( self, config_filename, tool_data_path, shed_tool_data_table_config, persist=False ):
"""
This method is called when a tool shed repository that includes a tool_data_table_conf.xml.sample file is being
installed into a local galaxy instance. We have 2 cases to handle, files whose root tag is <tables>, for example::
<tables>
<!-- Location of Tmap files -->
<table name="tmap_indexes" comment_char="#">
<columns>value, dbkey, name, path</columns>
<file path="tool-data/tmap_index.loc" />
</table>
</tables>
and files whose root tag is <table>, for example::
<!-- Location of Tmap files -->
<table name="tmap_indexes" comment_char="#">
<columns>value, dbkey, name, path</columns>
<file path="tool-data/tmap_index.loc" />
</table>
"""
error_message = ''
table_elems = []
try:
tree = util.parse_xml( config_filename )
root = tree.getroot()
except Exception, e:
error_message = 'Error attempting to parse file %s: %s' % ( str( os.path.split( config_filename )[ 1 ] ), str( e ) )
log.debug( error_message )
return table_elems, error_message
def reload(self):
if self._filename:
elem = parse_xml(self._filename).getroot()
elif self._elem:
elem = self._elem
else:
raise Exception("Unable to reload DisplayApplication %s." % (self.name))
# All toolshed-specific attributes added by e.g the registry will remain
attr_dict = self._get_attributes_from_elem(elem)
# We will not allow changing the id at this time (we'll need to fix several mappings upstream to handle this case)
assert attr_dict.get("id") == self.id, ValueError(
"You cannot reload a Display application where the ID has changed. You will need to restart the server instead."
)
# clear old links
for key in self.links.keys():
del self.links[key]
# clear data table versions:
for key in self._data_table_versions.keys():
del self._data_table_versions[key]
# Set new attributes
for key, value in attr_dict.iteritems():
setattr(self, key, value)
# Load new links
self._load_links_from_elem(elem)
return self
def load_from_xml(self, xml_filename, store_tool_path=True):
try:
tree = util.parse_xml(xml_filename)
except (IOError, OSError) as e:
if e.errno != errno.ENOENT or self.app.config.data_manager_config_file_set:
raise
return # default config option and it doesn't exist, which is fine
except Exception as e:
log.error(
'There was an error parsing your Data Manager config file "%s": %s'
% (xml_filename, e))
return # we are not able to load any data managers
root = tree.getroot()
if root.tag != 'data_managers':
log.error(
'A data managers configuration must have a "data_managers" tag as the root. "%s" is present'
% (root.tag))
return
if store_tool_path:
tool_path = root.get('tool_path', None)
if tool_path is None:
tool_path = self.app.config.tool_path
if not tool_path:
tool_path = '.'
self.tool_path = tool_path
for data_manager_elem in root.findall('data_manager'):
if not self.load_manager_from_elem(data_manager_elem,
tool_path=self.tool_path):
# Wasn't able to load manager, could happen when galaxy is managed by planemo.
# Fall back to loading relative to the data_manager_conf.xml file
tool_path = os.path.dirname(xml_filename)
self.load_manager_from_elem(data_manager_elem,
tool_path=tool_path)
def __parse_distributed_config(self, config):
tree = util.parse_xml(self.distributed_config)
root = tree.getroot()
log.debug('Loading backends for distributed object store from %s' % self.distributed_config)
self.global_max_percent_full = float(root.get('maxpctfull', 0))
for elem in [ e for e in root if e.tag == 'backend' ]:
id = elem.get('id')
weight = int(elem.get('weight', 1))
maxpctfull = float(elem.get('maxpctfull', 0))
if elem.get('type', 'disk'):
path = None
extra_dirs = {}
for sub in elem:
if sub.tag == 'files_dir':
path = sub.get('path')
elif sub.tag == 'extra_dir':
type = sub.get('type')
extra_dirs[type] = sub.get('path')
self.backends[id] = DiskObjectStore(config, file_path=path, extra_dirs=extra_dirs)
self.max_percent_full[id] = maxpctfull
log.debug("Loaded disk backend '%s' with weight %s and file_path: %s" % (id, weight, path))
if extra_dirs:
log.debug(" Extra directories:")
for type, dir in extra_dirs.items():
log.debug(" %s: %s" % (type, dir))
for i in range(0, weight):
# The simplest way to do weighting: add backend ids to a
# sequence the number of times equalling weight, then randomly
# choose a backend from that sequence at creation
self.weighted_backend_ids.append(id)
self.original_weighted_backend_ids = self.weighted_backend_ids
def _parse_oidc_backends_config(self, config_file):
self.oidc_backends_config = {}
self.oidc_backends_implementation = {}
try:
tree = parse_xml(config_file)
root = tree.getroot()
if root.tag != 'OIDC':
raise etree.ParseError("The root element in OIDC config xml file is expected to be `OIDC`, "
"found `{}` instead -- unable to continue.".format(root.tag))
for child in root:
if child.tag != 'provider':
log.error("Expect a node with `provider` tag, found a node with `{}` tag instead; "
"skipping the node.".format(child.tag))
continue
if 'name' not in child.attrib:
log.error("Could not find a node attribute 'name'; skipping the node '{}'.".format(child.tag))
continue
idp = child.get('name').lower()
if idp in BACKENDS_NAME:
self.oidc_backends_config[idp] = self._parse_idp_config(child)
self.oidc_backends_implementation[idp] = 'psa'
self.app.config.oidc[idp] = {'icon': self._get_idp_icon(idp)}
elif idp in KEYCLOAK_BACKENDS:
self.oidc_backends_config[idp] = self._parse_custos_config(child)
self.oidc_backends_implementation[idp] = 'custos'
self.app.config.oidc[idp] = {'icon': self._get_idp_icon(idp)}
else:
raise etree.ParseError("Unknown provider specified")
if len(self.oidc_backends_config) == 0:
raise etree.ParseError("No valid provider configuration parsed.")
except ImportError:
raise
except etree.ParseError as e:
raise etree.ParseError("Invalid configuration at `{}`: {} -- unable to continue.".format(config_file, e))
def _parse_oidc_config(self, config_file):
self.oidc_config = {}
try:
tree = parse_xml(config_file)
root = tree.getroot()
if root.tag != 'OIDC':
raise etree.ParseError("The root element in OIDC_Config xml file is expected to be `OIDC`, "
"found `{}` instead -- unable to continue.".format(root.tag))
for child in root:
if child.tag != 'Setter':
log.error("Expect a node with `Setter` tag, found a node with `{}` tag instead; "
"skipping this node.".format(child.tag))
continue
if 'Property' not in child.attrib or 'Value' not in child.attrib or 'Type' not in child.attrib:
log.error("Could not find the node attributes `Property` and/or `Value` and/or `Type`;"
" found these attributes: `{}`; skipping this node.".format(child.attrib))
continue
try:
if child.get('Type') == "bool":
func = string_as_bool
else:
func = getattr(builtins, child.get('Type'))
except AttributeError:
log.error("The value of attribute `Type`, `{}`, is not a valid built-in type;"
" skipping this node").format(child.get('Type'))
continue
self.oidc_config[child.get('Property')] = func(child.get('Value'))
except ImportError:
raise
except etree.ParseError as e:
raise etree.ParseError("Invalid configuration at `{}`: {} -- unable to continue.".format(config_file, e))
def to_xml_file(self,
shed_tool_data_table_config,
new_elems=None,
remove_elems=None):
"""
Write the current in-memory version of the shed_tool_data_table_conf.xml file to disk.
remove_elems are removed before new_elems are added.
"""
if not (new_elems or remove_elems):
log.debug(
'ToolDataTableManager.to_xml_file called without any elements to add or remove.'
)
return #no changes provided, no need to persist any changes
if not new_elems:
new_elems = []
if not remove_elems:
remove_elems = []
full_path = os.path.abspath(shed_tool_data_table_config)
#FIXME: we should lock changing this file by other threads / head nodes
try:
tree = util.parse_xml(full_path)
root = tree.getroot()
out_elems = [elem for elem in root]
except Exception, e:
out_elems = []
log.debug(
'Could not parse existing tool data table config, assume no existing elements: %s',
e)
def parse_test_file(workflow_test_file):
tree = parse_xml(workflow_test_file)
root = tree.getroot()
input_elems = root.findall("input")
required_files = []
dataset_dict = {}
for input_elem in input_elems:
name, value, attrib = parse_param_elem(input_elem)
require_file(name, value, attrib, required_files)
dataset_dict[name] = value
outputs = parse_output_elems(root)
workflow_file_rel_path = root.get('file')
if not workflow_file_rel_path:
raise Exception(
"Workflow test XML must declare file attribute pointing to workflow under test."
)
# TODO: Normalize this path, prevent it from accessing arbitrary files on system.
worfklow_file_abs_path = os.path.join(os.path.dirname(workflow_test_file),
workflow_file_rel_path)
return WorkflowTest(
dataset_dict,
required_files,
worfklow_file_abs_path,
outputs=outputs,
)
def load_from_xml(self,
xml_filename,
store_tool_path=True,
replace_existing=False):
try:
tree = util.parse_xml(xml_filename)
except Exception as e:
log.error(
'There was an error parsing your Data Manager config file "%s": %s'
% (xml_filename, e))
return # we are not able to load any data managers
root = tree.getroot()
if root.tag != 'data_managers':
log.error(
'A data managers configuration must have a "data_managers" tag as the root. "%s" is present'
% (root.tag))
return
if store_tool_path:
tool_path = root.get('tool_path', None)
if tool_path is None:
tool_path = self.app.config.tool_path
if not tool_path:
tool_path = '.'
self.tool_path = tool_path
for data_manager_elem in root.findall('data_manager'):
self.load_manager_from_elem(data_manager_elem,
replace_existing=replace_existing)
def reload(self):
if self._filename:
elem = parse_xml(self._filename).getroot()
elif self._elem:
elem = self._elem
else:
raise Exception("Unable to reload DisplayApplication %s." %
(self.name))
# All toolshed-specific attributes added by e.g the registry will remain
attr_dict = self._get_attributes_from_elem(elem)
# We will not allow changing the id at this time (we'll need to fix several mappings upstream to handle this case)
assert attr_dict.get('id') == self.id, ValueError(
"You cannot reload a Display application where the ID has changed. You will need to restart the server instead."
)
# clear old links
for key in self.links.keys():
del self.links[key]
# clear data table versions:
for key in self._data_table_versions.keys():
del self._data_table_versions[key]
# Set new attributes
for key, value in attr_dict.iteritems():
setattr(self, key, value)
# Load new links
self._load_links_from_elem(elem)
return self
def load_from_config_file(self, config_filename, tool_data_path, from_shed_config=False):
"""
This method is called under 3 conditions:
1. When the ToolDataTableManager is initialized (see __init__ above).
2. Just after the ToolDataTableManager is initialized and the additional entries defined by shed_tool_data_table_conf.xml
are being loaded into the ToolDataTableManager.data_tables.
3. When a tool shed repository that includes a tool_data_table_conf.xml.sample file is being installed into a local
Galaxy instance. In this case, we have 2 entry types to handle, files whose root tag is <tables>, for example:
"""
table_elems = []
if not isinstance(config_filename, list):
config_filename = [config_filename]
for filename in config_filename:
tree = util.parse_xml(filename)
root = tree.getroot()
for table_elem in root.findall('table'):
table = ToolDataTable.from_elem(table_elem, tool_data_path, from_shed_config, filename=filename, tool_data_path_files=self.tool_data_path_files)
table_elems.append(table_elem)
if table.name not in self.data_tables:
self.data_tables[table.name] = table
log.debug("Loaded tool data table '%s' from file '%s'", table.name, filename)
else:
log.debug("Loading another instance of data table '%s' from file '%s', attempting to merge content.", table.name, filename)
self.data_tables[table.name].merge_tool_data_table(table, allow_duplicates=False) # only merge content, do not persist to disk, do not allow duplicate rows when merging
# FIXME: This does not account for an entry with the same unique build ID, but a different path.
return table_elems
def __init_schedulers(self):
config_file = self.app.config.workflow_schedulers_config_file
use_default_scheduler = False
if not config_file or (
not os.path.exists(config_file) and
not self.app.config.is_set('workflow_schedulers_config_file')):
log.info(
"No workflow schedulers plugin config file defined, using default scheduler."
)
use_default_scheduler = True
elif not os.path.exists(config_file):
log.info(
f"Cannot find workflow schedulers plugin config file '{config_file}', using default scheduler."
)
use_default_scheduler = True
if use_default_scheduler:
self.__init_default_scheduler()
else:
self.DEFAULT_BASE_HANDLER_POOLS = ('workflow-schedulers', )
plugins_element = parse_xml(config_file).getroot()
self.__init_schedulers_for_element(plugins_element)
if not self.__handlers_configured and self.__stack_has_pool:
# Stack has a pool for us so override inherited config and use the pool
self.__init_handlers()
self.__handlers_configured = True
elif use_default_scheduler:
self._set_default_handler_assignment_methods()
def to_xml_file( self, shed_tool_data_table_config, new_elems=None, remove_elems=None ):
"""
Write the current in-memory version of the shed_tool_data_table_conf.xml file to disk.
remove_elems are removed before new_elems are added.
"""
if not ( new_elems or remove_elems ):
log.debug( 'ToolDataTableManager.to_xml_file called without any elements to add or remove.' )
return # no changes provided, no need to persist any changes
if not new_elems:
new_elems = []
if not remove_elems:
remove_elems = []
full_path = os.path.abspath( shed_tool_data_table_config )
# FIXME: we should lock changing this file by other threads / head nodes
try:
tree = util.parse_xml( full_path )
root = tree.getroot()
out_elems = [ elem for elem in root ]
except Exception as e:
out_elems = []
log.debug( 'Could not parse existing tool data table config, assume no existing elements: %s', e )
for elem in remove_elems:
# handle multiple occurrences of remove elem in existing elems
while elem in out_elems:
remove_elems.remove( elem )
# add new elems
out_elems.extend( new_elems )
with open( full_path, 'wb' ) as out:
out.write( '<?xml version="1.0"?>\n<tables>\n' )
for elem in out_elems:
out.write( util.xml_to_string( elem, pretty=True ) )
out.write( '</tables>\n' )
os.chmod( full_path, 0644 )
def __init_schedulers(self):
config_file = self.app.config.workflow_schedulers_config_file
use_default_scheduler = False
if not config_file or (
not os.path.exists(config_file)
and not self.app.config.workflow_schedulers_config_file_set):
log.info(
"No workflow schedulers plugin config file defined, using default scheduler."
)
use_default_scheduler = True
elif not os.path.exists(config_file):
log.info(
"Cannot find workflow schedulers plugin config file '%s', using default scheduler."
% config_file)
use_default_scheduler = True
if use_default_scheduler:
self.__init_default_scheduler()
else:
plugins_element = parse_xml(config_file).getroot()
self.__init_schedulers_for_element(plugins_element)
if not self.__handlers_configured and self.__stack_has_pool:
# Stack has a pool for us so override inherited config and use the pool
self.__init_handlers()
self.__handlers_configured = True
def load_from_config_file(self, config_filename, tool_data_path, from_shed_config=False):
"""
This method is called under 3 conditions:
1. When the ToolDataTableManager is initialized (see __init__ above).
2. Just after the ToolDataTableManager is initialized and the additional entries defined by shed_tool_data_table_conf.xml
are being loaded into the ToolDataTableManager.data_tables.
3. When a tool shed repository that includes a tool_data_table_conf.xml.sample file is being installed into a local
Galaxy instance. In this case, we have 2 entry types to handle, files whose root tag is <tables>, for example:
"""
table_elems = []
if not isinstance(config_filename, list):
config_filename = [config_filename]
for filename in config_filename:
tree = util.parse_xml(filename)
root = tree.getroot()
for table_elem in root.findall('table'):
table = ToolDataTable.from_elem(table_elem, tool_data_path, from_shed_config, filename=filename, tool_data_path_files=self.tool_data_path_files)
table_elems.append(table_elem)
if table.name not in self.data_tables:
self.data_tables[table.name] = table
log.debug("Loaded tool data table '%s' from file '%s'", table.name, filename)
else:
log.debug("Loading another instance of data table '%s' from file '%s', attempting to merge content.", table.name, filename)
self.data_tables[table.name].merge_tool_data_table(table, allow_duplicates=False) # only merge content, do not persist to disk, do not allow duplicate rows when merging
# FIXME: This does not account for an entry with the same unique build ID, but a different path.
return table_elems
def __parse_distributed_config(self, config):
tree = util.parse_xml(self.distributed_config)
root = tree.getroot()
log.debug('Loading backends for distributed object store from %s' % self.distributed_config)
self.global_max_percent_full = float(root.get('maxpctfull', 0))
for elem in [ e for e in root if e.tag == 'backend' ]:
id = elem.get('id')
weight = int(elem.get('weight', 1))
maxpctfull = float(elem.get('maxpctfull', 0))
if elem.get('type', 'disk'):
path = None
extra_dirs = {}
for sub in elem:
if sub.tag == 'files_dir':
path = sub.get('path')
elif sub.tag == 'extra_dir':
type = sub.get('type')
extra_dirs[type] = sub.get('path')
self.backends[id] = DiskObjectStore(config, file_path=path, extra_dirs=extra_dirs)
self.max_percent_full[id] = maxpctfull
log.debug("Loaded disk backend '%s' with weight %s and file_path: %s" % (id, weight, path))
if extra_dirs:
log.debug(" Extra directories:")
for type, dir in extra_dirs.items():
log.debug(" %s: %s" % (type, dir))
for i in range(0, weight):
# The simplest way to do weighting: add backend ids to a
# sequence the number of times equalling weight, then randomly
# choose a backend from that sequence at creation
self.weighted_backend_ids.append(id)
self.original_weighted_backend_ids = self.weighted_backend_ids
def parse_test_file( workflow_test_file ):
tree = parse_xml( workflow_test_file )
root = tree.getroot()
input_elems = root.findall( "input" )
required_files = []
dataset_dict = {}
for input_elem in input_elems:
name, value, attrib = parse_param_elem( input_elem )
require_file( name, value, attrib, required_files )
dataset_dict[ name ] = value
outputs = parse_output_elems( root )
workflow_file_rel_path = root.get( 'file' )
if not workflow_file_rel_path:
raise Exception( "Workflow test XML must declare file attribute pointing to workflow under test." )
# TODO: Normalize this path, prevent it from accessing arbitrary files on system.
worfklow_file_abs_path = os.path.join( os.path.dirname( workflow_test_file ), workflow_file_rel_path )
return WorkflowTest(
dataset_dict,
required_files,
worfklow_file_abs_path,
outputs=outputs,
)
def load_from_config_file( self, config_filename, tool_data_path, from_shed_config=False ):
"""
This method is called under 3 conditions:
1. When the ToolDataTableManager is initialized (see __init__ above).
2. Just after the ToolDataTableManager is initialized and the additional entries defined by shed_tool_data_table_conf.xml
are being loaded into the ToolDataTableManager.data_tables.
3. When a tool shed repository that includes a tool_data_table_conf.xml.sample file is being installed into a local
Galaxy instance. In this case, we have 2 entry types to handle, files whose root tag is <tables>, for example:
"""
tree = util.parse_xml( config_filename )
root = tree.getroot()
table_elems = []
for table_elem in root.findall( 'table' ):
type = table_elem.get( 'type', 'tabular' )
assert type in tool_data_table_types, "Unknown data table type '%s'" % type
table_elems.append( table_elem )
table_elem_name = table_elem.get( 'name', None )
if table_elem_name and table_elem_name not in self.data_table_elem_names:
self.data_table_elem_names.append( table_elem_name )
if from_shed_config:
self.shed_data_table_elems.append( table_elem )
table = tool_data_table_types[ type ]( table_elem, tool_data_path )
if table.name not in self.data_tables:
self.data_tables[ table.name ] = table
log.debug( "Loaded tool data table '%s'", table.name )
return table_elems
def plugin_source_from_path(path):
if path.endswith(".yaml") or path.endswith(".yml") or path.endswith(
".yaml.sample") or path.endswith(".yml.sample"):
return PluginConfigSource('dict', __read_yaml(path))
else:
return PluginConfigSource(
'xml',
parse_xml(path, remove_comments=True).getroot())
def parse_file(self, xml_filepath):
"""
Parse the given XML file for visualizations data.
:returns: visualization config dictionary
"""
xml_tree = util.parse_xml(xml_filepath)
visualization = self.parse_visualization(xml_tree.getroot())
return visualization
def build_object_store_from_config(config,
fsmon=False,
config_xml=None,
config_dict=None):
"""
Invoke the appropriate object store.
Will use the `object_store_config_file` attribute of the `config` object to
configure a new object store from the specified XML file.
Or you can specify the object store type in the `object_store` attribute of
the `config` object. Currently 'disk', 's3', 'swift', 'distributed',
'hierarchical', 'irods', and 'pulsar' are supported values.
"""
from_object = 'xml'
if config is None and config_dict is not None and 'config' in config_dict:
# Build a config object from to_dict of an ObjectStore.
config = Bunch(**config_dict["config"])
elif config is None:
raise Exception(
"build_object_store_from_config sent None as config parameter and one cannot be recovered from config_dict"
)
if config_xml is None and config_dict is None:
config_file = config.object_store_config_file
if os.path.exists(config_file):
if config_file.endswith(".xml") or config_file.endswith(
".xml.sample"):
# This is a top level invocation of build_object_store_from_config, and
# we have an object_store_conf.xml -- read the .xml and build
# accordingly
config_xml = parse_xml(
config.object_store_config_file).getroot()
store = config_xml.get('type')
else:
with open(config_file) as f:
config_dict = yaml.safe_load(f)
from_object = 'dict'
store = config_dict.get('type')
else:
store = config.object_store
elif config_xml is not None:
store = config_xml.get('type')
elif config_dict is not None:
from_object = 'dict'
store = config_dict.get('type')
objectstore_class, objectstore_constructor_kwds = type_to_object_store_class(
store, fsmon=fsmon)
if from_object == 'xml':
return objectstore_class.from_xml(config=config,
config_xml=config_xml,
**objectstore_constructor_kwds)
else:
return objectstore_class(config=config,
config_dict=config_dict,
**objectstore_constructor_kwds)
def get_proprietary_tool_panel_elems(self,
latest_tool_migration_script_number):
# Parse each config in self.proprietary_tool_confs (the default is tool_conf.xml) and generate a list of Elements that are
# either ToolSection elements or Tool elements. These will be used to generate new entries in the migrated_tools_conf.xml
# file for the installed tools.
tools_xml_file_path = os.path.abspath(
os.path.join(
'scripts', 'migrate_tools',
'%04d_tools.xml' % latest_tool_migration_script_number))
# Parse the XML and load the file attributes for later checking against the integrated elements from self.proprietary_tool_confs.
migrated_tool_configs = []
tree = util.parse_xml(tools_xml_file_path)
root = tree.getroot()
for elem in root:
if elem.tag == 'repository':
for tool_elem in elem:
migrated_tool_configs.append(tool_elem.get('file'))
# Parse each file in self.proprietary_tool_confs and generate the integrated list of tool panel Elements that contain them.
tool_panel_elems = []
for proprietary_tool_conf in self.proprietary_tool_confs:
tree = util.parse_xml(proprietary_tool_conf)
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
# Tools outside of sections.
file_path = elem.get('file', None)
if file_path:
name = suc.strip_path(file_path)
if name in migrated_tool_configs:
if elem not in tool_panel_elems:
tool_panel_elems.append(elem)
elif elem.tag == 'section':
# Tools contained in a section.
for section_elem in elem:
if section_elem.tag == 'tool':
file_path = section_elem.get('file', None)
if file_path:
name = suc.strip_path(file_path)
if name in migrated_tool_configs:
# Append the section, not the tool.
if elem not in tool_panel_elems:
tool_panel_elems.append(elem)
return tool_panel_elems
def __init__(self, dependencies_file):
self.root = parse_xml(dependencies_file).getroot()
dependencies = []
package_els = self.root.findall("package") or []
for package_el in package_els:
repository_el = package_el.find("repository")
if repository_el is None:
continue
dependency = RawDependency(self, package_el, repository_el)
dependencies.append(dependency)
self.dependencies = dependencies
def load_from_xml(self,
xml_filename,
store_tool_path=True,
replace_existing=False):
try:
tree = util.parse_xml(xml_filename)
except Exception, e:
log.error(
'There was an error parsing your Data Manager config file "%s": %s'
% (xml_filename, e))
return #we are not able to load any data managers
def get_non_shed_tool_panel_configs( app ):
# Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml.
config_filenames = []
for config_filename in app.config.tool_configs:
# Any config file that includes a tool_path attribute in the root tag set like the following is shed-related.
# <toolbox tool_path="../shed_tools">
tree = util.parse_xml( config_filename )
root = tree.getroot()
tool_path = root.get( 'tool_path', None )
if tool_path is None:
config_filenames.append( config_filename )
return config_filenames
def get_non_shed_tool_panel_configs( app ):
# Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml.
config_filenames = []
for config_filename in app.config.tool_configs:
# Any config file that includes a tool_path attribute in the root tag set like the following is shed-related.
# <toolbox tool_path="../shed_tools">
tree = util.parse_xml( config_filename )
root = tree.getroot()
tool_path = root.get( 'tool_path', None )
if tool_path is None:
config_filenames.append( config_filename )
return config_filenames
def load_from_element(self, elem, tool_path):
assert (
elem.tag == "data_manager"
), 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % (elem.tag)
self.declared_id = elem.get("id", None)
self.guid = elem.get("guid", None)
path = elem.get("tool_file", None)
self.version = elem.get("version", self.version)
tool_shed_repository_id = None
tool_guid = None
if path is None:
tool_elem = elem.find("tool")
assert tool_elem is not None, (
"Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s"
% (util.xml_to_string(elem))
)
path = tool_elem.get("file", None)
tool_guid = tool_elem.get("guid", None)
# need to determine repository info so that dependencies will work correctly
tool_shed = tool_elem.find("tool_shed").text
repository_name = tool_elem.find("repository_name").text
repository_owner = tool_elem.find("repository_owner").text
installed_changeset_revision = tool_elem.find("installed_changeset_revision").text
# save repository info here
self.tool_shed_repository_info_dict = dict(
tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision,
)
# get tool_shed repo id
tool_shed_repository = suc.get_tool_shed_repository_by_shed_name_owner_installed_changeset_revision(
self.data_managers.app, tool_shed, repository_name, repository_owner, installed_changeset_revision
)
if tool_shed_repository is None:
log.warning(
"Could not determine tool shed repository from database. This should only ever happen when running tests."
)
# we'll set tool_path manually here from shed_conf_file
tool_shed_repository_id = None
try:
tool_path = util.parse_xml(elem.get("shed_conf_file")).getroot().get("tool_path", tool_path)
except Exception, e:
log.error("Error determining tool_path for Data Manager during testing: %s", e)
else:
tool_shed_repository_id = self.data_managers.app.security.encode_id(tool_shed_repository.id)
# use shed_conf_file to determine tool_path
shed_conf_file = elem.get("shed_conf_file", None)
if shed_conf_file:
shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename(shed_conf_file, None)
if shed_conf:
tool_path = shed_conf.get("tool_path", tool_path)
def parse_tool_panel_config(config, shed_tools_dict):
"""
Parse a shed-related tool panel config to generate the shed_tools_dict. This only happens when testing tools installed from the tool shed.
"""
last_galaxy_test_file_dir = None
last_tested_repository_name = None
last_tested_changeset_revision = None
tool_path = None
has_test_data = False
tree = parse_xml(config)
root = tree.getroot()
tool_path = root.get('tool_path')
for elem in root:
if elem.tag == 'tool':
galaxy_test_file_dir, \
last_tested_repository_name, \
last_tested_changeset_revision = get_installed_repository_info( elem,
last_galaxy_test_file_dir,
last_tested_repository_name,
last_tested_changeset_revision,
tool_path )
if galaxy_test_file_dir:
if not has_test_data:
has_test_data = True
if galaxy_test_file_dir != last_galaxy_test_file_dir:
if not os.path.isabs(galaxy_test_file_dir):
galaxy_test_file_dir = os.path.join(
galaxy_root, galaxy_test_file_dir)
guid = elem.get('guid')
shed_tools_dict[guid] = galaxy_test_file_dir
last_galaxy_test_file_dir = galaxy_test_file_dir
elif elem.tag == 'section':
for section_elem in elem:
if section_elem.tag == 'tool':
galaxy_test_file_dir, \
last_tested_repository_name, \
last_tested_changeset_revision = get_installed_repository_info( section_elem,
last_galaxy_test_file_dir,
last_tested_repository_name,
last_tested_changeset_revision,
tool_path )
if galaxy_test_file_dir:
if not has_test_data:
has_test_data = True
if galaxy_test_file_dir != last_galaxy_test_file_dir:
if not os.path.isabs(galaxy_test_file_dir):
galaxy_test_file_dir = os.path.join(
galaxy_root, galaxy_test_file_dir)
guid = section_elem.get('guid')
shed_tools_dict[guid] = galaxy_test_file_dir
last_galaxy_test_file_dir = galaxy_test_file_dir
return has_test_data, shed_tools_dict
def load_from_config_file( self, config_filename ):
tree = util.parse_xml( config_filename )
root = tree.getroot()
table_elems = []
for table_elem in root.findall( 'table' ):
type = table_elem.get( 'type', 'tabular' )
assert type in tool_data_table_types, "Unknown data table type '%s'" % type
table_elems.append( table_elem )
table = tool_data_table_types[ type ]( table_elem )
if table.name not in self.data_tables:
self.data_tables[ table.name ] = table
log.debug( "Loaded tool data table '%s'", table.name )
return table_elems
def get_proprietary_tool_panel_elems( self, latest_tool_migration_script_number ):
# Parse each config in self.proprietary_tool_confs (the default is tool_conf.xml) and generate a list of Elements that are
# either ToolSection elements or Tool elements. These will be used to generate new entries in the migrated_tools_conf.xml
# file for the installed tools.
tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) )
# Parse the XML and load the file attributes for later checking against the integrated elements from self.proprietary_tool_confs.
migrated_tool_configs = []
tree = util.parse_xml( tools_xml_file_path )
root = tree.getroot()
for elem in root:
if elem.tag == 'repository':
for tool_elem in elem:
migrated_tool_configs.append( tool_elem.get( 'file' ) )
# Parse each file in self.proprietary_tool_confs and generate the integrated list of tool panel Elements that contain them.
tool_panel_elems = []
for proprietary_tool_conf in self.proprietary_tool_confs:
tree = util.parse_xml( proprietary_tool_conf )
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
# Tools outside of sections.
file_path = elem.get( 'file', None )
if file_path:
name = suc.strip_path( file_path )
if name in migrated_tool_configs:
if elem not in tool_panel_elems:
tool_panel_elems.append( elem )
elif elem.tag == 'section':
# Tools contained in a section.
for section_elem in elem:
if section_elem.tag == 'tool':
file_path = section_elem.get( 'file', None )
if file_path:
name = suc.strip_path( file_path )
if name in migrated_tool_configs:
# Append the section, not the tool.
if elem not in tool_panel_elems:
tool_panel_elems.append( elem )
return tool_panel_elems
def parse_tool_panel_config( config, shed_tools_dict ):
"""
Parse a shed-related tool panel config to generate the shed_tools_dict. This only happens when testing tools installed from the tool shed.
"""
last_galaxy_test_file_dir = None
last_tested_repository_name = None
last_tested_changeset_revision = None
tool_path = None
has_test_data = False
tree = parse_xml( config )
root = tree.getroot()
tool_path = root.get('tool_path')
for elem in root:
if elem.tag == 'tool':
galaxy_test_file_dir, \
last_tested_repository_name, \
last_tested_changeset_revision = get_installed_repository_info( elem,
last_galaxy_test_file_dir,
last_tested_repository_name,
last_tested_changeset_revision,
tool_path )
if galaxy_test_file_dir:
if not has_test_data:
has_test_data = True
if galaxy_test_file_dir != last_galaxy_test_file_dir:
if not os.path.isabs( galaxy_test_file_dir ):
galaxy_test_file_dir = os.path.join( galaxy_root, galaxy_test_file_dir )
guid = elem.get( 'guid' )
shed_tools_dict[ guid ] = galaxy_test_file_dir
last_galaxy_test_file_dir = galaxy_test_file_dir
elif elem.tag == 'section':
for section_elem in elem:
if section_elem.tag == 'tool':
galaxy_test_file_dir, \
last_tested_repository_name, \
last_tested_changeset_revision = get_installed_repository_info( section_elem,
last_galaxy_test_file_dir,
last_tested_repository_name,
last_tested_changeset_revision,
tool_path )
if galaxy_test_file_dir:
if not has_test_data:
has_test_data = True
if galaxy_test_file_dir != last_galaxy_test_file_dir:
if not os.path.isabs( galaxy_test_file_dir ):
galaxy_test_file_dir = os.path.join( galaxy_root, galaxy_test_file_dir )
guid = section_elem.get( 'guid' )
shed_tools_dict[ guid ] = galaxy_test_file_dir
last_galaxy_test_file_dir = galaxy_test_file_dir
return has_test_data, shed_tools_dict
def check_for_missing_tools( tool_panel_configs, latest_tool_migration_script_number ):
# Get the 000x_tools.xml file associated with the current migrate_tools version number.
tools_xml_file_path = os.path.abspath( os.path.join( 'scripts', 'migrate_tools', '%04d_tools.xml' % latest_tool_migration_script_number ) )
# Parse the XML and load the file attributes for later checking against the proprietary tool_panel_config.
migrated_tool_configs = []
tree = util.parse_xml( tools_xml_file_path )
root = tree.getroot()
for elem in root:
if elem.tag == 'repository':
for tool_elem in elem.findall( 'tool' ):
migrated_tool_configs.append( tool_elem.get( 'file' ) )
# Parse the proprietary tool_panel_configs (the default is tool_conf.xml) and generate the list of missing tool config file names.
missing_tool_configs = []
for tool_panel_config in tool_panel_configs:
tree = util.parse_xml( tool_panel_config )
root = tree.getroot()
for elem in root:
if elem.tag == 'tool':
missing_tool_configs = check_tool_tag_set( elem, migrated_tool_configs, missing_tool_configs )
elif elem.tag == 'section':
for section_elem in elem:
if section_elem.tag == 'tool':
missing_tool_configs = check_tool_tag_set( section_elem, migrated_tool_configs, missing_tool_configs )
return missing_tool_configs
def load_from_element( self, elem, tool_path ):
assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % ( elem.tag )
self.declared_id = elem.get( 'id', None )
self.guid = elem.get( 'guid', None )
path = elem.get( 'tool_file', None )
self.version = elem.get( 'version', self.version )
tool_shed_repository_id = None
tool_guid = None
if path is None:
tool_elem = elem.find( 'tool' )
assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % ( util.xml_to_string( elem ) )
path = tool_elem.get( "file", None )
tool_guid = tool_elem.get( "guid", None )
# need to determine repository info so that dependencies will work correctly
tool_shed_url = tool_elem.find( 'tool_shed' ).text
# Handle protocol changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.data_managers.app, tool_shed_url )
# The protocol is not stored in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url( tool_shed_url )
repository_name = tool_elem.find( 'repository_name' ).text
repository_owner = tool_elem.find( 'repository_owner' ).text
installed_changeset_revision = tool_elem.find( 'installed_changeset_revision' ).text
self.tool_shed_repository_info_dict = dict( tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision )
tool_shed_repository = \
suc.get_installed_repository( self.data_managers.app,
tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision )
if tool_shed_repository is None:
log.warning( 'Could not determine tool shed repository from database. This should only ever happen when running tests.' )
# we'll set tool_path manually here from shed_conf_file
tool_shed_repository_id = None
try:
tool_path = util.parse_xml( elem.get( 'shed_conf_file' ) ).getroot().get( 'tool_path', tool_path )
except Exception, e:
log.error( 'Error determining tool_path for Data Manager during testing: %s', e )
else:
tool_shed_repository_id = self.data_managers.app.security.encode_id( tool_shed_repository.id )
# use shed_conf_file to determine tool_path
shed_conf_file = elem.get( "shed_conf_file", None )
if shed_conf_file:
shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename( shed_conf_file, None )
if shed_conf:
tool_path = shed_conf.get( "tool_path", tool_path )
def load_all( self, config_filename ):
self.visible_external_service_types = []
tree = util.parse_xml( config_filename )
root = tree.getroot()
for elem in root:
try:
if elem.tag == 'external_service_type':
file_path = elem.get( "file" )
visible = util.string_as_bool( elem.get( "visible" ) )
external_service_type = self.load_external_service_type( os.path.join( self.root_dir, file_path ), visible )
self.all_external_service_types[ external_service_type.id ] = external_service_type
log.debug( "Loaded external_service_type: %s %s" % ( external_service_type.name, external_service_type.config_version ) )
if visible:
self.visible_external_service_types.append( external_service_type.id )
except:
log.exception( "error reading external_service_type from path: %s" % file_path )
def set_meta(self, dataset, **kwd):
tree = util.parse_xml(dataset.file_name)
root = tree.getroot()
dataset.metadata.format = root.get('Format')
dataset.metadata.tile_size = root.get('TileSize')
# DeepZoom image files can include
# xml namespace attributes.
if root.tag.find('Collection') >= 0:
dataset.metadata.max_level = root.get('MaxLevel')
dataset.metadata.quality = root.get('Quality')
elif root.tag.find('Image') >= 0:
dataset.metadata.overlap = root.get('Overlap')
for elem in root:
if elem.tag.find('Size') >= 0:
dataset.metadata.width = elem.get('Width')
dataset.metadata.height = elem.get('Height')
def __init__( self, root_dir=None, config=None ):
self.tool_sheds = odict()
if root_dir and config:
# Parse tool_sheds_conf.xml
tree = parse_xml( config )
root = tree.getroot()
log.debug( 'Loading references to tool sheds from %s' % config )
for elem in root.findall( 'tool_shed' ):
try:
name = elem.get( 'name', None )
url = elem.get( 'url', None )
if name and url:
self.tool_sheds[ name ] = url
log.debug( 'Loaded reference to tool shed: %s' % name )
except Exception, e:
log.warning( 'Error loading reference to tool shed "%s", problem: %s' % ( name, str( e ) ) )
def load_all( self, config_filename ):
self.visible_external_service_types = []
tree = util.parse_xml( config_filename )
root = tree.getroot()
for elem in root:
try:
if elem.tag == 'external_service_type':
file_path = elem.get( "file" )
visible = util.string_as_bool( elem.get( "visible" ) )
external_service_type = self.load_external_service_type( os.path.join( self.root_dir, file_path ), visible )
self.all_external_service_types[ external_service_type.id ] = external_service_type
log.debug( "Loaded external_service_type: %s %s" % ( external_service_type.name, external_service_type.config_version ) )
if visible:
self.visible_external_service_types.append( external_service_type.id )
except:
log.exception( "error reading external_service_type from path: %s" % file_path )
def from_xml(clazz, config, config_xml, fsmon=False):
legacy = False
if config_xml is None:
distributed_config = config.distributed_object_store_config_file
assert distributed_config is not None, \
"distributed object store ('object_store = distributed') " \
"requires a config file, please set one in " \
"'distributed_object_store_config_file')"
log.debug('Loading backends for distributed object store from %s', distributed_config)
config_xml = parse_xml(distributed_config).getroot()
legacy = True
else:
log.debug('Loading backends for distributed object store from %s', config_xml.get('id'))
config_dict = clazz.parse_xml(config_xml, legacy=legacy)
return clazz(config, config_dict, fsmon=fsmon)
def load_tool(path):
"""
Loads tool from file system and preprocesses tool macros.
"""
tree = parse_xml(path)
root = tree.getroot()
_import_macros(root, path)
# Expand xml macros
macro_dict = _macros_of_type(root, 'xml', lambda el: list(el.getchildren()))
_expand_macros([root], macro_dict)
# Expand tokens
macro_dict = _macros_of_type(root, 'token', lambda el: el.text)
_expand_tokens([root], macro_dict)
return tree
def load_from_xml(self, xml_filename, store_tool_path=True):
try:
tree = util.parse_xml(xml_filename)
except Exception as e:
log.error('There was an error parsing your Data Manager config file "%s": %s' % (xml_filename, e))
return # we are not able to load any data managers
root = tree.getroot()
if root.tag != 'data_managers':
log.error('A data managers configuration must have a "data_managers" tag as the root. "%s" is present' % (root.tag))
return
if store_tool_path:
tool_path = root.get('tool_path', None)
if tool_path is None:
tool_path = self.app.config.tool_path
if not tool_path:
tool_path = '.'
self.tool_path = tool_path
for data_manager_elem in root.findall('data_manager'):
self.load_manager_from_elem(data_manager_elem, tool_path=self.tool_path)
def parse_additional_template_paths( self, config_filepath, base_directory ):
"""
Parse an XML config file at `config_filepath` for template paths
(relative to `base_directory`) to add to each plugin's template lookup.
Allows having a set of common templates for import/inheritance in
plugin templates.
:type config_filepath: string
:param config_filepath: filesystem path to the config file
:type base_directory: string
:param base_directory: path prefixed to new, relative template paths
"""
additional_paths = []
xml_tree = util.parse_xml( config_filepath )
paths_list = xml_tree.getroot()
for rel_path_elem in paths_list.findall( 'path' ):
if rel_path_elem.text is not None:
additional_paths.append( os.path.join( base_directory, rel_path_elem.text ) )
return additional_paths
def add_new_entries_from_config_file( self, config_filename, tool_data_path, tool_data_table_config_path, persist=False ):
"""
This method is called when a tool shed repository that includes a tool_data_table_conf.xml.sample file is being
installed into a local galaxy instance. We have 2 cases to handle, files whose root tag is <tables>, for example:
<tables>
<!-- Location of Tmap files -->
<table name="tmap_indexes" comment_char="#">
<columns>value, dbkey, name, path</columns>
<file path="tool-data/tmap_index.loc" />
</table>
</tables>
and files whose root tag is <table>, for example:
<!-- Location of Tmap files -->
<table name="tmap_indexes" comment_char="#">
<columns>value, dbkey, name, path</columns>
<file path="tool-data/tmap_index.loc" />
</table>
"""
tree = util.parse_xml( config_filename )
root = tree.getroot()
# Make a copy of the current list of data_table_elem_names so we can persist later if changes to the config file are necessary.
original_data_table_elem_names = [ name for name in self.data_table_elem_names ]
if root.tag == 'tables':
table_elems = self.load_from_config_file( config_filename, tool_data_path )
else:
table_elems = []
type = root.get( 'type', 'tabular' )
assert type in tool_data_table_types, "Unknown data table type '%s'" % type
table_elems.append( root )
table_elem_name = root.get( 'name', None )
if table_elem_name and table_elem_name not in self.data_table_elem_names:
self.data_table_elem_names.append( table_elem_name )
self.data_table_elems.append( root )
table = tool_data_table_types[ type ]( root, tool_data_path )
if table.name not in self.data_tables:
self.data_tables[ table.name ] = table
log.debug( "Added new tool data table '%s'", table.name )
if persist and self.data_table_elem_names != original_data_table_elem_names:
# Persist Galaxy's version of the changed tool_data_table_conf.xml file.
self.to_xml_file( tool_data_table_config_path )
return table_elems
def to_xml_file( self, shed_tool_data_table_config, new_elems=None, remove_elems=None ):
"""
Write the current in-memory version of the shed_tool_data_table_conf.xml file to disk.
remove_elems are removed before new_elems are added.
"""
if not ( new_elems or remove_elems ):
log.debug( 'ToolDataTableManager.to_xml_file called without any elements to add or remove.' )
return #no changes provided, no need to persist any changes
if not new_elems:
new_elems = []
if not remove_elems:
remove_elems = []
full_path = os.path.abspath( shed_tool_data_table_config )
#FIXME: we should lock changing this file by other threads / head nodes
try:
tree = util.parse_xml( full_path )
root = tree.getroot()
out_elems = [ elem for elem in root ]
except Exception, e:
out_elems = []
log.debug( 'Could not parse existing tool data table config, assume no existing elements: %s', e )
def load_from_xml(self, xml_filename, store_tool_path=True):
try:
tree = util.parse_xml(xml_filename)
except Exception as e:
log.error('There was an error parsing your Data Manager config file "%s": %s' % (xml_filename, e))
return # we are not able to load any data managers
root = tree.getroot()
if root.tag != 'data_managers':
log.error('A data managers configuration must have a "data_managers" tag as the root. "%s" is present' % (root.tag))
return
if store_tool_path:
tool_path = root.get('tool_path', None)
if tool_path is None:
tool_path = self.app.config.tool_path
if not tool_path:
tool_path = '.'
self.tool_path = tool_path
for data_manager_elem in root.findall('data_manager'):
if not self.load_manager_from_elem(data_manager_elem, tool_path=self.tool_path):
# Wasn't able to load manager, could happen when galaxy is managed by planemo.
# Fall back to loading relative to the data_manager_conf.xml file
tool_path = os.path.dirname(xml_filename)
self.load_manager_from_elem(data_manager_elem, tool_path=tool_path)
def from_file(cls, filename, app):
return cls.from_elem(parse_xml(filename).getroot(), app, filename=filename)
def load_from_element(self, elem, tool_path):
assert elem.tag == 'data_manager', 'A data manager configuration must have a "data_manager" tag as the root. "%s" is present' % (elem.tag)
self.declared_id = elem.get('id', None)
self.guid = elem.get('guid', None)
path = elem.get('tool_file', None)
self.version = elem.get('version', self.version)
tool_shed_repository_id = None
tool_guid = None
if path is None:
tool_elem = elem.find('tool')
assert tool_elem is not None, "Error loading tool for data manager. Make sure that a tool_file attribute or a tool tag set has been defined:\n%s" % (util.xml_to_string(elem))
path = tool_elem.get("file", None)
tool_guid = tool_elem.get("guid", None)
# need to determine repository info so that dependencies will work correctly
if hasattr(self.data_managers.app, 'tool_cache') and tool_guid in self.data_managers.app.tool_cache._tool_paths_by_id:
path = self.data_managers.app.tool_cache._tool_paths_by_id[tool_guid]
tool = self.data_managers.app.tool_cache.get_tool(path)
tool_shed_repository = tool.tool_shed_repository
self.tool_shed_repository_info_dict = dict(tool_shed=tool_shed_repository.tool_shed,
name=tool_shed_repository.name,
owner=tool_shed_repository.owner,
installed_changeset_revision=tool_shed_repository.installed_changeset_revision)
tool_shed_repository_id = self.data_managers.app.security.encode_id(tool_shed_repository.id)
tool_path = ""
else:
tool_shed_url = tool_elem.find('tool_shed').text
# Handle protocol changes.
tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(self.data_managers.app, tool_shed_url)
# The protocol is not stored in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed_url)
repository_name = tool_elem.find('repository_name').text
repository_owner = tool_elem.find('repository_owner').text
installed_changeset_revision = tool_elem.find('installed_changeset_revision').text
self.tool_shed_repository_info_dict = dict(tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision)
tool_shed_repository = \
repository_util.get_installed_repository(self.data_managers.app,
tool_shed=tool_shed,
name=repository_name,
owner=repository_owner,
installed_changeset_revision=installed_changeset_revision)
if tool_shed_repository is None:
log.warning('Could not determine tool shed repository from database. This should only ever happen when running tests.')
# we'll set tool_path manually here from shed_conf_file
tool_shed_repository_id = None
try:
tool_path = util.parse_xml(elem.get('shed_conf_file')).getroot().get('tool_path', tool_path)
except Exception as e:
log.error('Error determining tool_path for Data Manager during testing: %s', e)
else:
tool_shed_repository_id = self.data_managers.app.security.encode_id(tool_shed_repository.id)
# use shed_conf_file to determine tool_path
shed_conf_file = elem.get("shed_conf_file", None)
if shed_conf_file:
shed_conf = self.data_managers.app.toolbox.get_shed_config_dict_by_filename(shed_conf_file, None)
if shed_conf:
tool_path = shed_conf.get("tool_path", tool_path)
assert path is not None, "A tool file path could not be determined:\n%s" % (util.xml_to_string(elem))
self.load_tool(os.path.join(tool_path, path),
guid=tool_guid,
data_manager_id=self.id,
tool_shed_repository_id=tool_shed_repository_id)
self.name = elem.get('name', self.tool.name)
self.description = elem.get('description', self.tool.description)
self.undeclared_tables = util.asbool(elem.get('undeclared_tables', self.undeclared_tables))
for data_table_elem in elem.findall('data_table'):
data_table_name = data_table_elem.get("name")
assert data_table_name is not None, "A name is required for a data table entry"
if data_table_name not in self.data_tables:
self.data_tables[data_table_name] = odict()
output_elem = data_table_elem.find('output')
if output_elem is not None:
for column_elem in output_elem.findall('column'):
column_name = column_elem.get('name', None)
assert column_name is not None, "Name is required for column entry"
data_table_coumn_name = column_elem.get('data_table_name', column_name)
self.data_tables[data_table_name][data_table_coumn_name] = column_name
output_ref = column_elem.get('output_ref', None)
if output_ref is not None:
if data_table_name not in self.output_ref_by_data_table:
self.output_ref_by_data_table[data_table_name] = {}
self.output_ref_by_data_table[data_table_name][data_table_coumn_name] = output_ref
value_translation_elems = column_elem.findall('value_translation')
if value_translation_elems is not None:
for value_translation_elem in value_translation_elems:
value_translation = value_translation_elem.text
if value_translation is not None:
value_translation_type = value_translation_elem.get('type', DEFAULT_VALUE_TRANSLATION_TYPE)
if data_table_name not in self.value_translation_by_data_table_column:
self.value_translation_by_data_table_column[data_table_name] = {}
if data_table_coumn_name not in self.value_translation_by_data_table_column[data_table_name]:
self.value_translation_by_data_table_column[data_table_name][data_table_coumn_name] = []
if value_translation_type == 'function':
if value_translation in VALUE_TRANSLATION_FUNCTIONS:
value_translation = VALUE_TRANSLATION_FUNCTIONS[value_translation]
else:
raise ValueError("Unsupported value translation function: '%s'" % (value_translation))
else:
assert value_translation_type == DEFAULT_VALUE_TRANSLATION_TYPE, ValueError("Unsupported value translation type: '%s'" % (value_translation_type))
self.value_translation_by_data_table_column[data_table_name][data_table_coumn_name].append(value_translation)
for move_elem in column_elem.findall('move'):
move_type = move_elem.get('type', 'directory')
relativize_symlinks = move_elem.get('relativize_symlinks', False) # TODO: should we instead always relativize links?
source_elem = move_elem.find('source')
if source_elem is None:
source_base = None
source_value = ''
else:
source_base = source_elem.get('base', None)
source_value = source_elem.text
target_elem = move_elem.find('target')
if target_elem is None:
target_base = None
target_value = ''
else:
target_base = target_elem.get('base', None)
target_value = target_elem.text
if data_table_name not in self.move_by_data_table_column:
self.move_by_data_table_column[data_table_name] = {}
self.move_by_data_table_column[data_table_name][data_table_coumn_name] = \
dict(type=move_type,
source_base=source_base,
source_value=source_value,
target_base=target_base,
target_value=target_value,
relativize_symlinks=relativize_symlinks)
def __init__( self, app, latest_migration_script_number, tool_shed_install_config, migrated_tools_config, install_dependencies ):
"""
Check tool settings in tool_shed_install_config and install all repositories that are not already installed. The tool
panel configuration file is the received migrated_tools_config, which is the reserved file named migrated_tools_conf.xml.
"""
self.app = app
self.toolbox = self.app.toolbox
self.migrated_tools_config = migrated_tools_config
# If install_dependencies is True but tool_dependency_dir is not set, do not attempt to install but print informative error message.
if install_dependencies and app.config.tool_dependency_dir is None:
message = 'You are attempting to install tool dependencies but do not have a value for "tool_dependency_dir" set in your universe_wsgi.ini '
message += 'file. Set this location value to the path where you want tool dependencies installed and rerun the migration script.'
raise Exception( message )
# Get the local non-shed related tool panel configs (there can be more than one, and the default name is tool_conf.xml).
self.proprietary_tool_confs = self.non_shed_tool_panel_configs
self.proprietary_tool_panel_elems = self.get_proprietary_tool_panel_elems( latest_migration_script_number )
# Set the location where the repositories will be installed by retrieving the tool_path setting from migrated_tools_config.
tree = util.parse_xml( migrated_tools_config )
root = tree.getroot()
self.tool_path = root.get( 'tool_path' )
print "Repositories will be installed into configured tool_path location ", str( self.tool_path )
# Parse tool_shed_install_config to check each of the tools.
self.tool_shed_install_config = tool_shed_install_config
tree = util.parse_xml( tool_shed_install_config )
root = tree.getroot()
self.tool_shed = suc.clean_tool_shed_url( root.get( 'name' ) )
self.repository_owner = common_util.REPOSITORY_OWNER
index, self.shed_config_dict = suc.get_shed_tool_conf_dict( app, self.migrated_tools_config )
# Since tool migration scripts can be executed any number of times, we need to make sure the appropriate tools are defined in
# tool_conf.xml. If no tools associated with the migration stage are defined, no repositories will be installed on disk.
# The default behavior is that the tool shed is down.
tool_shed_accessible = False
tool_panel_configs = common_util.get_non_shed_tool_panel_configs( app )
if tool_panel_configs:
# The missing_tool_configs_dict contents are something like:
# {'emboss_antigenic.xml': [('emboss', '5.0.0', 'package', '\nreadme blah blah blah\n')]}
tool_shed_accessible, missing_tool_configs_dict = common_util.check_for_missing_tools( app, tool_panel_configs, latest_migration_script_number )
else:
# It doesn't matter if the tool shed is accessible since there are no migrated tools defined in the local Galaxy instance, but
# we have to set the value of tool_shed_accessible to True so that the value of migrate_tools.version can be correctly set in
# the database.
tool_shed_accessible = True
missing_tool_configs_dict = odict()
if tool_shed_accessible:
if len( self.proprietary_tool_confs ) == 1:
plural = ''
file_names = self.proprietary_tool_confs[ 0 ]
else:
plural = 's'
file_names = ', '.join( self.proprietary_tool_confs )
if missing_tool_configs_dict:
for repository_elem in root:
self.install_repository( repository_elem, install_dependencies )
else:
message = "\nNo tools associated with migration stage %s are defined in your " % str( latest_migration_script_number )
message += "file%s named %s,\nso no repositories will be installed on disk.\n" % ( plural, file_names )
print message
else:
message = "\nThe main Galaxy tool shed is not currently available, so skipped migration stage %s.\n" % str( latest_migration_script_number )
message += "Try again later.\n"
print message