def data_manager_config_elems_to_xml_file(self, config_elems,
config_filename):
"""
Persist the current in-memory list of config_elems to a file named by the value
of config_filename.
"""
data_managers_path = self.data_managers_path
lock = threading.Lock()
lock.acquire(True)
try:
fh = open(config_filename, 'wb')
if data_managers_path is not None:
fh.write(
'<?xml version="1.0"?>\n<data_managers tool_path="%s">\n '
% data_managers_path)
else:
fh.write('<?xml version="1.0"?>\n<data_managers>\n ')
for elem in config_elems:
fh.write(xml_util.xml_to_string(elem))
fh.write('</data_managers>\n')
fh.close()
except Exception:
log.exception(
"Exception in DataManagerHandler.data_manager_config_elems_to_xml_file"
)
finally:
lock.release()
def data_manager_config_elems_to_xml_file( app, config_elems, config_filename ):#, shed_tool_conf_filename ):
# Persist the current in-memory list of config_elems to a file named by the value of config_filename.
fh = open( config_filename, 'wb' )
fh.write( '<?xml version="1.0"?>\n<data_managers>\n' )#% ( shed_tool_conf_filename ))
for elem in config_elems:
fh.write( xml_util.xml_to_string( elem ) )
fh.write( '</data_managers>\n' )
fh.close()
def data_manager_config_elems_to_xml_file( app, config_elems, config_filename ):#, shed_tool_conf_filename ):
# Persist the current in-memory list of config_elems to a file named by the value of config_filename.
fh = open( config_filename, 'wb' )
fh.write( '<?xml version="1.0"?>\n<data_managers>\n' )#% ( shed_tool_conf_filename ))
for elem in config_elems:
fh.write( xml_util.xml_to_string( elem ) )
fh.write( '</data_managers>\n' )
fh.close()
def data_manager_config_elems_to_xml_file( self, config_elems, config_filename ):
"""
Persist the current in-memory list of config_elems to a file named by the value
of config_filename.
"""
lock = threading.Lock()
lock.acquire( True )
try:
fh = open( config_filename, 'wb' )
fh.write( '<?xml version="1.0"?>\n<data_managers>\n' )
for elem in config_elems:
fh.write( xml_util.xml_to_string( elem ) )
fh.write( '</data_managers>\n' )
fh.close()
except Exception, e:
log.exception( "Exception in DataManagerHandler.data_manager_config_elems_to_xml_file: %s" % str( e ) )
def config_elems_to_xml_file( self, config_elems, config_filename, tool_path ):
"""
Persist the current in-memory list of config_elems to a file named by the
value of config_filename.
"""
lock = threading.Lock()
lock.acquire( True )
try:
fh = open( config_filename, 'wb' )
fh.write( '<?xml version="1.0"?>\n<toolbox tool_path="%s">\n' % str( tool_path ) )
for elem in config_elems:
fh.write( xml_util.xml_to_string( elem, use_indent=True ) )
fh.write( '</toolbox>\n' )
fh.close()
except Exception, e:
log.exception( "Exception in ToolPanelManager.config_elems_to_xml_file: %s" % str( e ) )
def config_elems_to_xml_file(self, config_elems, config_filename,
tool_path):
"""
Persist the current in-memory list of config_elems to a file named by the
value of config_filename.
"""
lock = threading.Lock()
lock.acquire(True)
try:
fh = open(config_filename, 'wb')
fh.write('<?xml version="1.0"?>\n<toolbox tool_path="%s">\n' %
str(tool_path))
for elem in config_elems:
fh.write(xml_util.xml_to_string(elem, use_indent=True))
fh.write('</toolbox>\n')
fh.close()
except Exception, e:
log.exception(
"Exception in ToolPanelManager.config_elems_to_xml_file: %s" %
str(e))
def data_manager_config_elems_to_xml_file( self, config_elems, config_filename ):
"""
Persist the current in-memory list of config_elems to a file named by the value
of config_filename.
"""
data_managers_path = self.data_managers_path
lock = threading.Lock()
lock.acquire( True )
try:
fh = open( config_filename, 'wb' )
if data_managers_path is not None:
fh.write( '<?xml version="1.0"?>\n<data_managers tool_path="%s">\n ' % data_managers_path )
else:
fh.write( '<?xml version="1.0"?>\n<data_managers>\n ' )
for elem in config_elems:
fh.write( xml_util.xml_to_string( elem ) )
fh.write( '</data_managers>\n' )
fh.close()
except Exception:
log.exception( "Exception in DataManagerHandler.data_manager_config_elems_to_xml_file" )
finally:
lock.release()
def alter_config_and_load_prorietary_datatypes(self,
datatypes_config,
relative_install_dir,
deactivate=False,
override=True):
"""
Parse a custom datatypes config (a datatypes_conf.xml file included in an installed
tool shed repository) and add information to appropriate element attributes that will
enable custom datatype class modules, datatypes converters and display applications
to be discovered and properly imported by the datatypes registry. The value of override
will be False when a tool shed repository is being installed. Since installation is
occurring after the datatypes registry has been initialized, the registry's contents
cannot be overridden by conflicting data types.
"""
tree, error_message = xml_util.parse_xml(datatypes_config)
if tree is None:
return None, None
datatypes_config_root = tree.getroot()
registration = datatypes_config_root.find('registration')
if registration is None:
# We have valid XML, but not a valid custom datatypes definition.
return None, None
sniffers = datatypes_config_root.find('sniffers')
converter_path, display_path = self.get_converter_and_display_paths(
registration, relative_install_dir)
if converter_path:
# Path to datatype converters
registration.attrib['proprietary_converter_path'] = converter_path
if display_path:
# Path to datatype display applications
registration.attrib['proprietary_display_path'] = display_path
relative_path_to_datatype_file_name = None
datatype_files = datatypes_config_root.find('datatype_files')
datatype_class_modules = []
if datatype_files is not None:
# The <datatype_files> tag set contains any number of <datatype_file> tags.
# <datatype_files>
# <datatype_file name="gmap.py"/>
# <datatype_file name="metagenomics.py"/>
# </datatype_files>
# We'll add attributes to the datatype tag sets so that the modules can be properly imported
# by the datatypes registry.
for elem in datatype_files.findall('datatype_file'):
datatype_file_name = elem.get('name', None)
if datatype_file_name:
# Find the file in the installed repository.
for root, dirs, files in os.walk(relative_install_dir):
if root.find('.hg') < 0:
for name in files:
if name == datatype_file_name:
datatype_class_modules.append(
os.path.join(root, name))
break
break
if datatype_class_modules:
for relative_path_to_datatype_file_name in datatype_class_modules:
datatype_file_name_path, datatype_file_name = os.path.split(
relative_path_to_datatype_file_name)
for elem in registration.findall('datatype'):
# Handle 'type' attribute which should be something like one of the following:
# type="gmap:GmapDB"
# type="galaxy.datatypes.gmap:GmapDB"
dtype = elem.get('type', None)
if dtype:
fields = dtype.split(':')
proprietary_datatype_module = fields[0]
if proprietary_datatype_module.find('.') >= 0:
# Handle the case where datatype_module is "galaxy.datatypes.gmap".
proprietary_datatype_module = proprietary_datatype_module.split(
'.')[-1]
# The value of proprietary_path must be an absolute path due to job_working_directory.
elem.attrib['proprietary_path'] = os.path.abspath(
datatype_file_name_path)
elem.attrib[
'proprietary_datatype_module'] = proprietary_datatype_module
# Temporarily persist the custom datatypes configuration file so it can be loaded into the
# datatypes registry.
fd, proprietary_datatypes_config = tempfile.mkstemp(
prefix="tmp-toolshed-acalpd")
os.write(fd, '<?xml version="1.0"?>\n')
os.write(fd, '<datatypes>\n')
os.write(fd, '%s' % xml_util.xml_to_string(registration))
if sniffers is not None:
os.write(fd, '%s' % xml_util.xml_to_string(sniffers))
os.write(fd, '</datatypes>\n')
os.close(fd)
os.chmod(proprietary_datatypes_config, 0644)
# Load custom datatypes
self.app.datatypes_registry.load_datatypes(
root_dir=self.app.config.root,
config=proprietary_datatypes_config,
deactivate=deactivate,
override=override)
if deactivate:
# Reload the upload tool to eliminate deactivated datatype extensions from the file_type
# select list.
tool_util.reload_upload_tools(self.app)
else:
self.append_to_datatypes_registry_upload_file_formats(registration)
tool_util.reload_upload_tools(self.app)
if datatype_files is not None:
try:
os.unlink(proprietary_datatypes_config)
except:
pass
return converter_path, display_path
def install_data_managers( self, shed_data_manager_conf_filename, metadata_dict, shed_config_dict,
relative_install_dir, repository, repository_tools_tups ):
rval = []
if 'data_manager' in metadata_dict:
tpm = tool_panel_manager.ToolPanelManager( self.app )
repository_tools_by_guid = {}
for tool_tup in repository_tools_tups:
repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] )
# Load existing data managers.
tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename )
if tree is None:
return rval
config_elems = [ elem for elem in tree.getroot() ]
repo_data_manager_conf_filename = metadata_dict['data_manager'].get( 'config_filename', None )
if repo_data_manager_conf_filename is None:
log.debug( "No data_manager_conf.xml file has been defined." )
return rval
data_manager_config_has_changes = False
relative_repo_data_manager_dir = os.path.join( shed_config_dict.get( 'tool_path', '' ), relative_install_dir )
repo_data_manager_conf_filename = os.path.join( relative_repo_data_manager_dir, repo_data_manager_conf_filename )
tree, error_message = xml_util.parse_xml( repo_data_manager_conf_filename )
if tree is None:
return rval
root = tree.getroot()
for elem in root:
if elem.tag == 'data_manager':
data_manager_id = elem.get( 'id', None )
if data_manager_id is None:
log.error( "A data manager was defined that does not have an id and will not be installed:\n%s" %
xml_util.xml_to_string( elem ) )
continue
data_manager_dict = metadata_dict['data_manager'].get( 'data_managers', {} ).get( data_manager_id, None )
if data_manager_dict is None:
log.error( "Data manager metadata is not defined properly for '%s'." % ( data_manager_id ) )
continue
guid = data_manager_dict.get( 'guid', None )
if guid is None:
log.error( "Data manager guid '%s' is not set in metadata for '%s'." % ( guid, data_manager_id ) )
continue
elem.set( 'guid', guid )
tool_guid = data_manager_dict.get( 'tool_guid', None )
if tool_guid is None:
log.error( "Data manager tool guid '%s' is not set in metadata for '%s'." % ( tool_guid, data_manager_id ) )
continue
tool_dict = repository_tools_by_guid.get( tool_guid, None )
if tool_dict is None:
log.error( "Data manager tool guid '%s' could not be found for '%s'. Perhaps the tool is invalid?" %
( tool_guid, data_manager_id ) )
continue
tool = tool_dict.get( 'tool', None )
if tool is None:
log.error( "Data manager tool with guid '%s' could not be found for '%s'. Perhaps the tool is invalid?" %
( tool_guid, data_manager_id ) )
continue
tool_config_filename = tool_dict.get( 'tool_config_filename', None )
if tool_config_filename is None:
log.error( "Data manager metadata is missing 'tool_config_file' for '%s'." % ( data_manager_id ) )
continue
elem.set( 'shed_conf_file', shed_config_dict['config_filename'] )
if elem.get( 'tool_file', None ) is not None:
del elem.attrib[ 'tool_file' ] # remove old tool_file info
tool_elem = tpm.generate_tool_elem( repository.tool_shed,
repository.name,
repository.installed_changeset_revision,
repository.owner,
tool_config_filename,
tool,
None )
elem.insert( 0, tool_elem )
data_manager = \
self.app.data_managers.load_manager_from_elem( elem,
tool_path=shed_config_dict.get( 'tool_path', '' ),
replace_existing=True )
if data_manager:
rval.append( data_manager )
else:
log.warning( "Encountered unexpected element '%s':\n%s" % ( elem.tag, xml_util.xml_to_string( elem ) ) )
config_elems.append( elem )
data_manager_config_has_changes = True
# Persist the altered shed_data_manager_config file.
if data_manager_config_has_changes:
self.data_manager_config_elems_to_xml_file( config_elems, shed_data_manager_conf_filename )
return rval
def install_data_managers( self, shed_data_manager_conf_filename, metadata_dict, shed_config_dict,
relative_install_dir, repository, repository_tools_tups ):
rval = []
if 'data_manager' in metadata_dict:
tpm = tool_panel_manager.ToolPanelManager( self.app )
repository_tools_by_guid = {}
for tool_tup in repository_tools_tups:
repository_tools_by_guid[ tool_tup[ 1 ] ] = dict( tool_config_filename=tool_tup[ 0 ], tool=tool_tup[ 2 ] )
# Load existing data managers.
tree, error_message = xml_util.parse_xml( shed_data_manager_conf_filename )
if tree is None:
return rval
config_elems = [ elem for elem in tree.getroot() ]
repo_data_manager_conf_filename = metadata_dict['data_manager'].get( 'config_filename', None )
if repo_data_manager_conf_filename is None:
log.debug( "No data_manager_conf.xml file has been defined." )
return rval
data_manager_config_has_changes = False
relative_repo_data_manager_dir = os.path.join( shed_config_dict.get( 'tool_path', '' ), relative_install_dir )
repo_data_manager_conf_filename = os.path.join( relative_repo_data_manager_dir, repo_data_manager_conf_filename )
tree, error_message = xml_util.parse_xml( repo_data_manager_conf_filename )
if tree is None:
return rval
root = tree.getroot()
for elem in root:
if elem.tag == 'data_manager':
data_manager_id = elem.get( 'id', None )
if data_manager_id is None:
log.error( "A data manager was defined that does not have an id and will not be installed:\n%s" % \
xml_util.xml_to_string( elem ) )
continue
data_manager_dict = metadata_dict['data_manager'].get( 'data_managers', {} ).get( data_manager_id, None )
if data_manager_dict is None:
log.error( "Data manager metadata is not defined properly for '%s'." % ( data_manager_id ) )
continue
guid = data_manager_dict.get( 'guid', None )
if guid is None:
log.error( "Data manager guid '%s' is not set in metadata for '%s'." % ( guid, data_manager_id ) )
continue
elem.set( 'guid', guid )
tool_guid = data_manager_dict.get( 'tool_guid', None )
if tool_guid is None:
log.error( "Data manager tool guid '%s' is not set in metadata for '%s'." % ( tool_guid, data_manager_id ) )
continue
tool_dict = repository_tools_by_guid.get( tool_guid, None )
if tool_dict is None:
log.error( "Data manager tool guid '%s' could not be found for '%s'. Perhaps the tool is invalid?" % \
( tool_guid, data_manager_id ) )
continue
tool = tool_dict.get( 'tool', None )
if tool is None:
log.error( "Data manager tool with guid '%s' could not be found for '%s'. Perhaps the tool is invalid?" % \
( tool_guid, data_manager_id ) )
continue
tool_config_filename = tool_dict.get( 'tool_config_filename', None )
if tool_config_filename is None:
log.error( "Data manager metadata is missing 'tool_config_file' for '%s'." % ( data_manager_id ) )
continue
elem.set( 'shed_conf_file', shed_config_dict['config_filename'] )
if elem.get( 'tool_file', None ) is not None:
del elem.attrib[ 'tool_file' ] #remove old tool_file info
tool_elem = tpm.generate_tool_elem( repository.tool_shed,
repository.name,
repository.installed_changeset_revision,
repository.owner,
tool_config_filename,
tool,
None )
elem.insert( 0, tool_elem )
data_manager = \
self.app.data_managers.load_manager_from_elem( elem,
tool_path=shed_config_dict.get( 'tool_path', '' ),
replace_existing=True )
if data_manager:
rval.append( data_manager )
else:
log.warning( "Encountered unexpected element '%s':\n%s" % ( elem.tag, xml_util.xml_to_string( elem ) ) )
config_elems.append( elem )
data_manager_config_has_changes = True
# Persist the altered shed_data_manager_config file.
if data_manager_config_has_changes:
self.data_manager_config_elems_to_xml_file( config_elems, shed_data_manager_conf_filename )
return rval
def alter_config_and_load_prorietary_datatypes( self, datatypes_config, relative_install_dir,
deactivate=False, override=True ):
"""
Parse a custom datatypes config (a datatypes_conf.xml file included in an installed
tool shed repository) and add information to appropriate element attributes that will
enable custom datatype class modules, datatypes converters and display applications
to be discovered and properly imported by the datatypes registry. The value of override
will be False when a tool shed repository is being installed. Since installation is
occurring after the datatypes registry has been initialized, the registry's contents
cannot be overridden by conflicting data types.
"""
tree, error_message = xml_util.parse_xml( datatypes_config )
if tree is None:
return None, None
datatypes_config_root = tree.getroot()
registration = datatypes_config_root.find( 'registration' )
if registration is None:
# We have valid XML, but not a valid custom datatypes definition.
return None, None
sniffers = datatypes_config_root.find( 'sniffers' )
converter_path, display_path = self.get_converter_and_display_paths( registration,
relative_install_dir )
if converter_path:
# Path to datatype converters
registration.attrib[ 'proprietary_converter_path' ] = converter_path
if display_path:
# Path to datatype display applications
registration.attrib[ 'proprietary_display_path' ] = display_path
relative_path_to_datatype_file_name = None
datatype_files = datatypes_config_root.find( 'datatype_files' )
datatype_class_modules = []
if datatype_files is not None:
# The <datatype_files> tag set contains any number of <datatype_file> tags.
# <datatype_files>
# <datatype_file name="gmap.py"/>
# <datatype_file name="metagenomics.py"/>
# </datatype_files>
# We'll add attributes to the datatype tag sets so that the modules can be properly imported
# by the datatypes registry.
for elem in datatype_files.findall( 'datatype_file' ):
datatype_file_name = elem.get( 'name', None )
if datatype_file_name:
# Find the file in the installed repository.
for root, dirs, files in os.walk( relative_install_dir ):
if root.find( '.hg' ) < 0:
for name in files:
if name == datatype_file_name:
datatype_class_modules.append( os.path.join( root, name ) )
break
break
if datatype_class_modules:
for relative_path_to_datatype_file_name in datatype_class_modules:
datatype_file_name_path, datatype_file_name = os.path.split( relative_path_to_datatype_file_name )
for elem in registration.findall( 'datatype' ):
# Handle 'type' attribute which should be something like one of the following:
# type="gmap:GmapDB"
# type="galaxy.datatypes.gmap:GmapDB"
dtype = elem.get( 'type', None )
if dtype:
fields = dtype.split( ':' )
proprietary_datatype_module = fields[ 0 ]
if proprietary_datatype_module.find( '.' ) >= 0:
# Handle the case where datatype_module is "galaxy.datatypes.gmap".
proprietary_datatype_module = proprietary_datatype_module.split( '.' )[ -1 ]
# The value of proprietary_path must be an absolute path due to job_working_directory.
elem.attrib[ 'proprietary_path' ] = os.path.abspath( datatype_file_name_path )
elem.attrib[ 'proprietary_datatype_module' ] = proprietary_datatype_module
# Temporarily persist the custom datatypes configuration file so it can be loaded into the
# datatypes registry.
fd, proprietary_datatypes_config = tempfile.mkstemp( prefix="tmp-toolshed-acalpd" )
os.write( fd, '<?xml version="1.0"?>\n' )
os.write( fd, '<datatypes>\n' )
os.write( fd, '%s' % xml_util.xml_to_string( registration ) )
if sniffers is not None:
os.write( fd, '%s' % xml_util.xml_to_string( sniffers ) )
os.write( fd, '</datatypes>\n' )
os.close( fd )
os.chmod( proprietary_datatypes_config, 0o644 )
# Load custom datatypes
self.app.datatypes_registry.load_datatypes( root_dir=self.app.config.root,
config=proprietary_datatypes_config,
deactivate=deactivate,
override=override )
if deactivate:
# Reload the upload tool to eliminate deactivated datatype extensions from the file_type
# select list.
tool_util.reload_upload_tools( self.app )
else:
self.append_to_datatypes_registry_upload_file_formats( registration )
tool_util.reload_upload_tools( self.app )
if datatype_files is not None:
try:
os.unlink( proprietary_datatypes_config )
except:
pass
return converter_path, display_path
