def install_repository( self, repository_elem, install_dependencies ):
# Install a single repository, loading contained tools into the tool panel.
name = repository_elem.get( 'name' )
description = repository_elem.get( 'description' )
installed_changeset_revision = repository_elem.get( 'changeset_revision' )
# Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision>
relative_clone_dir = os.path.join( self.tool_shed, 'repos', self.repository_owner, name, installed_changeset_revision )
clone_dir = os.path.join( self.tool_path, relative_clone_dir )
if self.__isinstalled( clone_dir ):
print "Skipping automatic install of repository '", name, "' because it has already been installed in location ", clone_dir
else:
tool_shed_url = self.__get_url_from_tool_shed( self.tool_shed )
repository_clone_url = os.path.join( tool_shed_url, 'repos', self.repository_owner, name )
relative_install_dir = os.path.join( relative_clone_dir, name )
install_dir = os.path.join( clone_dir, name )
ctx_rev = suc.get_ctx_rev( tool_shed_url, name, self.repository_owner, installed_changeset_revision )
tool_shed_repository = suc.create_or_update_tool_shed_repository( app=self.app,
name=name,
description=description,
installed_changeset_revision=installed_changeset_revision,
ctx_rev=ctx_rev,
repository_clone_url=repository_clone_url,
metadata_dict={},
status=self.app.model.ToolShedRepository.installation_status.NEW,
current_changeset_revision=None,
owner=self.repository_owner,
dist_to_shed=True )
shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.CLONING )
cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev )
if cloned_ok:
self.handle_repository_contents( tool_shed_repository=tool_shed_repository,
repository_clone_url=repository_clone_url,
relative_install_dir=relative_install_dir,
repository_elem=repository_elem,
install_dependencies=install_dependencies )
self.app.sa_session.refresh( tool_shed_repository )
metadata_dict = tool_shed_repository.metadata
if 'tools' in metadata_dict:
shed_util.update_tool_shed_repository_status( self.app,
tool_shed_repository,
self.app.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS )
# Get the tool_versions from the tool shed for each tool in the installed change set.
url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \
( tool_shed_url, tool_shed_repository.name, self.repository_owner, installed_changeset_revision )
response = urllib2.urlopen( url )
text = response.read()
response.close()
if text:
tool_version_dicts = from_json_string( text )
shed_util.handle_tool_versions( self.app, tool_version_dicts, tool_shed_repository )
else:
# Set the tool versions since they seem to be missing for this repository in the tool shed.
# CRITICAL NOTE: These default settings may not properly handle all parent/child associations.
for tool_dict in metadata_dict[ 'tools' ]:
flush_needed = False
tool_id = tool_dict[ 'guid' ]
old_tool_id = tool_dict[ 'id' ]
tool_version = tool_dict[ 'version' ]
tool_version_using_old_id = shed_util.get_tool_version( self.app, old_tool_id )
tool_version_using_guid = shed_util.get_tool_version( self.app, tool_id )
if not tool_version_using_old_id:
tool_version_using_old_id = self.app.model.ToolVersion( tool_id=old_tool_id,
tool_shed_repository=tool_shed_repository )
self.app.sa_session.add( tool_version_using_old_id )
self.app.sa_session.flush()
if not tool_version_using_guid:
tool_version_using_guid = self.app.model.ToolVersion( tool_id=tool_id,
tool_shed_repository=tool_shed_repository )
self.app.sa_session.add( tool_version_using_guid )
self.app.sa_session.flush()
# Associate the two versions as parent / child.
tool_version_association = shed_util.get_tool_version_association( self.app,
tool_version_using_old_id,
tool_version_using_guid )
if not tool_version_association:
tool_version_association = self.app.model.ToolVersionAssociation( tool_id=tool_version_using_guid.id,
parent_id=tool_version_using_old_id.id )
self.app.sa_session.add( tool_version_association )
self.app.sa_session.flush()
shed_util.update_tool_shed_repository_status( self.app, tool_shed_repository, self.app.model.ToolShedRepository.installation_status.INSTALLED )
def install_repository(self, repository_elem, install_dependencies):
# Install a single repository, loading contained tools into the tool panel.
name = repository_elem.get('name')
description = repository_elem.get('description')
installed_changeset_revision = repository_elem.get(
'changeset_revision')
# Install path is of the form: <tool path>/<tool shed>/repos/<repository owner>/<repository name>/<installed changeset revision>
relative_clone_dir = os.path.join(self.tool_shed, 'repos',
self.repository_owner, name,
installed_changeset_revision)
clone_dir = os.path.join(self.tool_path, relative_clone_dir)
if self.__isinstalled(clone_dir):
print "Skipping automatic install of repository '", name, "' because it has already been installed in location ", clone_dir
else:
tool_shed_url = self.__get_url_from_tool_shed(self.tool_shed)
repository_clone_url = os.path.join(tool_shed_url, 'repos',
self.repository_owner, name)
relative_install_dir = os.path.join(relative_clone_dir, name)
install_dir = os.path.join(clone_dir, name)
ctx_rev = suc.get_ctx_rev(tool_shed_url, name,
self.repository_owner,
installed_changeset_revision)
tool_shed_repository = suc.create_or_update_tool_shed_repository(
app=self.app,
name=name,
description=description,
installed_changeset_revision=installed_changeset_revision,
ctx_rev=ctx_rev,
repository_clone_url=repository_clone_url,
metadata_dict={},
status=self.app.model.ToolShedRepository.installation_status.
NEW,
current_changeset_revision=None,
owner=self.repository_owner,
dist_to_shed=True)
shed_util.update_tool_shed_repository_status(
self.app, tool_shed_repository,
self.app.model.ToolShedRepository.installation_status.CLONING)
cloned_ok, error_message = suc.clone_repository(
repository_clone_url, os.path.abspath(install_dir), ctx_rev)
if cloned_ok:
self.handle_repository_contents(
tool_shed_repository=tool_shed_repository,
repository_clone_url=repository_clone_url,
relative_install_dir=relative_install_dir,
repository_elem=repository_elem,
install_dependencies=install_dependencies)
self.app.sa_session.refresh(tool_shed_repository)
metadata_dict = tool_shed_repository.metadata
if 'tools' in metadata_dict:
shed_util.update_tool_shed_repository_status(
self.app, tool_shed_repository,
self.app.model.ToolShedRepository.installation_status.
SETTING_TOOL_VERSIONS)
# Get the tool_versions from the tool shed for each tool in the installed change set.
url = '%s/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \
( tool_shed_url, tool_shed_repository.name, self.repository_owner, installed_changeset_revision )
response = urllib2.urlopen(url)
text = response.read()
response.close()
if text:
tool_version_dicts = from_json_string(text)
shed_util.handle_tool_versions(self.app,
tool_version_dicts,
tool_shed_repository)
else:
# Set the tool versions since they seem to be missing for this repository in the tool shed.
# CRITICAL NOTE: These default settings may not properly handle all parent/child associations.
for tool_dict in metadata_dict['tools']:
flush_needed = False
tool_id = tool_dict['guid']
old_tool_id = tool_dict['id']
tool_version = tool_dict['version']
tool_version_using_old_id = shed_util.get_tool_version(
self.app, old_tool_id)
tool_version_using_guid = shed_util.get_tool_version(
self.app, tool_id)
if not tool_version_using_old_id:
tool_version_using_old_id = self.app.model.ToolVersion(
tool_id=old_tool_id,
tool_shed_repository=tool_shed_repository)
self.app.sa_session.add(
tool_version_using_old_id)
self.app.sa_session.flush()
if not tool_version_using_guid:
tool_version_using_guid = self.app.model.ToolVersion(
tool_id=tool_id,
tool_shed_repository=tool_shed_repository)
self.app.sa_session.add(
tool_version_using_guid)
self.app.sa_session.flush()
# Associate the two versions as parent / child.
tool_version_association = shed_util.get_tool_version_association(
self.app, tool_version_using_old_id,
tool_version_using_guid)
if not tool_version_association:
tool_version_association = self.app.model.ToolVersionAssociation(
tool_id=tool_version_using_guid.id,
parent_id=tool_version_using_old_id.id)
self.app.sa_session.add(
tool_version_association)
self.app.sa_session.flush()
shed_util.update_tool_shed_repository_status(
self.app, tool_shed_repository, self.app.model.
ToolShedRepository.installation_status.INSTALLED)
def handle_repository_contents( self, tool_shed_repository, repository_clone_url, relative_install_dir, repository_elem, install_dependencies ):
"""Generate the metadata for the installed tool shed repository, among other things."""
tool_panel_dict_for_display = odict()
if self.tool_path:
repo_install_dir = os.path.join( self.tool_path, relative_install_dir )
else:
repo_install_dir = relative_install_dir
for tool_elem in repository_elem:
# The tool_elem looks something like this: <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" />
tool_config = tool_elem.get( 'file' )
guid = self.get_guid( repository_clone_url, relative_install_dir, tool_config )
# See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems.
is_displayed, tool_sections = self.get_containing_tool_sections( tool_config )
if is_displayed:
tool_panel_dict_for_tool_config = shed_util.generate_tool_panel_dict_for_tool_config( guid, tool_config, tool_sections=tool_sections )
for k, v in tool_panel_dict_for_tool_config.items():
tool_panel_dict_for_display[ k ] = v
else:
print 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' \
% ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) )
metadata_dict, invalid_file_tups = suc.generate_metadata_for_changeset_revision( app=self.app,
repository=tool_shed_repository,
changeset_revision=tool_shed_repository.changeset_revision,
repository_clone_url=repository_clone_url,
shed_config_dict = self.shed_config_dict,
relative_install_dir=relative_install_dir,
repository_files_dir=None,
resetting_all_metadata_on_repository=False,
updating_installed_repository=False,
persist=True )
tool_shed_repository.metadata = metadata_dict
self.app.sa_session.add( tool_shed_repository )
self.app.sa_session.flush()
if 'tool_dependencies' in metadata_dict:
# All tool_dependency objects must be created before the tools are processed even if no tool dependencies will be installed.
tool_dependencies = shed_util.create_tool_dependency_objects( self.app, tool_shed_repository, relative_install_dir, set_status=True )
else:
tool_dependencies = None
if 'tools' in metadata_dict:
sample_files = metadata_dict.get( 'sample_files', [] )
sample_files = [ str( s ) for s in sample_files ]
tool_index_sample_files = shed_util.get_tool_index_sample_files( sample_files )
shed_util.copy_sample_files( self.app, tool_index_sample_files, tool_path=self.tool_path )
sample_files_copied = [ s for s in tool_index_sample_files ]
repository_tools_tups = suc.get_repository_tools_tups( self.app, metadata_dict )
if repository_tools_tups:
# Handle missing data table entries for tool parameters that are dynamically generated select lists.
repository_tools_tups = shed_util.handle_missing_data_table_entry( self.app, relative_install_dir, self.tool_path, repository_tools_tups )
# Handle missing index files for tool parameters that are dynamically generated select lists.
repository_tools_tups, sample_files_copied = shed_util.handle_missing_index_file( self.app,
self.tool_path,
sample_files,
repository_tools_tups,
sample_files_copied )
# Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance.
shed_util.copy_sample_files( self.app, sample_files, tool_path=self.tool_path, sample_files_copied=sample_files_copied )
if install_dependencies and tool_dependencies and 'tool_dependencies' in metadata_dict:
# Install tool dependencies.
shed_util.update_tool_shed_repository_status( self.app,
tool_shed_repository,
self.app.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES )
# Get the tool_dependencies.xml file from disk.
tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', repo_install_dir )
installed_tool_dependencies = shed_util.handle_tool_dependencies( app=self.app,
tool_shed_repository=tool_shed_repository,
tool_dependencies_config=tool_dependencies_config,
tool_dependencies=tool_dependencies )
for installed_tool_dependency in installed_tool_dependencies:
if installed_tool_dependency.status == self.app.model.ToolDependency.installation_status.ERROR:
print '\nThe following error occurred from the InstallManager while installing tool dependency ', installed_tool_dependency.name, ':'
print installed_tool_dependency.error_message, '\n\n'
shed_util.add_to_tool_panel( self.app,
tool_shed_repository.name,
repository_clone_url,
tool_shed_repository.installed_changeset_revision,
repository_tools_tups,
self.repository_owner,
self.migrated_tools_config,
tool_panel_dict=tool_panel_dict_for_display,
new_install=True )
if 'datatypes' in metadata_dict:
tool_shed_repository.status = self.app.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES
if not tool_shed_repository.includes_datatypes:
tool_shed_repository.includes_datatypes = True
self.app.sa_session.add( tool_shed_repository )
self.app.sa_session.flush()
work_dir = tempfile.mkdtemp()
datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', repo_install_dir )
# Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started
# after this installation completes.
converter_path, display_path = shed_util.alter_config_and_load_prorietary_datatypes( self.app, datatypes_config, repo_install_dir, override=False ) #repo_install_dir was relative_install_dir
if converter_path or display_path:
# Create a dictionary of tool shed repository related information.
repository_dict = shed_util.create_repository_dict_for_proprietary_datatypes( tool_shed=self.tool_shed,
name=tool_shed_repository.name,
owner=self.repository_owner,
installed_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dicts=metadata_dict.get( 'tools', [] ),
converter_path=converter_path,
display_path=display_path )
if converter_path:
# Load proprietary datatype converters
self.app.datatypes_registry.load_datatype_converters( self.toolbox, installed_repository_dict=repository_dict )
if display_path:
# Load proprietary datatype display applications
self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict )
try:
shutil.rmtree( work_dir )
except:
pass
def handle_repository_contents(self, tool_shed_repository,
repository_clone_url, relative_install_dir,
repository_elem, install_dependencies):
"""Generate the metadata for the installed tool shed repository, among other things."""
tool_panel_dict_for_display = odict()
if self.tool_path:
repo_install_dir = os.path.join(self.tool_path,
relative_install_dir)
else:
repo_install_dir = relative_install_dir
for tool_elem in repository_elem:
# The tool_elem looks something like this: <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" />
tool_config = tool_elem.get('file')
guid = self.get_guid(repository_clone_url, relative_install_dir,
tool_config)
# See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems.
is_displayed, tool_sections = self.get_containing_tool_sections(
tool_config)
if is_displayed:
tool_panel_dict_for_tool_config = shed_util.generate_tool_panel_dict_for_tool_config(
guid, tool_config, tool_sections=tool_sections)
for k, v in tool_panel_dict_for_tool_config.items():
tool_panel_dict_for_display[k] = v
else:
print 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' \
% ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) )
metadata_dict, invalid_file_tups = suc.generate_metadata_for_changeset_revision(
app=self.app,
repository=tool_shed_repository,
changeset_revision=tool_shed_repository.changeset_revision,
repository_clone_url=repository_clone_url,
shed_config_dict=self.shed_config_dict,
relative_install_dir=relative_install_dir,
repository_files_dir=None,
resetting_all_metadata_on_repository=False,
updating_installed_repository=False,
persist=True)
tool_shed_repository.metadata = metadata_dict
self.app.sa_session.add(tool_shed_repository)
self.app.sa_session.flush()
if 'tool_dependencies' in metadata_dict:
# All tool_dependency objects must be created before the tools are processed even if no tool dependencies will be installed.
tool_dependencies = shed_util.create_tool_dependency_objects(
self.app,
tool_shed_repository,
relative_install_dir,
set_status=True)
else:
tool_dependencies = None
if 'tools' in metadata_dict:
sample_files = metadata_dict.get('sample_files', [])
sample_files = [str(s) for s in sample_files]
tool_index_sample_files = shed_util.get_tool_index_sample_files(
sample_files)
shed_util.copy_sample_files(self.app,
tool_index_sample_files,
tool_path=self.tool_path)
sample_files_copied = [s for s in tool_index_sample_files]
repository_tools_tups = suc.get_repository_tools_tups(
self.app, metadata_dict)
if repository_tools_tups:
# Handle missing data table entries for tool parameters that are dynamically generated select lists.
repository_tools_tups = shed_util.handle_missing_data_table_entry(
self.app, relative_install_dir, self.tool_path,
repository_tools_tups)
# Handle missing index files for tool parameters that are dynamically generated select lists.
repository_tools_tups, sample_files_copied = shed_util.handle_missing_index_file(
self.app, self.tool_path, sample_files,
repository_tools_tups, sample_files_copied)
# Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance.
shed_util.copy_sample_files(
self.app,
sample_files,
tool_path=self.tool_path,
sample_files_copied=sample_files_copied)
if install_dependencies and tool_dependencies and 'tool_dependencies' in metadata_dict:
# Install tool dependencies.
shed_util.update_tool_shed_repository_status(
self.app, tool_shed_repository,
self.app.model.ToolShedRepository.installation_status.
INSTALLING_TOOL_DEPENDENCIES)
# Get the tool_dependencies.xml file from disk.
tool_dependencies_config = suc.get_config_from_disk(
'tool_dependencies.xml', repo_install_dir)
installed_tool_dependencies = shed_util.handle_tool_dependencies(
app=self.app,
tool_shed_repository=tool_shed_repository,
tool_dependencies_config=tool_dependencies_config,
tool_dependencies=tool_dependencies)
for installed_tool_dependency in installed_tool_dependencies:
if installed_tool_dependency.status == self.app.model.ToolDependency.installation_status.ERROR:
print '\nThe following error occurred from the InstallManager while installing tool dependency ', installed_tool_dependency.name, ':'
print installed_tool_dependency.error_message, '\n\n'
shed_util.add_to_tool_panel(
self.app,
tool_shed_repository.name,
repository_clone_url,
tool_shed_repository.installed_changeset_revision,
repository_tools_tups,
self.repository_owner,
self.migrated_tools_config,
tool_panel_dict=tool_panel_dict_for_display,
new_install=True)
if 'datatypes' in metadata_dict:
tool_shed_repository.status = self.app.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES
if not tool_shed_repository.includes_datatypes:
tool_shed_repository.includes_datatypes = True
self.app.sa_session.add(tool_shed_repository)
self.app.sa_session.flush()
work_dir = tempfile.mkdtemp()
datatypes_config = suc.get_config_from_disk(
'datatypes_conf.xml', repo_install_dir)
# Load proprietary data types required by tools. The value of override is not important here since the Galaxy server will be started
# after this installation completes.
converter_path, display_path = shed_util.alter_config_and_load_prorietary_datatypes(
self.app, datatypes_config, repo_install_dir,
override=False) #repo_install_dir was relative_install_dir
if converter_path or display_path:
# Create a dictionary of tool shed repository related information.
repository_dict = shed_util.create_repository_dict_for_proprietary_datatypes(
tool_shed=self.tool_shed,
name=tool_shed_repository.name,
owner=self.repository_owner,
installed_changeset_revision=tool_shed_repository.
installed_changeset_revision,
tool_dicts=metadata_dict.get('tools', []),
converter_path=converter_path,
display_path=display_path)
if converter_path:
# Load proprietary datatype converters
self.app.datatypes_registry.load_datatype_converters(
self.toolbox, installed_repository_dict=repository_dict)
if display_path:
# Load proprietary datatype display applications
self.app.datatypes_registry.load_display_applications(
installed_repository_dict=repository_dict)
try:
shutil.rmtree(work_dir)
except:
pass
