def reset_metadata_on_installed_repositories(self, trans, payload, **kwd):
"""
PUT /api/tool_shed_repositories/reset_metadata_on_installed_repositories
Resets all metadata on all repositories installed into Galaxy in an "orderly fashion".
:param key: the API key of the Galaxy admin user.
"""
start_time = strftime("%Y-%m-%d %H:%M:%S")
results = dict(start_time=start_time,
successful_count=0,
unsuccessful_count=0,
repository_status=[])
# Make sure the current user's API key proves he is an admin user in this Galaxy instance.
if not trans.user_is_admin():
raise HTTPForbidden(
detail=
'You are not authorized to reset metadata on repositories installed into this Galaxy instance.'
)
irmm = InstalledRepositoryMetadataManager(trans.app)
query = irmm.get_query_for_setting_metadata_on_repositories(
order=False)
# Now reset metadata on all remaining repositories.
for repository in query:
repository_id = trans.security.encode_id(repository.id)
try:
invalid_file_tups, metadata_dict = irmm.reset_all_metadata_on_installed_repository(
repository_id)
if invalid_file_tups:
message = tool_util.generate_message_for_invalid_tools(
trans.app,
invalid_file_tups,
repository,
None,
as_html=False)
results['unsuccessful_count'] += 1
else:
message = "Successfully reset metadata on repository %s owned by %s" % \
( str( repository.name ), str( repository.owner ) )
results['successful_count'] += 1
except Exception, e:
message = "Error resetting metadata on repository %s owned by %s: %s" % \
( str( repository.name ), str( repository.owner ), str( e ) )
results['unsuccessful_count'] += 1
results['repository_status'].append(message)
def reset_metadata_on_selected_installed_repositories(self, trans, **kwd):
repository_ids = util.listify(kwd.get("repository_ids"))
if repository_ids:
irmm = InstalledRepositoryMetadataManager(self.app)
failed = []
successful = []
for repository_id in repository_ids:
try:
repository = repository_util.get_installed_tool_shed_repository(
self.app, repository_id)
irmm.set_repository(repository)
irmm.reset_all_metadata_on_installed_repository()
if irmm.invalid_file_tups:
failed.append(repository_id)
else:
successful.append(repository_id)
except Exception:
failed.append(repository_id)
if successful:
message = "Successful reset of metadata for %s." % len(
successful)
if failed:
message += " Failed for %s." % len(failed)
elif failed:
message = "Failed to reset metadata for %s." % len(failed)
return dict(message=message, successful=successful, failed=failed)
else:
raise exceptions.MessageException(
"Please specify repository ids [repository_ids].")
def reset_metadata_on_installed_repositories( self, trans, payload, **kwd ):
"""
PUT /api/tool_shed_repositories/reset_metadata_on_installed_repositories
Resets all metadata on all repositories installed into Galaxy in an "orderly fashion".
:param key: the API key of the Galaxy admin user.
"""
start_time = strftime( "%Y-%m-%d %H:%M:%S" )
results = dict( start_time=start_time,
successful_count=0,
unsuccessful_count=0,
repository_status=[] )
# Make sure the current user's API key proves he is an admin user in this Galaxy instance.
if not trans.user_is_admin():
raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' )
irmm = InstalledRepositoryMetadataManager( trans.app )
query = irmm.get_query_for_setting_metadata_on_repositories( order=False )
# Now reset metadata on all remaining repositories.
for repository in query:
repository_id = trans.security.encode_id( repository.id )
try:
invalid_file_tups, metadata_dict = irmm.reset_all_metadata_on_installed_repository( repository_id )
if invalid_file_tups:
message = tool_util.generate_message_for_invalid_tools( trans.app,
invalid_file_tups,
repository,
None,
as_html=False )
results[ 'unsuccessful_count' ] += 1
else:
message = "Successfully reset metadata on repository %s owned by %s" % \
( str( repository.name ), str( repository.owner ) )
results[ 'successful_count' ] += 1
except Exception, e:
message = "Error resetting metadata on repository %s owned by %s: %s" % \
( str( repository.name ), str( repository.owner ), str( e ) )
results[ 'unsuccessful_count' ] += 1
results[ 'repository_status' ].append( message )
% ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) )
if tool_configs_to_filter:
lock = threading.Lock()
lock.acquire( True )
try:
self.filter_and_persist_proprietary_tool_panel_configs( tool_configs_to_filter )
except Exception, e:
log.exception( "Exception attempting to filter and persist non-shed-related tool panel configs:\n%s" % str( e ) )
finally:
lock.release()
irmm = InstalledRepositoryMetadataManager( app=self.app,
tpm=self.tpm,
repository=tool_shed_repository,
changeset_revision=tool_shed_repository.changeset_revision,
repository_clone_url=repository_clone_url,
shed_config_dict = self.shed_config_dict,
relative_install_dir=relative_install_dir,
repository_files_dir=None,
resetting_all_metadata_on_repository=False,
updating_installed_repository=False,
persist=True )
irmm.generate_metadata_for_changeset_revision()
irmm_metadata_dict = irmm.get_metadata_dict()
tool_shed_repository.metadata = irmm_metadata_dict
self.app.install_model.context.add( tool_shed_repository )
self.app.install_model.context.flush()
has_tool_dependencies = self.__has_tool_dependencies( irmm_metadata_dict )
if has_tool_dependencies:
# All tool_dependency objects must be created before the tools are processed even if no
# tool dependencies will be installed.
tool_dependencies = tool_dependency_util.create_tool_dependency_objects( self.app,
def handle_repository_contents(self, tool_shed_repository, repository_clone_url, relative_install_dir, repository_elem,
install_dependencies, is_repository_dependency=False):
"""
Generate the metadata for the installed tool shed repository, among other things. If the installed tool_shed_repository
contains tools that are loaded into the Galaxy tool panel, this method will automatically eliminate all entries for each
of the tools defined in the received repository_elem from all non-shed-related tool panel configuration files since the
entries are automatically added to the reserved migrated_tools_conf.xml file as part of the migration process.
"""
tool_configs_to_filter = []
tool_panel_dict_for_display = odict()
if self.tool_path:
repo_install_dir = os.path.join(self.tool_path, relative_install_dir)
else:
repo_install_dir = relative_install_dir
if not is_repository_dependency:
for tool_elem in repository_elem:
# The tool_elem looks something like this:
# <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" />
tool_config = tool_elem.get('file')
guid = self.get_guid(repository_clone_url, relative_install_dir, tool_config)
# See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems.
is_displayed, tool_sections = self.get_containing_tool_sections(tool_config)
if is_displayed:
tool_panel_dict_for_tool_config = \
self.tpm.generate_tool_panel_dict_for_tool_config(guid,
tool_config,
tool_sections=tool_sections)
# The tool-panel_dict has the following structure.
# {<Tool guid> : [{ tool_config : <tool_config_file>,
# id: <ToolSection id>,
# version : <ToolSection version>,
# name : <TooSection name>}]}
for k, v in tool_panel_dict_for_tool_config.items():
tool_panel_dict_for_display[k] = v
for tool_panel_dict in v:
# Keep track of tool config file names associated with entries that have been made to the
# migrated_tools_conf.xml file so they can be eliminated from all non-shed-related tool panel configs.
if tool_config not in tool_configs_to_filter:
tool_configs_to_filter.append(tool_config)
else:
log.error('The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).'
% (guid, tool_config, ", ".join(self.proprietary_tool_confs or [])))
if tool_configs_to_filter:
lock = threading.Lock()
lock.acquire(True)
try:
self.filter_and_persist_proprietary_tool_panel_configs(tool_configs_to_filter)
except Exception:
log.exception("Exception attempting to filter and persist non-shed-related tool panel configs")
finally:
lock.release()
irmm = InstalledRepositoryMetadataManager(app=self.app,
tpm=self.tpm,
repository=tool_shed_repository,
changeset_revision=tool_shed_repository.changeset_revision,
repository_clone_url=repository_clone_url,
shed_config_dict=self.shed_config_dict,
relative_install_dir=relative_install_dir,
repository_files_dir=None,
resetting_all_metadata_on_repository=False,
updating_installed_repository=False,
persist=True)
irmm.generate_metadata_for_changeset_revision()
irmm_metadata_dict = irmm.get_metadata_dict()
tool_shed_repository.metadata = irmm_metadata_dict
self.app.install_model.context.add(tool_shed_repository)
self.app.install_model.context.flush()
has_tool_dependencies = self.__has_tool_dependencies(irmm_metadata_dict)
if has_tool_dependencies:
# All tool_dependency objects must be created before the tools are processed even if no
# tool dependencies will be installed.
tool_dependencies = tool_dependency_util.create_tool_dependency_objects(self.app,
tool_shed_repository,
relative_install_dir,
set_status=True)
else:
tool_dependencies = None
if 'tools' in irmm_metadata_dict:
tdtm = data_table_manager.ToolDataTableManager(self.app)
sample_files = irmm_metadata_dict.get('sample_files', [])
sample_files = [str(s) for s in sample_files]
tool_index_sample_files = tdtm.get_tool_index_sample_files(sample_files)
tool_util.copy_sample_files(self.app, tool_index_sample_files, tool_path=self.tool_path)
sample_files_copied = [s for s in tool_index_sample_files]
repository_tools_tups = irmm.get_repository_tools_tups()
if repository_tools_tups:
# Handle missing data table entries for tool parameters that are dynamically
# generated select lists.
repository_tools_tups = tdtm.handle_missing_data_table_entry(relative_install_dir,
self.tool_path,
repository_tools_tups)
# Handle missing index files for tool parameters that are dynamically generated select lists.
repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file(self.app,
self.tool_path,
sample_files,
repository_tools_tups,
sample_files_copied)
# Copy remaining sample files included in the repository to the ~/tool-data
# directory of the local Galaxy instance.
tool_util.copy_sample_files(self.app,
sample_files,
tool_path=self.tool_path,
sample_files_copied=sample_files_copied)
if not is_repository_dependency:
self.tpm.add_to_tool_panel(tool_shed_repository.name,
repository_clone_url,
tool_shed_repository.installed_changeset_revision,
repository_tools_tups,
self.repository_owner,
self.migrated_tools_config,
tool_panel_dict=tool_panel_dict_for_display,
new_install=True)
if install_dependencies and tool_dependencies and has_tool_dependencies:
# Install tool dependencies.
irm = install_manager.InstallRepositoryManager(self.app, self.tpm)
itdm = install_manager.InstallToolDependencyManager(self.app)
irm.update_tool_shed_repository_status(tool_shed_repository,
self.app.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES)
# Get the tool_dependencies.xml file from disk.
tool_dependencies_config = hg_util.get_config_from_disk('tool_dependencies.xml', repo_install_dir)
installed_tool_dependencies = itdm.install_specified_tool_dependencies(tool_shed_repository=tool_shed_repository,
tool_dependencies_config=tool_dependencies_config,
tool_dependencies=tool_dependencies,
from_tool_migration_manager=True)
for installed_tool_dependency in installed_tool_dependencies:
if installed_tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR:
log.error(
'The ToolMigrationManager returned the following error while installing tool dependency %s: %s',
installed_tool_dependency.name, installed_tool_dependency.error_message)
if 'datatypes' in irmm_metadata_dict:
cdl = custom_datatype_manager.CustomDatatypeLoader(self.app)
tool_shed_repository.status = self.app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES
if not tool_shed_repository.includes_datatypes:
tool_shed_repository.includes_datatypes = True
self.app.install_model.context.add(tool_shed_repository)
self.app.install_model.context.flush()
work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-hrc")
datatypes_config = hg_util.get_config_from_disk(suc.DATATYPES_CONFIG_FILENAME, repo_install_dir)
# Load proprietary data types required by tools. The value of override is not
# important here since the Galaxy server will be started after this installation
# completes.
converter_path, display_path = \
cdl.alter_config_and_load_prorietary_datatypes(datatypes_config,
repo_install_dir,
override=False)
if converter_path or display_path:
# Create a dictionary of tool shed repository related information.
repository_dict = \
cdl.create_repository_dict_for_proprietary_datatypes(tool_shed=self.tool_shed_url,
name=tool_shed_repository.name,
owner=self.repository_owner,
installed_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dicts=irmm_metadata_dict.get('tools', []),
converter_path=converter_path,
display_path=display_path)
if converter_path:
# Load proprietary datatype converters
self.app.datatypes_registry.load_datatype_converters(self.toolbox,
installed_repository_dict=repository_dict)
if display_path:
# Load proprietary datatype display applications
self.app.datatypes_registry.load_display_applications(self.app, installed_repository_dict=repository_dict)
basic_util.remove_dir(work_dir)
def handle_repository_contents(self,
tool_shed_repository,
repository_clone_url,
relative_install_dir,
repository_elem,
install_dependencies,
is_repository_dependency=False):
"""
Generate the metadata for the installed tool shed repository, among other things. If the installed tool_shed_repository
contains tools that are loaded into the Galaxy tool panel, this method will automatically eliminate all entries for each
of the tools defined in the received repository_elem from all non-shed-related tool panel configuration files since the
entries are automatically added to the reserved migrated_tools_conf.xml file as part of the migration process.
"""
tool_configs_to_filter = []
tool_panel_dict_for_display = odict()
if self.tool_path:
repo_install_dir = os.path.join(self.tool_path,
relative_install_dir)
else:
repo_install_dir = relative_install_dir
if not is_repository_dependency:
for tool_elem in repository_elem:
# The tool_elem looks something like this:
# <tool id="EMBOSS: antigenic1" version="5.0.0" file="emboss_antigenic.xml" />
tool_config = tool_elem.get('file')
guid = self.get_guid(repository_clone_url,
relative_install_dir, tool_config)
# See if tool_config is defined inside of a section in self.proprietary_tool_panel_elems.
is_displayed, tool_sections = self.get_containing_tool_sections(
tool_config)
if is_displayed:
tool_panel_dict_for_tool_config = \
self.tpm.generate_tool_panel_dict_for_tool_config(guid,
tool_config,
tool_sections=tool_sections)
# The tool-panel_dict has the following structure.
# {<Tool guid> : [{ tool_config : <tool_config_file>,
# id: <ToolSection id>,
# version : <ToolSection version>,
# name : <TooSection name>}]}
for k, v in tool_panel_dict_for_tool_config.items():
tool_panel_dict_for_display[k] = v
for tool_panel_dict in v:
# Keep track of tool config file names associated with entries that have been made to the
# migrated_tools_conf.xml file so they can be eliminated from all non-shed-related tool panel configs.
if tool_config not in tool_configs_to_filter:
tool_configs_to_filter.append(tool_config)
else:
log.error(
'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).'
% (guid, tool_config, ", ".join(
self.proprietary_tool_confs or [])))
if tool_configs_to_filter:
lock = threading.Lock()
lock.acquire(True)
try:
self.filter_and_persist_proprietary_tool_panel_configs(
tool_configs_to_filter)
except Exception:
log.exception(
"Exception attempting to filter and persist non-shed-related tool panel configs"
)
finally:
lock.release()
irmm = InstalledRepositoryMetadataManager(
app=self.app,
tpm=self.tpm,
repository=tool_shed_repository,
changeset_revision=tool_shed_repository.changeset_revision,
repository_clone_url=repository_clone_url,
shed_config_dict=self.shed_config_dict,
relative_install_dir=relative_install_dir,
repository_files_dir=None,
resetting_all_metadata_on_repository=False,
updating_installed_repository=False,
persist=True)
irmm.generate_metadata_for_changeset_revision()
irmm_metadata_dict = irmm.get_metadata_dict()
tool_shed_repository.metadata = irmm_metadata_dict
self.app.install_model.context.add(tool_shed_repository)
self.app.install_model.context.flush()
has_tool_dependencies = self.__has_tool_dependencies(
irmm_metadata_dict)
if has_tool_dependencies:
# All tool_dependency objects must be created before the tools are processed even if no
# tool dependencies will be installed.
tool_dependencies = tool_dependency_util.create_tool_dependency_objects(
self.app,
tool_shed_repository,
relative_install_dir,
set_status=True)
else:
tool_dependencies = None
if 'tools' in irmm_metadata_dict:
tdtm = data_table_manager.ToolDataTableManager(self.app)
sample_files = irmm_metadata_dict.get('sample_files', [])
sample_files = [str(s) for s in sample_files]
tool_index_sample_files = tdtm.get_tool_index_sample_files(
sample_files)
tool_util.copy_sample_files(self.app,
tool_index_sample_files,
tool_path=self.tool_path)
sample_files_copied = [s for s in tool_index_sample_files]
repository_tools_tups = irmm.get_repository_tools_tups()
if repository_tools_tups:
# Handle missing data table entries for tool parameters that are dynamically
# generated select lists.
repository_tools_tups = tdtm.handle_missing_data_table_entry(
relative_install_dir, self.tool_path,
repository_tools_tups)
# Handle missing index files for tool parameters that are dynamically generated select lists.
repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file(
self.app, self.tool_path, sample_files,
repository_tools_tups, sample_files_copied)
# Copy remaining sample files included in the repository to the ~/tool-data
# directory of the local Galaxy instance.
tool_util.copy_sample_files(
self.app,
sample_files,
tool_path=self.tool_path,
sample_files_copied=sample_files_copied)
if not is_repository_dependency:
self.tpm.add_to_tool_panel(
tool_shed_repository.name,
repository_clone_url,
tool_shed_repository.installed_changeset_revision,
repository_tools_tups,
self.repository_owner,
self.migrated_tools_config,
tool_panel_dict=tool_panel_dict_for_display,
new_install=True)
if install_dependencies and tool_dependencies and has_tool_dependencies:
# Install tool dependencies.
irm = install_manager.InstallRepositoryManager(self.app, self.tpm)
itdm = install_manager.InstallToolDependencyManager(self.app)
irm.update_tool_shed_repository_status(
tool_shed_repository,
self.app.install_model.ToolShedRepository.installation_status.
INSTALLING_TOOL_DEPENDENCIES)
# Get the tool_dependencies.xml file from disk.
tool_dependencies_config = hg_util.get_config_from_disk(
'tool_dependencies.xml', repo_install_dir)
installed_tool_dependencies = itdm.install_specified_tool_dependencies(
tool_shed_repository=tool_shed_repository,
tool_dependencies_config=tool_dependencies_config,
tool_dependencies=tool_dependencies,
from_tool_migration_manager=True)
for installed_tool_dependency in installed_tool_dependencies:
if installed_tool_dependency.status == self.app.install_model.ToolDependency.installation_status.ERROR:
log.error(
'The ToolMigrationManager returned the following error while installing tool dependency %s: %s',
installed_tool_dependency.name,
installed_tool_dependency.error_message)
if 'datatypes' in irmm_metadata_dict:
cdl = custom_datatype_manager.CustomDatatypeLoader(self.app)
tool_shed_repository.status = self.app.install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES
if not tool_shed_repository.includes_datatypes:
tool_shed_repository.includes_datatypes = True
self.app.install_model.context.add(tool_shed_repository)
self.app.install_model.context.flush()
work_dir = tempfile.mkdtemp(prefix="tmp-toolshed-hrc")
datatypes_config = hg_util.get_config_from_disk(
suc.DATATYPES_CONFIG_FILENAME, repo_install_dir)
# Load proprietary data types required by tools. The value of override is not
# important here since the Galaxy server will be started after this installation
# completes.
converter_path, display_path = \
cdl.alter_config_and_load_prorietary_datatypes(datatypes_config,
repo_install_dir,
override=False)
if converter_path or display_path:
# Create a dictionary of tool shed repository related information.
repository_dict = \
cdl.create_repository_dict_for_proprietary_datatypes(tool_shed=self.tool_shed_url,
name=tool_shed_repository.name,
owner=self.repository_owner,
installed_changeset_revision=tool_shed_repository.installed_changeset_revision,
tool_dicts=irmm_metadata_dict.get('tools', []),
converter_path=converter_path,
display_path=display_path)
if converter_path:
# Load proprietary datatype converters
self.app.datatypes_registry.load_datatype_converters(
self.toolbox, installed_repository_dict=repository_dict)
if display_path:
# Load proprietary datatype display applications
self.app.datatypes_registry.load_display_applications(
self.app, installed_repository_dict=repository_dict)
basic_util.remove_dir(work_dir)
lock.acquire(True)
try:
self.filter_and_persist_proprietary_tool_panel_configs(
tool_configs_to_filter)
except Exception, e:
log.exception(
"Exception attempting to filter and persist non-shed-related tool panel configs:\n%s"
% str(e))
finally:
lock.release()
irmm = InstalledRepositoryMetadataManager(
app=self.app,
tpm=self.tpm,
repository=tool_shed_repository,
changeset_revision=tool_shed_repository.changeset_revision,
repository_clone_url=repository_clone_url,
shed_config_dict=self.shed_config_dict,
relative_install_dir=relative_install_dir,
repository_files_dir=None,
resetting_all_metadata_on_repository=False,
updating_installed_repository=False,
persist=True)
irmm.generate_metadata_for_changeset_revision()
irmm_metadata_dict = irmm.get_metadata_dict()
tool_shed_repository.metadata = irmm_metadata_dict
self.app.install_model.context.add(tool_shed_repository)
self.app.install_model.context.flush()
has_tool_dependencies = self.__has_tool_dependencies(
irmm_metadata_dict)
if has_tool_dependencies:
# All tool_dependency objects must be created before the tools are processed even if no
# tool dependencies will be installed.
# migrated_tools_conf.xml file so they can be eliminated from all non-shed-related tool panel configs.
if tool_config not in tool_configs_to_filter:
tool_configs_to_filter.append( tool_config )
else:
print 'The tool "%s" (%s) has not been enabled because it is not defined in a proprietary tool config (%s).' \
% ( guid, tool_config, ", ".join( self.proprietary_tool_confs or [] ) )
if tool_configs_to_filter:
lock = threading.Lock()
lock.acquire( True )
try:
self.filter_and_persist_proprietary_tool_panel_configs( tool_configs_to_filter )
except Exception, e:
log.exception( "Exception attempting to filter and persist non-shed-related tool panel configs:\n%s" % str( e ) )
finally:
lock.release()
irmm = InstalledRepositoryMetadataManager( self.app, self.tpm )
metadata_dict, invalid_file_tups = \
irmm.generate_metadata_for_changeset_revision( repository=tool_shed_repository,
changeset_revision=tool_shed_repository.changeset_revision,
repository_clone_url=repository_clone_url,
shed_config_dict = self.shed_config_dict,
relative_install_dir=relative_install_dir,
repository_files_dir=None,
resetting_all_metadata_on_repository=False,
updating_installed_repository=False,
persist=True )
tool_shed_repository.metadata = metadata_dict
self.app.install_model.context.add( tool_shed_repository )
self.app.install_model.context.flush()
has_tool_dependencies = self.__has_tool_dependencies( metadata_dict )
if has_tool_dependencies:
