Exemple #1
0
 def build_repository_containers( self, repository, changeset_revision, repository_dependencies, repository_metadata,
                                  exclude=None ):
     """
     Return a dictionary of containers for the received repository's dependencies and
     contents for display in the Tool Shed.
     """
     if exclude is None:
         exclude = []
     containers_dict = dict( datatypes=None,
                             invalid_tools=None,
                             readme_files=None,
                             repository_dependencies=None,
                             tool_dependencies=None,
                             valid_tools=None,
                             workflows=None,
                             valid_data_managers=None
                             )
     if repository_metadata:
         metadata = repository_metadata.metadata
         lock = threading.Lock()
         lock.acquire( True )
         try:
             folder_id = 0
             # Datatypes container.
             if metadata:
                 if 'datatypes' not in exclude and 'datatypes' in metadata:
                     datatypes = metadata[ 'datatypes' ]
                     folder_id, datatypes_root_folder = self.build_datatypes_folder( folder_id, datatypes )
                     containers_dict[ 'datatypes' ] = datatypes_root_folder
             # Invalid repository dependencies container.
             if metadata:
                 if 'invalid_repository_dependencies' not in exclude and 'invalid_repository_dependencies' in metadata:
                     invalid_repository_dependencies = metadata[ 'invalid_repository_dependencies' ]
                     folder_id, invalid_repository_dependencies_root_folder = \
                         self.build_invalid_repository_dependencies_root_folder( folder_id,
                                                                                 invalid_repository_dependencies )
                     containers_dict[ 'invalid_repository_dependencies' ] = invalid_repository_dependencies_root_folder
             # Invalid tool dependencies container.
             if metadata:
                 if 'invalid_tool_dependencies' not in exclude and 'invalid_tool_dependencies' in metadata:
                     invalid_tool_dependencies = metadata[ 'invalid_tool_dependencies' ]
                     folder_id, invalid_tool_dependencies_root_folder = \
                         self.build_invalid_tool_dependencies_root_folder( folder_id,
                                                                           invalid_tool_dependencies )
                     containers_dict[ 'invalid_tool_dependencies' ] = invalid_tool_dependencies_root_folder
             # Invalid tools container.
             if metadata:
                 if 'invalid_tools' not in exclude and 'invalid_tools' in metadata:
                     invalid_tool_configs = metadata[ 'invalid_tools' ]
                     folder_id, invalid_tools_root_folder = \
                         self.build_invalid_tools_folder( folder_id,
                                                          invalid_tool_configs,
                                                          changeset_revision,
                                                          repository=repository,
                                                          label='Invalid tools' )
                     containers_dict[ 'invalid_tools' ] = invalid_tools_root_folder
             # Readme files container.
             if metadata:
                 if 'readme_files' not in exclude and 'readme_files' in metadata:
                     readme_files_dict = readme_util.build_readme_files_dict( self.app, repository, changeset_revision, metadata )
                     folder_id, readme_files_root_folder = self.build_readme_files_folder( folder_id, readme_files_dict )
                     containers_dict[ 'readme_files' ] = readme_files_root_folder
             if 'repository_dependencies' not in exclude:
                 # Repository dependencies container.
                 folder_id, repository_dependencies_root_folder = \
                     self.build_repository_dependencies_folder( folder_id=folder_id,
                                                                repository_dependencies=repository_dependencies,
                                                                label='Repository dependencies',
                                                                installed=False )
                 if repository_dependencies_root_folder:
                     containers_dict[ 'repository_dependencies' ] = repository_dependencies_root_folder
             # Tool dependencies container.
             if metadata:
                 if 'tool_dependencies' not in exclude and 'tool_dependencies' in metadata:
                     tool_dependencies = metadata[ 'tool_dependencies' ]
                     if 'orphan_tool_dependencies' in metadata:
                         # The use of the orphan_tool_dependencies category in metadata has been deprecated,
                         # but we still need to check in case the metadata is out of date.
                         orphan_tool_dependencies = metadata[ 'orphan_tool_dependencies' ]
                         tool_dependencies.update( orphan_tool_dependencies )
                     # Tool dependencies can be categorized as orphans only if the repository contains tools.
                     if 'tools' not in exclude:
                         tools = metadata.get( 'tools', [] )
                         tools.extend( metadata.get( 'invalid_tools', [] ) )
                     folder_id, tool_dependencies_root_folder = \
                         self.build_tool_dependencies_folder( folder_id,
                                                              tool_dependencies,
                                                              missing=False,
                                                              new_install=False )
                     containers_dict[ 'tool_dependencies' ] = tool_dependencies_root_folder
             # Valid tools container.
             if metadata:
                 if 'tools' not in exclude and 'tools' in metadata:
                     valid_tools = metadata[ 'tools' ]
                     folder_id, valid_tools_root_folder = self.build_tools_folder( folder_id,
                                                                                   valid_tools,
                                                                                   repository,
                                                                                   changeset_revision,
                                                                                   label='Valid tools' )
                     containers_dict[ 'valid_tools' ] = valid_tools_root_folder
             # Workflows container.
             if metadata:
                 if 'workflows' not in exclude and 'workflows' in metadata:
                     workflows = metadata[ 'workflows' ]
                     folder_id, workflows_root_folder = \
                         self.build_workflows_folder( folder_id=folder_id,
                                                      workflows=workflows,
                                                      repository_metadata_id=repository_metadata.id,
                                                      repository_id=None,
                                                      label='Workflows' )
                     containers_dict[ 'workflows' ] = workflows_root_folder
             # Valid Data Managers container
             if metadata:
                 if 'data_manager' not in exclude and 'data_manager' in metadata:
                     data_managers = metadata['data_manager'].get( 'data_managers', None )
                     folder_id, data_managers_root_folder = \
                         self.build_data_managers_folder( folder_id, data_managers, label="Data Managers" )
                     containers_dict[ 'valid_data_managers' ] = data_managers_root_folder
                     error_messages = metadata['data_manager'].get( 'error_messages', None )
                     data_managers = metadata['data_manager'].get( 'invalid_data_managers', None )
                     folder_id, data_managers_root_folder = \
                         self.build_invalid_data_managers_folder( folder_id,
                                                                  data_managers,
                                                                  error_messages,
                                                                  label="Invalid Data Managers" )
                     containers_dict[ 'invalid_data_managers' ] = data_managers_root_folder
         except Exception as e:
             log.exception( "Exception in build_repository_containers: %s" % str( e ) )
         finally:
             lock.release()
     return containers_dict
Exemple #2
0
 def populate_containers_dict_from_repository_metadata(
         self,
         tool_shed_url,
         tool_path,
         repository,
         reinstalling=False,
         required_repo_info_dicts=None):
     """
     Retrieve necessary information from the received repository's metadata to populate the
     containers_dict for display.  This method is called only from Galaxy (not the tool shed)
     when displaying repository dependencies for installed repositories and when displaying
     them for uninstalled repositories that are being reinstalled.
     """
     metadata = repository.metadata
     if metadata:
         # Handle proprietary datatypes.
         datatypes = metadata.get('datatypes', None)
         # Handle invalid tools.
         invalid_tools = metadata.get('invalid_tools', None)
         # Handle README files.
         if repository.has_readme_files:
             if reinstalling or repository.status not in \
                 [ self.app.install_model.ToolShedRepository.installation_status.DEACTIVATED,
                   self.app.install_model.ToolShedRepository.installation_status.INSTALLED ]:
                 # Since we're reinstalling, we need to send a request to the tool shed to get the README files.
                 tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
                     self.app, tool_shed_url)
                 params = dict(name=str(repository.name),
                               owner=str(repository.owner),
                               changeset_revision=str(
                                   repository.installed_changeset_revision))
                 pathspec = ['repository', 'get_readme_files']
                 raw_text = common_util.tool_shed_get(self.app,
                                                      tool_shed_url,
                                                      pathspec=pathspec,
                                                      params=params)
                 readme_files_dict = json.loads(raw_text)
             else:
                 readme_files_dict = readme_util.build_readme_files_dict(
                     self.app, repository, repository.changeset_revision,
                     repository.metadata, tool_path)
         else:
             readme_files_dict = None
         # Handle repository dependencies.
         installed_repository_dependencies, missing_repository_dependencies = \
             self.app.installed_repository_manager.get_installed_and_missing_repository_dependencies( repository )
         # Handle the current repository's tool dependencies.
         repository_tool_dependencies = metadata.get(
             'tool_dependencies', None)
         # Make sure to display missing tool dependencies as well.
         repository_invalid_tool_dependencies = metadata.get(
             'invalid_tool_dependencies', None)
         if repository_invalid_tool_dependencies is not None:
             if repository_tool_dependencies is None:
                 repository_tool_dependencies = {}
             repository_tool_dependencies.update(
                 repository_invalid_tool_dependencies)
         repository_installed_tool_dependencies, repository_missing_tool_dependencies = \
             self.get_installed_and_missing_tool_dependencies_for_installed_repository( repository,
                                                                                        repository_tool_dependencies )
         if reinstalling:
             installed_tool_dependencies, missing_tool_dependencies = \
                 self.populate_tool_dependencies_dicts( tool_shed_url,
                                                        tool_path,
                                                        repository_installed_tool_dependencies,
                                                        repository_missing_tool_dependencies,
                                                        required_repo_info_dicts )
         else:
             installed_tool_dependencies = repository_installed_tool_dependencies
             missing_tool_dependencies = repository_missing_tool_dependencies
         # Handle valid tools.
         valid_tools = metadata.get('tools', None)
         # Handle workflows.
         workflows = metadata.get('workflows', None)
         # Handle Data Managers
         valid_data_managers = None
         invalid_data_managers = None
         data_managers_errors = None
         if 'data_manager' in metadata:
             valid_data_managers = metadata['data_manager'].get(
                 'data_managers', None)
             invalid_data_managers = metadata['data_manager'].get(
                 'invalid_data_managers', None)
             data_managers_errors = metadata['data_manager'].get(
                 'messages', None)
         gucm = GalaxyUtilityContainerManager(self.app)
         containers_dict = gucm.build_repository_containers(
             repository=repository,
             datatypes=datatypes,
             invalid_tools=invalid_tools,
             missing_repository_dependencies=missing_repository_dependencies,
             missing_tool_dependencies=missing_tool_dependencies,
             readme_files_dict=readme_files_dict,
             repository_dependencies=installed_repository_dependencies,
             tool_dependencies=installed_tool_dependencies,
             valid_tools=valid_tools,
             workflows=workflows,
             valid_data_managers=valid_data_managers,
             invalid_data_managers=invalid_data_managers,
             data_managers_errors=data_managers_errors,
             new_install=False,
             reinstalling=reinstalling)
     else:
         containers_dict = dict(datatypes=None,
                                invalid_tools=None,
                                readme_files_dict=None,
                                repository_dependencies=None,
                                tool_dependencies=None,
                                valid_tools=None,
                                workflows=None)
     return containers_dict
Exemple #3
0
 def build_repository_containers(self,
                                 repository,
                                 changeset_revision,
                                 repository_dependencies,
                                 repository_metadata,
                                 exclude=None):
     """
     Return a dictionary of containers for the received repository's dependencies and
     contents for display in the Tool Shed.
     """
     if exclude is None:
         exclude = []
     containers_dict = dict(datatypes=None,
                            invalid_tools=None,
                            readme_files=None,
                            repository_dependencies=None,
                            tool_dependencies=None,
                            valid_tools=None,
                            workflows=None,
                            valid_data_managers=None)
     if repository_metadata:
         metadata = repository_metadata.metadata
         lock = threading.Lock()
         lock.acquire(True)
         try:
             folder_id = 0
             # Datatypes container.
             if metadata:
                 if 'datatypes' not in exclude and 'datatypes' in metadata:
                     datatypes = metadata['datatypes']
                     folder_id, datatypes_root_folder = self.build_datatypes_folder(
                         folder_id, datatypes)
                     containers_dict['datatypes'] = datatypes_root_folder
             # Invalid repository dependencies container.
             if metadata:
                 if 'invalid_repository_dependencies' not in exclude and 'invalid_repository_dependencies' in metadata:
                     invalid_repository_dependencies = metadata[
                         'invalid_repository_dependencies']
                     folder_id, invalid_repository_dependencies_root_folder = \
                         self.build_invalid_repository_dependencies_root_folder(folder_id,
                                                                                invalid_repository_dependencies)
                     containers_dict[
                         'invalid_repository_dependencies'] = invalid_repository_dependencies_root_folder
             # Invalid tool dependencies container.
             if metadata:
                 if 'invalid_tool_dependencies' not in exclude and 'invalid_tool_dependencies' in metadata:
                     invalid_tool_dependencies = metadata[
                         'invalid_tool_dependencies']
                     folder_id, invalid_tool_dependencies_root_folder = \
                         self.build_invalid_tool_dependencies_root_folder(folder_id,
                                                                          invalid_tool_dependencies)
                     containers_dict[
                         'invalid_tool_dependencies'] = invalid_tool_dependencies_root_folder
             # Invalid tools container.
             if metadata:
                 if 'invalid_tools' not in exclude and 'invalid_tools' in metadata:
                     invalid_tool_configs = metadata['invalid_tools']
                     folder_id, invalid_tools_root_folder = \
                         self.build_invalid_tools_folder(folder_id,
                                                         invalid_tool_configs,
                                                         changeset_revision,
                                                         repository=repository,
                                                         label='Invalid tools')
                     containers_dict[
                         'invalid_tools'] = invalid_tools_root_folder
             # Readme files container.
             if metadata:
                 if 'readme_files' not in exclude and 'readme_files' in metadata:
                     readme_files_dict = readme_util.build_readme_files_dict(
                         self.app, repository, changeset_revision, metadata)
                     folder_id, readme_files_root_folder = self.build_readme_files_folder(
                         folder_id, readme_files_dict)
                     containers_dict[
                         'readme_files'] = readme_files_root_folder
             if 'repository_dependencies' not in exclude:
                 # Repository dependencies container.
                 folder_id, repository_dependencies_root_folder = \
                     self.build_repository_dependencies_folder(folder_id=folder_id,
                                                               repository_dependencies=repository_dependencies,
                                                               label='Repository dependencies',
                                                               installed=False)
                 if repository_dependencies_root_folder:
                     containers_dict[
                         'repository_dependencies'] = repository_dependencies_root_folder
             # Tool dependencies container.
             if metadata:
                 if 'tool_dependencies' not in exclude and 'tool_dependencies' in metadata:
                     tool_dependencies = metadata['tool_dependencies']
                     if 'orphan_tool_dependencies' in metadata:
                         # The use of the orphan_tool_dependencies category in metadata has been deprecated,
                         # but we still need to check in case the metadata is out of date.
                         orphan_tool_dependencies = metadata[
                             'orphan_tool_dependencies']
                         tool_dependencies.update(orphan_tool_dependencies)
                     # Tool dependencies can be categorized as orphans only if the repository contains tools.
                     if 'tools' not in exclude:
                         tools = metadata.get('tools', [])
                         tools.extend(metadata.get('invalid_tools', []))
                     folder_id, tool_dependencies_root_folder = \
                         self.build_tool_dependencies_folder(folder_id,
                                                             tool_dependencies,
                                                             missing=False,
                                                             new_install=False)
                     containers_dict[
                         'tool_dependencies'] = tool_dependencies_root_folder
             # Valid tools container.
             if metadata:
                 if 'tools' not in exclude and 'tools' in metadata:
                     valid_tools = metadata['tools']
                     folder_id, valid_tools_root_folder = self.build_tools_folder(
                         folder_id,
                         valid_tools,
                         repository,
                         changeset_revision,
                         label='Valid tools')
                     containers_dict[
                         'valid_tools'] = valid_tools_root_folder
             # Workflows container.
             if metadata:
                 if 'workflows' not in exclude and 'workflows' in metadata:
                     workflows = metadata['workflows']
                     folder_id, workflows_root_folder = \
                         self.build_workflows_folder(folder_id=folder_id,
                                                     workflows=workflows,
                                                     repository_metadata_id=repository_metadata.id,
                                                     repository_id=None,
                                                     label='Workflows')
                     containers_dict['workflows'] = workflows_root_folder
             # Valid Data Managers container
             if metadata:
                 if 'data_manager' not in exclude and 'data_manager' in metadata:
                     data_managers = metadata['data_manager'].get(
                         'data_managers', None)
                     folder_id, data_managers_root_folder = \
                         self.build_data_managers_folder(folder_id, data_managers, label="Data Managers")
                     containers_dict[
                         'valid_data_managers'] = data_managers_root_folder
                     error_messages = metadata['data_manager'].get(
                         'error_messages', None)
                     data_managers = metadata['data_manager'].get(
                         'invalid_data_managers', None)
                     folder_id, data_managers_root_folder = \
                         self.build_invalid_data_managers_folder(folder_id,
                                                                 data_managers,
                                                                 error_messages,
                                                                 label="Invalid Data Managers")
                     containers_dict[
                         'invalid_data_managers'] = data_managers_root_folder
         except Exception:
             log.exception("Exception in build_repository_containers")
         finally:
             lock.release()
     return containers_dict
 def populate_containers_dict_from_repository_metadata( self, tool_shed_url, tool_path, repository, reinstalling=False,
                                                        required_repo_info_dicts=None ):
     """
     Retrieve necessary information from the received repository's metadata to populate the
     containers_dict for display.  This method is called only from Galaxy (not the tool shed)
     when displaying repository dependencies for installed repositories and when displaying
     them for uninstalled repositories that are being reinstalled.
     """
     metadata = repository.metadata
     if metadata:
         # Handle proprietary datatypes.
         datatypes = metadata.get( 'datatypes', None )
         # Handle invalid tools.
         invalid_tools = metadata.get( 'invalid_tools', None )
         # Handle README files.
         if repository.has_readme_files:
             if reinstalling or repository.status not in \
                 [ self.app.install_model.ToolShedRepository.installation_status.DEACTIVATED,
                   self.app.install_model.ToolShedRepository.installation_status.INSTALLED ]:
                 # Since we're reinstalling, we need to send a request to the tool shed to get the README files.
                 tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed_url )
                 params = dict( name=str( repository.name ),
                                owner=str( repository.owner ),
                                changeset_revision=str( repository.installed_changeset_revision ) )
                 pathspec = [ 'repository', 'get_readme_files' ]
                 raw_text = util.url_get( tool_shed_url, password_mgr=self.app.tool_shed_registry.url_auth( tool_shed_url ), pathspec=pathspec, params=params )
                 readme_files_dict = json.loads( raw_text )
             else:
                 readme_files_dict = readme_util.build_readme_files_dict( self.app,
                                                                          repository,
                                                                          repository.changeset_revision,
                                                                          repository.metadata, tool_path )
         else:
             readme_files_dict = None
         # Handle repository dependencies.
         installed_repository_dependencies, missing_repository_dependencies = \
             self.app.installed_repository_manager.get_installed_and_missing_repository_dependencies( repository )
         # Handle the current repository's tool dependencies.
         repository_tool_dependencies = metadata.get( 'tool_dependencies', None )
         # Make sure to display missing tool dependencies as well.
         repository_invalid_tool_dependencies = metadata.get( 'invalid_tool_dependencies', None )
         if repository_invalid_tool_dependencies is not None:
             if repository_tool_dependencies is None:
                 repository_tool_dependencies = {}
             repository_tool_dependencies.update( repository_invalid_tool_dependencies )
         repository_installed_tool_dependencies, repository_missing_tool_dependencies = \
             self.get_installed_and_missing_tool_dependencies_for_installed_repository( repository,
                                                                                        repository_tool_dependencies )
         if reinstalling:
             installed_tool_dependencies, missing_tool_dependencies = \
                 self.populate_tool_dependencies_dicts( tool_shed_url,
                                                        tool_path,
                                                        repository_installed_tool_dependencies,
                                                        repository_missing_tool_dependencies,
                                                        required_repo_info_dicts )
         else:
             installed_tool_dependencies = repository_installed_tool_dependencies
             missing_tool_dependencies = repository_missing_tool_dependencies
         # Handle valid tools.
         valid_tools = metadata.get( 'tools', None )
         # Handle workflows.
         workflows = metadata.get( 'workflows', None )
         # Handle Data Managers
         valid_data_managers = None
         invalid_data_managers = None
         data_managers_errors = None
         if 'data_manager' in metadata:
             valid_data_managers = metadata['data_manager'].get( 'data_managers', None )
             invalid_data_managers = metadata['data_manager'].get( 'invalid_data_managers', None )
             data_managers_errors = metadata['data_manager'].get( 'messages', None )
         gucm = GalaxyUtilityContainerManager( self.app )
         containers_dict = gucm.build_repository_containers( repository=repository,
                                                             datatypes=datatypes,
                                                             invalid_tools=invalid_tools,
                                                             missing_repository_dependencies=missing_repository_dependencies,
                                                             missing_tool_dependencies=missing_tool_dependencies,
                                                             readme_files_dict=readme_files_dict,
                                                             repository_dependencies=installed_repository_dependencies,
                                                             tool_dependencies=installed_tool_dependencies,
                                                             valid_tools=valid_tools,
                                                             workflows=workflows,
                                                             valid_data_managers=valid_data_managers,
                                                             invalid_data_managers=invalid_data_managers,
                                                             data_managers_errors=data_managers_errors,
                                                             new_install=False,
                                                             reinstalling=reinstalling )
     else:
         containers_dict = dict( datatypes=None,
                                 invalid_tools=None,
                                 readme_files_dict=None,
                                 repository_dependencies=None,
                                 tool_dependencies=None,
                                 valid_tools=None,
                                 workflows=None )
     return containers_dict