def get_repository_ids( trans, repo_info_dicts ):
    repository_ids = []
    for repo_info_dict in repo_info_dicts:
        for repository_name, repo_info_tup in repo_info_dict.items():
            description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
                suc.get_repo_info_tuple_contents( repo_info_tup )
            repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner )
            repository_ids.append( trans.security.encode_id( repository.id ) )
    return repository_ids
def populate_tool_dependencies_dicts( trans, tool_shed_url, tool_path, repository_installed_tool_dependencies, repository_missing_tool_dependencies,
                                      required_repo_info_dicts ):
    """
    Return the populated installed_tool_dependencies and missing_tool_dependencies dictionaries for all repositories defined by entries in the received
    required_repo_info_dicts.
    """
    installed_tool_dependencies = None
    missing_tool_dependencies = None
    if repository_installed_tool_dependencies is None:
        repository_installed_tool_dependencies = {}
    else:
        # Add the install_dir attribute to the tool_dependencies.
        repository_installed_tool_dependencies = add_installation_directories_to_tool_dependencies( trans=trans,
                                                                                                    tool_dependencies=repository_installed_tool_dependencies )
    if repository_missing_tool_dependencies is None:
        repository_missing_tool_dependencies = {}
    else:
        # Add the install_dir attribute to the tool_dependencies.
        repository_missing_tool_dependencies = add_installation_directories_to_tool_dependencies( trans=trans,
                                                                                                  tool_dependencies=repository_missing_tool_dependencies )
    if required_repo_info_dicts:
        # Handle the tool dependencies defined for each of the repository's repository dependencies.
        for rid in required_repo_info_dicts:
            for name, repo_info_tuple in rid.items():
                description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
                    suc.get_repo_info_tuple_contents( repo_info_tuple )
                if tool_dependencies:
                    # Add the install_dir attribute to the tool_dependencies.
                    tool_dependencies = add_installation_directories_to_tool_dependencies( trans=trans,
                                                                                           tool_dependencies=tool_dependencies )
                    # The required_repository may have been installed with a different changeset revision.
                    required_repository, installed_changeset_revision = suc.repository_was_previously_installed( trans, tool_shed_url, name, repo_info_tuple )
                    if required_repository:
                        required_repository_installed_tool_dependencies, required_repository_missing_tool_dependencies = \
                            get_installed_and_missing_tool_dependencies_for_installed_repository( trans, required_repository, tool_dependencies )
                        if required_repository_installed_tool_dependencies:
                            # Add the install_dir attribute to the tool_dependencies.
                            required_repository_installed_tool_dependencies = \
                                add_installation_directories_to_tool_dependencies( trans=trans,
                                                                                   tool_dependencies=required_repository_installed_tool_dependencies )
                            for td_key, td_dict in required_repository_installed_tool_dependencies.items():
                                if td_key not in repository_installed_tool_dependencies:
                                    repository_installed_tool_dependencies[ td_key ] = td_dict
                        if required_repository_missing_tool_dependencies:
                            # Add the install_dir attribute to the tool_dependencies.
                            required_repository_missing_tool_dependencies = \
                                add_installation_directories_to_tool_dependencies( trans=trans,
                                                                                   tool_dependencies=required_repository_missing_tool_dependencies )
                            for td_key, td_dict in required_repository_missing_tool_dependencies.items():
                                if td_key not in repository_missing_tool_dependencies:
                                    repository_missing_tool_dependencies[ td_key ] = td_dict
    if repository_installed_tool_dependencies:
        installed_tool_dependencies = repository_installed_tool_dependencies
    if repository_missing_tool_dependencies:
        missing_tool_dependencies = repository_missing_tool_dependencies
    return installed_tool_dependencies, missing_tool_dependencies
def get_components_from_repo_info_dict( trans, repo_info_dict ):
    """
    Return the repository and the associated latest installable changeset_revision (including updates) for the repository defined by the received
    repo_info_dict.
    """
    for repository_name, repo_info_tup in repo_info_dict.items():
        # There should only be one entry in the received repo_info_dict.
        description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
            suc.get_repo_info_tuple_contents( repo_info_tup )
        repository = suc.get_repository_by_name_and_owner( trans.app, repository_name, repository_owner )
        repository_metadata = suc.get_current_repository_metadata_for_changeset_revision( trans, repository, changeset_revision )
        if repository_metadata:
            return repository, repository_metadata.changeset_revision
    return None, None
Exemple #4
0
def get_readme_files_dict_for_display( trans, tool_shed_url, repo_info_dict ):
    """
    Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section 
    selection page.
    """
    name = repo_info_dict.keys()[ 0 ]
    repo_info_tuple = repo_info_dict[ name ]
    description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
        suc.get_repo_info_tuple_contents( repo_info_tuple )
    # Handle README files.
    url = suc.url_join( tool_shed_url,
                       'repository/get_readme_files?name=%s&owner=%s&changeset_revision=%s' % ( name, repository_owner, changeset_revision ) )
    raw_text = common_util.tool_shed_get( trans.app, tool_shed_url, url )
    readme_files_dict = json.from_json_string( raw_text )
    return readme_files_dict
Exemple #5
0
def get_installed_and_missing_repository_dependencies_for_new_install( trans, repo_info_tuple ):
    """
    Parse the received repository_dependencies dictionary that is associated with a repository being installed into Galaxy for the first time
    and attempt to determine repository dependencies that are already installed and those that are not.
    """
    missing_repository_dependencies = {}
    installed_repository_dependencies = {}
    missing_rd_tups = []
    installed_rd_tups = []
    description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
        suc.get_repo_info_tuple_contents( repo_info_tuple )
    if repository_dependencies:
        description = repository_dependencies[ 'description' ]
        root_key = repository_dependencies[ 'root_key' ]
        # The repository dependencies container will include only the immediate repository dependencies of this repository, so the container will be
        # only a single level in depth.
        for key, rd_tups in repository_dependencies.items():
            if key in [ 'description', 'root_key' ]:
                continue
            for rd_tup in rd_tups:
                tool_shed, name, owner, changeset_revision, prior_installation_required = suc.parse_repository_dependency_tuple( rd_tup )
                # Updates to installed repository revisions may have occurred, so make sure to locate the appropriate repository revision if one exists.
                # We need to create a temporary repo_info_tuple that includes the correct repository owner which we get from the current rd_tup.  The current
                # tuple looks like: ( description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td )
                tmp_clone_url = suc.generate_clone_url_from_repo_info_tup( rd_tup )
                tmp_repo_info_tuple = ( None, tmp_clone_url, changeset_revision, None, owner, None, None )
                repository, current_changeset_revision = suc.repository_was_previously_installed( trans, tool_shed, name, tmp_repo_info_tuple )
                if repository:
                    new_rd_tup = [ tool_shed, name, owner, changeset_revision, str( prior_installation_required ), repository.id, repository.status ]
                    if repository.status == trans.model.ToolShedRepository.installation_status.INSTALLED:
                        if new_rd_tup not in installed_rd_tups:
                            installed_rd_tups.append( new_rd_tup )
                    else:
                        if new_rd_tup not in missing_rd_tups:
                            missing_rd_tups.append( new_rd_tup )
                else:
                    new_rd_tup = [ tool_shed, name, owner, changeset_revision, str( prior_installation_required ), None, 'Never installed' ]
                    if new_rd_tup not in missing_rd_tups:
                        missing_rd_tups.append( new_rd_tup )
    if installed_rd_tups:
        installed_repository_dependencies[ 'root_key' ] = root_key
        installed_repository_dependencies[ root_key ] = installed_rd_tups
        installed_repository_dependencies[ 'description' ] = description
    if missing_rd_tups:
        missing_repository_dependencies[ 'root_key' ] = root_key
        missing_repository_dependencies[ root_key ] = missing_rd_tups
        missing_repository_dependencies[ 'description' ] = description
    return installed_repository_dependencies, missing_repository_dependencies
def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ):
    """
    Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section
    selection page.
    """
    name = repo_info_dict.keys()[ 0 ]
    repo_info_tuple = repo_info_dict[ name ]
    description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
        suc.get_repo_info_tuple_contents( repo_info_tuple )
    # Handle changing HTTP protocols over time.
    tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
    params = dict( name=name, owner=repository_owner, changeset_revision=changeset_revision )
    pathspec = [ 'repository', 'get_readme_files' ]
    raw_text = common_util.tool_shed_get( app, tool_shed_url, pathspec=pathspec, params=params )
    readme_files_dict = json.loads( raw_text )
    return readme_files_dict
def get_readme_files_dict_for_display( app, tool_shed_url, repo_info_dict ):
    """
    Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section
    selection page.
    """
    name = repo_info_dict.keys()[ 0 ]
    repo_info_tuple = repo_info_dict[ name ]
    description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
        suc.get_repo_info_tuple_contents( repo_info_tuple )
    # Handle changing HTTP protocols over time.
    tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( app, tool_shed_url )
    params = '?name=%s&owner=%s&changeset_revision=%s' % ( name, repository_owner, changeset_revision )
    url = common_util.url_join( tool_shed_url,
                                'repository/get_readme_files%s' % params )
    raw_text = common_util.tool_shed_get( app, tool_shed_url, url )
    readme_files_dict = json.from_json_string( raw_text )
    return readme_files_dict
Exemple #8
0
def get_readme_files_dict_for_display(app, tool_shed_url, repo_info_dict):
    """
    Return a dictionary of README files contained in the single repository being installed so they can be displayed on the tool panel section
    selection page.
    """
    name = repo_info_dict.keys()[0]
    repo_info_tuple = repo_info_dict[name]
    description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
        suc.get_repo_info_tuple_contents( repo_info_tuple )
    # Handle changing HTTP protocols over time.
    tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry(
        app, tool_shed_url)
    params = dict(name=name,
                  owner=repository_owner,
                  changeset_revision=changeset_revision)
    pathspec = ['repository', 'get_readme_files']
    raw_text = url_get(
        tool_shed_url,
        password_mgr=app.tool_shed_registry.url_auth(tool_shed_url),
        pathspec=pathspec,
        params=params)
    readme_files_dict = json.loads(raw_text)
    return readme_files_dict
def get_required_repo_info_dicts( trans, tool_shed_url, repo_info_dicts ):
    """
    Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of them to the list.  All
    repository_dependencies entries in each of the received repo_info_dicts includes all required repositories, so only one pass through
    this method is required to retrieve all repository dependencies.
    """
    all_required_repo_info_dict = {}
    all_repo_info_dicts = []
    if repo_info_dicts:
        # We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool shed to discover repository ids.
        required_repository_tups = []
        for repo_info_dict in repo_info_dicts:
            if repo_info_dict not in all_repo_info_dicts:
                all_repo_info_dicts.append( repo_info_dict )
            for repository_name, repo_info_tup in repo_info_dict.items():
                description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
                    suc.get_repo_info_tuple_contents( repo_info_tup )
                if repository_dependencies:
                    for key, val in repository_dependencies.items():
                        if key in [ 'root_key', 'description' ]:
                            continue
                        repository_components_tuple = container_util.get_components_from_key( key )
                        components_list = suc.extract_components_from_tuple( repository_components_tuple )
                        # Skip listing a repository dependency if it is required only to compile a tool dependency defined for the dependent repository since
                        # in this case, the repository dependency is really a dependency of the dependent repository's contained tool dependency, and only if
                        # that tool dependency requires compilation.
                        # For backward compatibility to the 12/20/12 Galaxy release.
                        prior_installation_required = 'False'
                        only_if_compiling_contained_td = 'False'
                        if len( components_list ) == 4:
                            prior_installation_required = 'False'
                            only_if_compiling_contained_td = 'False'
                        elif len( components_list ) == 5:
                            prior_installation_required = components_list[ 4 ]
                            only_if_compiling_contained_td = 'False'
                        if not util.asbool( only_if_compiling_contained_td ):
                            if components_list not in required_repository_tups:
                                required_repository_tups.append( components_list )
                        for components_list in val:
                            try:
                                only_if_compiling_contained_td = components_list[ 5 ]
                            except:
                                only_if_compiling_contained_td = 'False'
                            # Skip listing a repository dependency if it is required only to compile a tool dependency defined for the dependent repository
                            # (see above comment).
                            if not util.asbool( only_if_compiling_contained_td ):
                                if components_list not in required_repository_tups:
                                    required_repository_tups.append( components_list )
                else:
                    # We have a single repository with no dependencies.
                    components_list = [ tool_shed_url, repository_name, repository_owner, changeset_revision ]
                    required_repository_tups.append( components_list )
            if required_repository_tups:
                # The value of required_repository_tups is a list of tuples, so we need to encode it.
                encoded_required_repository_tups = []
                for required_repository_tup in required_repository_tups:
                    # Convert every item in required_repository_tup to a string.
                    required_repository_tup = [ str( item ) for item in required_repository_tup ]
                    encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) )
                encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups )
                encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str )
                url = suc.url_join( tool_shed_url, '/repository/get_required_repo_info_dict' )
                request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) )
                response = urllib2.urlopen( request ).read()
                if response:
                    try:
                        required_repo_info_dict = json.from_json_string( response )
                    except Exception, e:
                        log.exception( e )
                        return all_repo_info_dicts
                    required_repo_info_dicts = []
                    for k, v in required_repo_info_dict.items():
                        if k == 'repo_info_dicts':
                            encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ]
                            for encoded_dict_str in encoded_dict_strings:
                                decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str )
                                required_repo_info_dicts.append( decoded_dict )
                        else:
                            if k not in all_required_repo_info_dict:
                                all_required_repo_info_dict[ k ] = v
                            else:
                                if v and not all_required_repo_info_dict[ k ]:
                                    all_required_repo_info_dict[ k ] = v
                        if required_repo_info_dicts:
                            for required_repo_info_dict in required_repo_info_dicts:
                                if required_repo_info_dict not in all_repo_info_dicts:
                                    all_repo_info_dicts.append( required_repo_info_dict )
                    all_required_repo_info_dict[ 'all_repo_info_dicts' ] = all_repo_info_dicts
 def build_repository_dependency_relationships( self, repo_info_dicts, tool_shed_repositories ):
     """
     Build relationships between installed tool shed repositories and other installed
     tool shed repositories upon which they depend.  These relationships are defined in
     the repository_dependencies entry for each dictionary in the received list of repo_info_dicts.
     Each of these dictionaries is associated with a repository in the received tool_shed_repositories
     list.
     """
     install_model = self.app.install_model
     log.debug( "Building repository dependency relationships..." )
     for repo_info_dict in repo_info_dicts:
         for name, repo_info_tuple in repo_info_dict.items():
             description, \
                 repository_clone_url, \
                 changeset_revision, \
                 ctx_rev, \
                 repository_owner, \
                 repository_dependencies, \
                 tool_dependencies = \
                 suc.get_repo_info_tuple_contents( repo_info_tuple )
             if repository_dependencies:
                 for key, val in repository_dependencies.items():
                     if key in [ 'root_key', 'description' ]:
                         continue
                     d_repository = None
                     repository_components_tuple = container_util.get_components_from_key( key )
                     components_list = suc.extract_components_from_tuple( repository_components_tuple )
                     d_toolshed, d_name, d_owner, d_changeset_revision = components_list[ 0:4 ]
                     for tsr in tool_shed_repositories:
                         # Get the the tool_shed_repository defined by name, owner and changeset_revision.  This is
                         # the repository that will be dependent upon each of the tool shed repositories contained in
                         # val.  We'll need to check tool_shed_repository.tool_shed as well if/when repository dependencies
                         # across tool sheds is supported.
                         if tsr.name == d_name and tsr.owner == d_owner and tsr.changeset_revision == d_changeset_revision:
                             d_repository = tsr
                             break
                     if d_repository is None:
                         # The dependent repository is not in the received list so look in the database.
                         d_repository = self.get_or_create_tool_shed_repository( d_toolshed,
                                                                                 d_name,
                                                                                 d_owner,
                                                                                 d_changeset_revision )
                     # Process each repository_dependency defined for the current dependent repository.
                     for repository_dependency_components_list in val:
                         required_repository = None
                         rd_toolshed, \
                             rd_name, \
                             rd_owner, \
                             rd_changeset_revision, \
                             rd_prior_installation_required, \
                             rd_only_if_compiling_contained_td = \
                             common_util.parse_repository_dependency_tuple( repository_dependency_components_list )
                         # Get the the tool_shed_repository defined by rd_name, rd_owner and rd_changeset_revision.  This
                         # is the repository that will be required by the current d_repository.
                         # TODO: Check tool_shed_repository.tool_shed as well when repository dependencies across tool sheds is supported.
                         for tsr in tool_shed_repositories:
                             if tsr.name == rd_name and tsr.owner == rd_owner and tsr.changeset_revision == rd_changeset_revision:
                                 required_repository = tsr
                                 break
                         if required_repository is None:
                             # The required repository is not in the received list so look in the database.
                             required_repository = self.get_or_create_tool_shed_repository( rd_toolshed,
                                                                                            rd_name,
                                                                                            rd_owner,
                                                                                            rd_changeset_revision )
                         # Ensure there is a repository_dependency relationship between d_repository and required_repository.
                         rrda = None
                         for rd in d_repository.repository_dependencies:
                             if rd.id == required_repository.id:
                                 rrda = rd
                                 break
                         if not rrda:
                             # Make sure required_repository is in the repository_dependency table.
                             repository_dependency = self.get_repository_dependency_by_repository_id( install_model,
                                                                                                      required_repository.id )
                             if not repository_dependency:
                                 log.debug( 'Creating new repository_dependency record for installed revision %s of repository: %s owned by %s.' %
                                            ( str( required_repository.installed_changeset_revision ),
                                              str( required_repository.name ),
                                                str( required_repository.owner ) ) )
                                 repository_dependency = install_model.RepositoryDependency( tool_shed_repository_id=required_repository.id )
                                 install_model.context.add( repository_dependency )
                                 install_model.context.flush()
                             # Build the relationship between the d_repository and the required_repository.
                             rrda = install_model.RepositoryRepositoryDependencyAssociation( tool_shed_repository_id=d_repository.id,
                                                                                             repository_dependency_id=repository_dependency.id )
                             install_model.context.add( rrda )
                             install_model.context.flush()
Exemple #11
0
def get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies ):
    """
    Return dictionaries containing the sets of installed and missing tool dependencies and repository dependencies associated with the repository defined
    by the received repo_info_dict.
    """
    name = repo_info_dict.keys()[ 0 ]
    repo_info_tuple = repo_info_dict[ name ]
    description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td = \
        suc.get_repo_info_tuple_contents( repo_info_tuple )
    if repository_dependencies:
        # We have a repository with one or more defined repository dependencies.
        missing_td = {}
        # Handle the scenario where a repository was installed, then uninstalled and an error occurred during the re-installation process.
        # In this case, a record for the repository will exist in the database with the status of 'New'.
        repository = suc.get_repository_for_dependency_relationship( trans.app, tool_shed_url, name, repository_owner, changeset_revision )
        if repository and repository.metadata:
            installed_rd, missing_rd = get_installed_and_missing_repository_dependencies( trans, repository )
        else:
            installed_rd, missing_rd = get_installed_and_missing_repository_dependencies_for_new_install( trans, repo_info_tuple )
        # Discover all repository dependencies and retrieve information for installing them.
        all_repo_info_dict = get_required_repo_info_dicts( trans, tool_shed_url, util.listify( repo_info_dict ) )
        has_repository_dependencies = all_repo_info_dict.get( 'has_repository_dependencies', False )
        includes_tools_for_display_in_tool_panel = all_repo_info_dict.get( 'includes_tools_for_display_in_tool_panel', False )
        includes_tool_dependencies = all_repo_info_dict.get( 'includes_tool_dependencies', False )
        includes_tools = all_repo_info_dict.get( 'includes_tools', False )        
        required_repo_info_dicts = all_repo_info_dict.get( 'all_repo_info_dicts', [] )
        # Display tool dependencies defined for each of the repository dependencies.
        if required_repo_info_dicts:
            all_tool_dependencies = {}
            for rid in required_repo_info_dicts:
                for name, repo_info_tuple in rid.items():
                    description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, rid_installed_td = \
                        suc.get_repo_info_tuple_contents( repo_info_tuple )
                    if rid_installed_td:
                        for td_key, td_dict in rid_installed_td.items():
                            if td_key not in all_tool_dependencies:
                                all_tool_dependencies[ td_key ] = td_dict
            if all_tool_dependencies:
                if installed_td is None:
                    installed_td = {}
                else:
                    # Move all tool dependencies to the missing_tool_dependencies container.
                    for td_key, td_dict in installed_td.items():
                        if td_key not in missing_td:
                            missing_td[ td_key ] = td_dict
                    installed_td = {}
                # Discover and categorize all tool dependencies defined for this repository's repository dependencies.
                required_tool_dependencies, required_missing_tool_dependencies = \
                    get_installed_and_missing_tool_dependencies_for_new_install( trans, all_tool_dependencies )
                if required_tool_dependencies:
                    if not includes_tool_dependencies:
                        includes_tool_dependencies = True
                    for td_key, td_dict in required_tool_dependencies.items():
                        if td_key not in installed_td:
                            installed_td[ td_key ] = td_dict
                if required_missing_tool_dependencies:
                    if not includes_tool_dependencies:
                        includes_tool_dependencies = True
                    for td_key, td_dict in required_missing_tool_dependencies.items():
                        if td_key not in missing_td:
                            missing_td[ td_key ] = td_dict
    else:
        # We have a single repository with no defined repository dependencies.
        all_repo_info_dict = get_required_repo_info_dicts( trans, tool_shed_url, util.listify( repo_info_dict ) )
        has_repository_dependencies = all_repo_info_dict.get( 'has_repository_dependencies', False )
        includes_tools_for_display_in_tool_panel = all_repo_info_dict.get( 'includes_tools_for_display_in_tool_panel', False )
        includes_tool_dependencies = all_repo_info_dict.get( 'includes_tool_dependencies', False )
        includes_tools = all_repo_info_dict.get( 'includes_tools', False )        
        required_repo_info_dicts = all_repo_info_dict.get( 'all_repo_info_dicts', [] )
        installed_rd = None
        missing_rd = None
        missing_td = None
    dependencies_for_repository_dict = dict( changeset_revision=changeset_revision,
                                             has_repository_dependencies=has_repository_dependencies,
                                             includes_tool_dependencies=includes_tool_dependencies,
                                             includes_tools=includes_tools,
                                             includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel,
                                             installed_repository_dependencies=installed_rd,
                                             installed_tool_dependencies=installed_td,
                                             missing_repository_dependencies=missing_rd,
                                             missing_tool_dependencies=missing_td,
                                             name=name,
                                             repository_owner=repository_owner )
    return dependencies_for_repository_dict
def get_dependencies_for_repository( trans, tool_shed_url, repo_info_dict, includes_tool_dependencies ):
    """
    Return dictionaries containing the sets of installed and missing tool dependencies and repository dependencies associated
    with the repository defined by the received repo_info_dict.
    """
    repository = None
    installed_rd = {}
    installed_td = {}
    missing_rd = {}
    missing_td = {}
    name = repo_info_dict.keys()[ 0 ]
    repo_info_tuple = repo_info_dict[ name ]
    description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
        suc.get_repo_info_tuple_contents( repo_info_tuple )
    if tool_dependencies:
        if not includes_tool_dependencies:
            includes_tool_dependencies = True
        # Inspect the tool_dependencies dictionary to separate the installed and missing tool dependencies.  We don't add to installed_td
        # and missing_td here because at this point they are empty.
        installed_td, missing_td = get_installed_and_missing_tool_dependencies_for_installing_repository( trans, tool_shed_url, tool_dependencies )
    # In cases where a repository dependency is required only for compiling a dependent repository's tool dependency, the value of
    # repository_dependencies will be an empty dictionary here.
    if repository_dependencies:
        # We have a repository with one or more defined repository dependencies.
        if not repository:
            repository = suc.get_repository_for_dependency_relationship( trans.app, tool_shed_url, name, repository_owner, changeset_revision )
        if repository and repository.metadata:
            installed_rd, missing_rd = get_installed_and_missing_repository_dependencies( trans, repository )
        else:
            installed_rd, missing_rd = get_installed_and_missing_repository_dependencies_for_new_install( trans, repo_info_tuple )
        # Discover all repository dependencies and retrieve information for installing them.
        all_repo_info_dict = get_required_repo_info_dicts( trans, tool_shed_url, util.listify( repo_info_dict ) )
        has_repository_dependencies = all_repo_info_dict.get( 'has_repository_dependencies', False )
        has_repository_dependencies_only_if_compiling_contained_td = \
            all_repo_info_dict.get( 'has_repository_dependencies_only_if_compiling_contained_td', False )
        includes_tools_for_display_in_tool_panel = all_repo_info_dict.get( 'includes_tools_for_display_in_tool_panel', False )
        includes_tool_dependencies = all_repo_info_dict.get( 'includes_tool_dependencies', False )
        includes_tools = all_repo_info_dict.get( 'includes_tools', False )
        required_repo_info_dicts = all_repo_info_dict.get( 'all_repo_info_dicts', [] )
        # Display tool dependencies defined for each of the repository dependencies.
        if required_repo_info_dicts:
            required_tool_dependencies = {}
            for rid in required_repo_info_dicts:
                for name, repo_info_tuple in rid.items():
                    description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, rid_repository_dependencies, rid_tool_dependencies = \
                        suc.get_repo_info_tuple_contents( repo_info_tuple )
                    if rid_tool_dependencies:
                        for td_key, td_dict in rid_tool_dependencies.items():
                            if td_key not in required_tool_dependencies:
                                required_tool_dependencies[ td_key ] = td_dict
            if required_tool_dependencies:
                # Discover and categorize all tool dependencies defined for this repository's repository dependencies.
                required_installed_td, required_missing_td = \
                    get_installed_and_missing_tool_dependencies_for_installing_repository( trans, tool_shed_url, required_tool_dependencies )
                if required_installed_td:
                    if not includes_tool_dependencies:
                        includes_tool_dependencies = True
                    for td_key, td_dict in required_installed_td.items():
                        if td_key not in installed_td:
                            installed_td[ td_key ] = td_dict
                if required_missing_td:
                    if not includes_tool_dependencies:
                        includes_tool_dependencies = True
                    for td_key, td_dict in required_missing_td.items():
                        if td_key not in missing_td:
                            missing_td[ td_key ] = td_dict
    else:
        # We have a single repository with (possibly) no defined repository dependencies.
        all_repo_info_dict = get_required_repo_info_dicts( trans, tool_shed_url, util.listify( repo_info_dict ) )
        has_repository_dependencies = all_repo_info_dict.get( 'has_repository_dependencies', False )
        has_repository_dependencies_only_if_compiling_contained_td = \
            all_repo_info_dict.get( 'has_repository_dependencies_only_if_compiling_contained_td', False )
        includes_tools_for_display_in_tool_panel = all_repo_info_dict.get( 'includes_tools_for_display_in_tool_panel', False )
        includes_tool_dependencies = all_repo_info_dict.get( 'includes_tool_dependencies', False )
        includes_tools = all_repo_info_dict.get( 'includes_tools', False )
        required_repo_info_dicts = all_repo_info_dict.get( 'all_repo_info_dicts', [] )
    dependencies_for_repository_dict = \
        dict( changeset_revision=changeset_revision,
              has_repository_dependencies=has_repository_dependencies,
              has_repository_dependencies_only_if_compiling_contained_td=has_repository_dependencies_only_if_compiling_contained_td,
              includes_tool_dependencies=includes_tool_dependencies,
              includes_tools=includes_tools,
              includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel,
              installed_repository_dependencies=installed_rd,
              installed_tool_dependencies=installed_td,
              missing_repository_dependencies=missing_rd,
              missing_tool_dependencies=missing_td,
              name=name,
              repository_owner=repository_owner )
    return dependencies_for_repository_dict
Exemple #13
0
def get_required_repo_info_dicts( trans, tool_shed_url, repo_info_dicts ):
    """
    Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of them to the list.  All
    repository_dependencies entries in each of the received repo_info_dicts includes all required repositories, so only one pass through
    this method is required to retrieve all repository dependencies.
    """
    all_required_repo_info_dict = {}
    all_repo_info_dicts = []
    if repo_info_dicts:
        # We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool shed to discover repository ids.
        required_repository_tups = []
        for repo_info_dict in repo_info_dicts:
            if repo_info_dict not in all_repo_info_dicts:
                all_repo_info_dicts.append( repo_info_dict )
            for repository_name, repo_info_tup in repo_info_dict.items():
                description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
                    suc.get_repo_info_tuple_contents( repo_info_tup )
                if repository_dependencies:
                    for key, val in repository_dependencies.items():
                        if key in [ 'root_key', 'description' ]:
                            continue
                        try:
                            toolshed, name, owner, changeset_revision, prior_installation_required = container_util.get_components_from_key( key )
                            components_list = [ toolshed, name, owner, changeset_revision, prior_installation_required ]
                        except ValueError:
                            # For backward compatibility to the 12/20/12 Galaxy release, default prior_installation_required to False in the caller.
                            toolshed, name, owner, changeset_revision = container_util.get_components_from_key( key )
                            components_list = [ toolshed, name, owner, changeset_revision ]
                        if components_list not in required_repository_tups:
                            required_repository_tups.append( components_list )
                        for components_list in val:
                            if components_list not in required_repository_tups:
                                required_repository_tups.append( components_list )
                else:
                    # We have a single repository with no dependencies.
                    components_list = [ tool_shed_url, repository_name, repository_owner, changeset_revision, 'False' ]
                    required_repository_tups.append( components_list )
            if required_repository_tups:
                # The value of required_repository_tups is a list of tuples, so we need to encode it.
                encoded_required_repository_tups = []
                for required_repository_tup in required_repository_tups:
                    # Convert every item in required_repository_tup to a string.
                    required_repository_tup = [ str( item ) for item in required_repository_tup ]
                    encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) )
                encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups )
                encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str )
                url = suc.url_join( tool_shed_url, '/repository/get_required_repo_info_dict' )
                request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) )
                response = urllib2.urlopen( request ).read()
                if response:
                    try:
                        required_repo_info_dict = json.from_json_string( response )
                    except Exception, e:
                        log.exception( e )
                        return all_repo_info_dicts
                    required_repo_info_dicts = []
                    for k, v in required_repo_info_dict.items():
                        if k == 'repo_info_dicts':
                            encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ]
                            for encoded_dict_str in encoded_dict_strings:
                                decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str )
                                required_repo_info_dicts.append( decoded_dict )
                        else:
                            if k not in all_required_repo_info_dict:
                                all_required_repo_info_dict[ k ] = v
                            else:
                                if v and not all_required_repo_info_dict[ k ]:
                                    all_required_repo_info_dict[ k ] = v
                        if required_repo_info_dicts:                            
                            for required_repo_info_dict in required_repo_info_dicts:
                                if required_repo_info_dict not in all_repo_info_dicts:
                                    all_repo_info_dicts.append( required_repo_info_dict )
                    all_required_repo_info_dict[ 'all_repo_info_dicts' ] = all_repo_info_dicts
def get_repository_ids(trans, repo_info_dicts):
    """Return a list of repository ids associated with each dictionary in the received repo_info_dicts."""
    repository_ids = []
    for repo_info_dict in repo_info_dicts:
        for repository_name, repo_info_tup in repo_info_dict.items():
            description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = suc.get_repo_info_tuple_contents(
                repo_info_tup
            )
            repository = suc.get_repository_by_name_and_owner(trans.app, repository_name, repository_owner)
            repository_ids.append(trans.security.encode_id(repository.id))
    return repository_ids
def get_installed_and_missing_repository_dependencies_for_new_install( trans, repo_info_tuple ):
    """
    Parse the received repository_dependencies dictionary that is associated with a repository being installed into Galaxy for the first time
    and attempt to determine repository dependencies that are already installed and those that are not.
    """
    missing_repository_dependencies = {}
    installed_repository_dependencies = {}
    missing_rd_tups = []
    installed_rd_tups = []
    description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = \
        suc.get_repo_info_tuple_contents( repo_info_tuple )
    if repository_dependencies:
        description = repository_dependencies[ 'description' ]
        root_key = repository_dependencies[ 'root_key' ]
        # The repository dependencies container will include only the immediate repository dependencies of this repository, so the container will be
        # only a single level in depth.
        for key, rd_tups in repository_dependencies.items():
            if key in [ 'description', 'root_key' ]:
                continue
            for rd_tup in rd_tups:
                tool_shed, name, owner, changeset_revision, prior_installation_required, only_if_compiling_contained_td = \
                    common_util.parse_repository_dependency_tuple( rd_tup )
                # Updates to installed repository revisions may have occurred, so make sure to locate the appropriate repository revision if one exists.
                # We need to create a temporary repo_info_tuple that includes the correct repository owner which we get from the current rd_tup.  The current
                # tuple looks like: ( description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, installed_td )
                tmp_clone_url = suc.generate_clone_url_from_repo_info_tup( rd_tup )
                tmp_repo_info_tuple = ( None, tmp_clone_url, changeset_revision, None, owner, None, None )
                repository, installed_changeset_revision = suc.repository_was_previously_installed( trans, tool_shed, name, tmp_repo_info_tuple )
                if repository:
                    new_rd_tup = [ tool_shed,
                                  name,
                                  owner,
                                  changeset_revision,
                                  prior_installation_required,
                                  only_if_compiling_contained_td,
                                  repository.id,
                                  repository.status ]
                    if repository.status == trans.install_model.ToolShedRepository.installation_status.INSTALLED:
                        if new_rd_tup not in installed_rd_tups:
                            installed_rd_tups.append( new_rd_tup )
                    else:
                        # A repository dependency that is not installed will not be considered missing if it's value for only_if_compiling_contained_td is
                        # True  This is because this type of repository dependency will only be considered at the time that the specified tool dependency
                        # is being installed, and even then only if the compiled binary of the tool dependency could not be installed due to the unsupported
                        # installation environment.
                        if not util.asbool( only_if_compiling_contained_td ):
                            if new_rd_tup not in missing_rd_tups:
                                missing_rd_tups.append( new_rd_tup )
                else:
                    new_rd_tup = [ tool_shed,
                                  name,
                                  owner,
                                  changeset_revision,
                                  prior_installation_required,
                                  only_if_compiling_contained_td,
                                  None,
                                  'Never installed' ]
                    if not util.asbool( only_if_compiling_contained_td ):
                        # A repository dependency that is not installed will not be considered missing if it's value for only_if_compiling_contained_td is
                        # True - see above...
                        if new_rd_tup not in missing_rd_tups:
                            missing_rd_tups.append( new_rd_tup )
    if installed_rd_tups:
        installed_repository_dependencies[ 'root_key' ] = root_key
        installed_repository_dependencies[ root_key ] = installed_rd_tups
        installed_repository_dependencies[ 'description' ] = description
    if missing_rd_tups:
        missing_repository_dependencies[ 'root_key' ] = root_key
        missing_repository_dependencies[ root_key ] = missing_rd_tups
        missing_repository_dependencies[ 'description' ] = description
    return installed_repository_dependencies, missing_repository_dependencies
 def build_repository_dependency_relationships( self, repo_info_dicts, tool_shed_repositories ):
     """
     Build relationships between installed tool shed repositories and other installed
     tool shed repositories upon which they depend.  These relationships are defined in
     the repository_dependencies entry for each dictionary in the received list of repo_info_dicts.
     Each of these dictionaries is associated with a repository in the received tool_shed_repositories
     list.
     """
     install_model = self.app.install_model
     log.debug( "Building repository dependency relationships..." )
     for repo_info_dict in repo_info_dicts:
         for name, repo_info_tuple in repo_info_dict.items():
             description, \
             repository_clone_url, \
             changeset_revision, \
             ctx_rev, \
             repository_owner, \
             repository_dependencies, \
             tool_dependencies = \
             suc.get_repo_info_tuple_contents( repo_info_tuple )
             if repository_dependencies:
                 for key, val in repository_dependencies.items():
                     if key in [ 'root_key', 'description' ]:
                         continue
                     d_repository = None
                     repository_components_tuple = container_util.get_components_from_key( key )
                     components_list = suc.extract_components_from_tuple( repository_components_tuple )
                     d_toolshed, d_name, d_owner, d_changeset_revision = components_list[ 0:4 ]
                     for tsr in tool_shed_repositories:
                         # Get the the tool_shed_repository defined by name, owner and changeset_revision.  This is
                         # the repository that will be dependent upon each of the tool shed repositories contained in
                         # val.  We'll need to check tool_shed_repository.tool_shed as well if/when repository dependencies
                         # across tool sheds is supported.
                         if tsr.name == d_name and tsr.owner == d_owner and tsr.changeset_revision == d_changeset_revision:
                             d_repository = tsr
                             break
                     if d_repository is None:
                         # The dependent repository is not in the received list so look in the database.
                         d_repository = self.get_or_create_tool_shed_repository( d_toolshed,
                                                                                 d_name,
                                                                                 d_owner,
                                                                                 d_changeset_revision )
                     # Process each repository_dependency defined for the current dependent repository.
                     for repository_dependency_components_list in val:
                         required_repository = None
                         rd_toolshed, \
                         rd_name, \
                         rd_owner, \
                         rd_changeset_revision, \
                         rd_prior_installation_required, \
                         rd_only_if_compiling_contained_td = \
                         common_util.parse_repository_dependency_tuple( repository_dependency_components_list )
                         # Get the the tool_shed_repository defined by rd_name, rd_owner and rd_changeset_revision.  This
                         # is the repository that will be required by the current d_repository.
                         # TODO: Check tool_shed_repository.tool_shed as well when repository dependencies across tool sheds is supported.
                         for tsr in tool_shed_repositories:
                             if tsr.name == rd_name and tsr.owner == rd_owner and tsr.changeset_revision == rd_changeset_revision:
                                 required_repository = tsr
                                 break
                         if required_repository is None:
                             # The required repository is not in the received list so look in the database.
                             required_repository = self.get_or_create_tool_shed_repository( rd_toolshed,
                                                                                            rd_name,
                                                                                            rd_owner,
                                                                                            rd_changeset_revision )
                         # Ensure there is a repository_dependency relationship between d_repository and required_repository.
                         rrda = None
                         for rd in d_repository.repository_dependencies:
                             if rd.id == required_repository.id:
                                 rrda = rd
                                 break
                         if not rrda:
                             # Make sure required_repository is in the repository_dependency table.
                             repository_dependency = self.get_repository_dependency_by_repository_id( install_model,
                                                                                                      required_repository.id )
                             if not repository_dependency:
                                 log.debug( 'Creating new repository_dependency record for installed revision %s of repository: %s owned by %s.' % \
                                     ( str( required_repository.installed_changeset_revision ),
                                       str( required_repository.name ),
                                       str( required_repository.owner ) ) )
                                 repository_dependency = install_model.RepositoryDependency( tool_shed_repository_id=required_repository.id )
                                 install_model.context.add( repository_dependency )
                                 install_model.context.flush()
                             # Build the relationship between the d_repository and the required_repository.
                             rrda = install_model.RepositoryRepositoryDependencyAssociation( tool_shed_repository_id=d_repository.id,
                                                                                             repository_dependency_id=repository_dependency.id )
                             install_model.context.add( rrda )
                             install_model.context.flush()
 def create_repository_dependency_objects( self, tool_path, tool_shed_url, repo_info_dicts, install_repository_dependencies=False,
                                           no_changes_checked=False, tool_panel_section_id=None, new_tool_panel_section_label=None ):
     """
     Discover all repository dependencies and make sure all tool_shed_repository and
     associated repository_dependency records exist as well as the dependency relationships
     between installed repositories.  This method is called when uninstalled repositories
     are being reinstalled.  If the user elected to install repository dependencies, all
     items in the all_repo_info_dicts list will be processed.  However, if repository
     dependencies are not to be installed, only those items contained in the received
     repo_info_dicts list will be processed.
     """
     install_model = self.app.install_model
     log.debug( "Creating repository dependency objects..." )
     # The following list will be maintained within this method to contain all created
     # or updated tool shed repositories, including repository dependencies that may not
     # be installed.
     all_created_or_updated_tool_shed_repositories = []
     # There will be a one-to-one mapping between items in 3 lists:
     # created_or_updated_tool_shed_repositories, tool_panel_section_keys
     # and filtered_repo_info_dicts.  The 3 lists will filter out repository
     # dependencies that are not to be installed.
     created_or_updated_tool_shed_repositories = []
     tool_panel_section_keys = []
     # Repositories will be filtered (e.g., if already installed, if elected
     # to not be installed, etc), so filter the associated repo_info_dicts accordingly.
     filtered_repo_info_dicts = []
     # Discover all repository dependencies and retrieve information for installing
     # them.  Even if the user elected to not install repository dependencies we have
     # to make sure all repository dependency objects exist so that the appropriate
     # repository dependency relationships can be built.
     all_required_repo_info_dict = self.get_required_repo_info_dicts( tool_shed_url, repo_info_dicts )
     all_repo_info_dicts = all_required_repo_info_dict.get( 'all_repo_info_dicts', [] )
     if not all_repo_info_dicts:
         # No repository dependencies were discovered so process the received repositories.
         all_repo_info_dicts = [ rid for rid in repo_info_dicts ]
     for repo_info_dict in all_repo_info_dicts:
         # If the user elected to install repository dependencies, all items in the
         # all_repo_info_dicts list will be processed.  However, if repository dependencies
         # are not to be installed, only those items contained in the received repo_info_dicts
         # list will be processed but the all_repo_info_dicts list will be used to create all
         # defined repository dependency relationships.
         if self.is_in_repo_info_dicts( repo_info_dict, repo_info_dicts ) or install_repository_dependencies:
             for name, repo_info_tuple in repo_info_dict.items():
                 can_update_db_record = False
                 description, \
                     repository_clone_url, \
                     changeset_revision, \
                     ctx_rev, \
                     repository_owner, \
                     repository_dependencies, \
                     tool_dependencies = \
                     suc.get_repo_info_tuple_contents( repo_info_tuple )
                 # See if the repository has an existing record in the database.
                 repository_db_record, installed_changeset_revision = \
                     suc.repository_was_previously_installed( self.app, tool_shed_url, name, repo_info_tuple, from_tip=False )
                 if repository_db_record:
                     if repository_db_record.status in [ install_model.ToolShedRepository.installation_status.INSTALLED,
                                                         install_model.ToolShedRepository.installation_status.CLONING,
                                                         install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS,
                                                         install_model.ToolShedRepository.installation_status.INSTALLING_REPOSITORY_DEPENDENCIES,
                                                         install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES,
                                                         install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ]:
                         debug_msg = "Skipping installation of revision %s of repository '%s' because it was installed " % \
                             ( str( changeset_revision ), str( repository_db_record.name ) )
                         debug_msg += "with the (possibly updated) revision %s and its current installation status is '%s'." % \
                             ( str( installed_changeset_revision ), str( repository_db_record.status ) )
                         log.debug( debug_msg )
                         can_update_db_record = False
                     else:
                         if repository_db_record.status in [ install_model.ToolShedRepository.installation_status.ERROR,
                                                             install_model.ToolShedRepository.installation_status.NEW,
                                                             install_model.ToolShedRepository.installation_status.UNINSTALLED ]:
                             # The current tool shed repository is not currently installed, so we can update its
                             # record in the database.
                             name = repository_db_record.name
                             installed_changeset_revision = repository_db_record.installed_changeset_revision
                             can_update_db_record = True
                         elif repository_db_record.status in [ install_model.ToolShedRepository.installation_status.DEACTIVATED ]:
                             # The current tool shed repository is deactivated, so updating its database record
                             # is not necessary - just activate it.
                             log.debug( "Reactivating deactivated tool_shed_repository '%s'." % str( repository_db_record.name ) )
                             self.app.installed_repository_manager.activate_repository( repository_db_record )
                             # No additional updates to the database record are necessary.
                             can_update_db_record = False
                         elif repository_db_record.status not in [ install_model.ToolShedRepository.installation_status.NEW ]:
                             # Set changeset_revision here so suc.create_or_update_tool_shed_repository will find
                             # the previously installed and uninstalled repository instead of creating a new record.
                             changeset_revision = repository_db_record.installed_changeset_revision
                             self.reset_previously_installed_repository( repository_db_record )
                             can_update_db_record = True
                 else:
                     # No record exists in the database for the repository currently being processed.
                     installed_changeset_revision = changeset_revision
                     can_update_db_record = True
                 if can_update_db_record:
                     # The database record for the tool shed repository currently being processed can be updated.
                     # Get the repository metadata to see where it was previously located in the tool panel.
                     tpm = tool_panel_manager.ToolPanelManager( self.app )
                     if repository_db_record and repository_db_record.metadata:
                         _, tool_panel_section_key = \
                             tpm.handle_tool_panel_selection( toolbox=self.app.toolbox,
                                                              metadata=repository_db_record.metadata,
                                                              no_changes_checked=no_changes_checked,
                                                              tool_panel_section_id=tool_panel_section_id,
                                                              new_tool_panel_section_label=new_tool_panel_section_label )
                     else:
                         # We're installing a new tool shed repository that does not yet have a database record.
                         tool_panel_section_key, _ = \
                             tpm.handle_tool_panel_section( self.app.toolbox,
                                                            tool_panel_section_id=tool_panel_section_id,
                                                            new_tool_panel_section_label=new_tool_panel_section_label )
                     tool_shed_repository = \
                         suc.create_or_update_tool_shed_repository( app=self.app,
                                                                    name=name,
                                                                    description=description,
                                                                    installed_changeset_revision=installed_changeset_revision,
                                                                    ctx_rev=ctx_rev,
                                                                    repository_clone_url=repository_clone_url,
                                                                    metadata_dict={},
                                                                    status=install_model.ToolShedRepository.installation_status.NEW,
                                                                    current_changeset_revision=changeset_revision,
                                                                    owner=repository_owner,
                                                                    dist_to_shed=False )
                     if tool_shed_repository not in all_created_or_updated_tool_shed_repositories:
                         all_created_or_updated_tool_shed_repositories.append( tool_shed_repository )
                     # Only append the tool shed repository to the list of created_or_updated_tool_shed_repositories if
                     # it is supposed to be installed.
                     if install_repository_dependencies or self.is_in_repo_info_dicts( repo_info_dict, repo_info_dicts ):
                         if tool_shed_repository not in created_or_updated_tool_shed_repositories:
                             # Keep the one-to-one mapping between items in 3 lists.
                             created_or_updated_tool_shed_repositories.append( tool_shed_repository )
                             tool_panel_section_keys.append( tool_panel_section_key )
                             filtered_repo_info_dicts.append( repo_info_dict )
     # Build repository dependency relationships even if the user chose to not install repository dependencies.
     self.build_repository_dependency_relationships( all_repo_info_dicts, all_created_or_updated_tool_shed_repositories )
     return created_or_updated_tool_shed_repositories, tool_panel_section_keys, all_repo_info_dicts, filtered_repo_info_dicts
 def create_repository_dependency_objects( self, tool_path, tool_shed_url, repo_info_dicts, install_repository_dependencies=False,
                                           no_changes_checked=False, tool_panel_section_id=None, new_tool_panel_section_label=None ):
     """
     Discover all repository dependencies and make sure all tool_shed_repository and
     associated repository_dependency records exist as well as the dependency relationships
     between installed repositories.  This method is called when uninstalled repositories
     are being reinstalled.  If the user elected to install repository dependencies, all
     items in the all_repo_info_dicts list will be processed.  However, if repository
     dependencies are not to be installed, only those items contained in the received
     repo_info_dicts list will be processed.
     """
     install_model = self.app.install_model
     log.debug( "Creating repository dependency objects..." )
     # The following list will be maintained within this method to contain all created
     # or updated tool shed repositories, including repository dependencies that may not
     # be installed.
     all_created_or_updated_tool_shed_repositories = []
     # There will be a one-to-one mapping between items in 3 lists:
     # created_or_updated_tool_shed_repositories, tool_panel_section_keys
     # and filtered_repo_info_dicts.  The 3 lists will filter out repository
     # dependencies that are not to be installed.
     created_or_updated_tool_shed_repositories = []
     tool_panel_section_keys = []
     # Repositories will be filtered (e.g., if already installed, if elected
     # to not be installed, etc), so filter the associated repo_info_dicts accordingly.
     filtered_repo_info_dicts = []
     # Discover all repository dependencies and retrieve information for installing
     # them.  Even if the user elected to not install repository dependencies we have
     # to make sure all repository dependency objects exist so that the appropriate
     # repository dependency relationships can be built.
     all_required_repo_info_dict = self.get_required_repo_info_dicts( tool_shed_url, repo_info_dicts )
     all_repo_info_dicts = all_required_repo_info_dict.get( 'all_repo_info_dicts', [] )
     if not all_repo_info_dicts:
         # No repository dependencies were discovered so process the received repositories.
         all_repo_info_dicts = [ rid for rid in repo_info_dicts ]
     for repo_info_dict in all_repo_info_dicts:
         # If the user elected to install repository dependencies, all items in the
         # all_repo_info_dicts list will be processed.  However, if repository dependencies
         # are not to be installed, only those items contained in the received repo_info_dicts
         # list will be processed but the all_repo_info_dicts list will be used to create all
         # defined repository dependency relationships.
         if self.is_in_repo_info_dicts( repo_info_dict, repo_info_dicts ) or install_repository_dependencies:
             for name, repo_info_tuple in repo_info_dict.items():
                 can_update_db_record = False
                 description, \
                 repository_clone_url, \
                 changeset_revision, \
                 ctx_rev, \
                 repository_owner, \
                 repository_dependencies, \
                 tool_dependencies = \
                 suc.get_repo_info_tuple_contents( repo_info_tuple )
                 # See if the repository has an existing record in the database.
                 repository_db_record, installed_changeset_revision = \
                     suc.repository_was_previously_installed( self.app, tool_shed_url, name, repo_info_tuple, from_tip=False )
                 if repository_db_record:
                     if repository_db_record.status in [ install_model.ToolShedRepository.installation_status.INSTALLED,
                                                         install_model.ToolShedRepository.installation_status.CLONING,
                                                         install_model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS,
                                                         install_model.ToolShedRepository.installation_status.INSTALLING_REPOSITORY_DEPENDENCIES,
                                                         install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES,
                                                         install_model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES ]:
                         debug_msg = "Skipping installation of revision %s of repository '%s' because it was installed " % \
                             ( str( changeset_revision ), str( repository_db_record.name ) )
                         debug_msg += "with the (possibly updated) revision %s and its current installation status is '%s'." % \
                             ( str( installed_changeset_revision ), str( repository_db_record.status ) )
                         log.debug( debug_msg )
                         can_update_db_record = False
                     else:
                         if repository_db_record.status in [ install_model.ToolShedRepository.installation_status.ERROR,
                                                             install_model.ToolShedRepository.installation_status.NEW,
                                                             install_model.ToolShedRepository.installation_status.UNINSTALLED ]:
                             # The current tool shed repository is not currently installed, so we can update its
                             # record in the database.
                             name = repository_db_record.name
                             installed_changeset_revision = repository_db_record.installed_changeset_revision
                             metadata_dict = repository_db_record.metadata
                             dist_to_shed = repository_db_record.dist_to_shed
                             can_update_db_record = True
                         elif repository_db_record.status in [ install_model.ToolShedRepository.installation_status.DEACTIVATED ]:
                             # The current tool shed repository is deactivated, so updating its database record
                             # is not necessary - just activate it.
                             log.debug( "Reactivating deactivated tool_shed_repository '%s'." % str( repository_db_record.name ) )
                             self.app.installed_repository_manager.activate_repository( repository_db_record )
                             # No additional updates to the database record are necessary.
                             can_update_db_record = False
                         elif repository_db_record.status not in [ install_model.ToolShedRepository.installation_status.NEW ]:
                             # Set changeset_revision here so suc.create_or_update_tool_shed_repository will find
                             # the previously installed and uninstalled repository instead of creating a new record.
                             changeset_revision = repository_db_record.installed_changeset_revision
                             self.reset_previously_installed_repository( repository_db_record )
                             can_update_db_record = True
                 else:
                     # No record exists in the database for the repository currently being processed.
                     installed_changeset_revision = changeset_revision
                     metadata_dict = {}
                     dist_to_shed = False
                     can_update_db_record = True
                 if can_update_db_record:
                     # The database record for the tool shed repository currently being processed can be updated.
                     # Get the repository metadata to see where it was previously located in the tool panel.
                     tpm = tool_panel_manager.ToolPanelManager( self.app )
                     if repository_db_record and repository_db_record.metadata:
                         tool_section, tool_panel_section_key = \
                             tpm.handle_tool_panel_selection( toolbox=self.app.toolbox,
                                                              metadata=repository_db_record.metadata,
                                                              no_changes_checked=no_changes_checked,
                                                              tool_panel_section_id=tool_panel_section_id,
                                                              new_tool_panel_section_label=new_tool_panel_section_label )
                     else:
                         # We're installing a new tool shed repository that does not yet have a database record.
                         tool_panel_section_key, tool_section = \
                             tpm.handle_tool_panel_section( self.app.toolbox,
                                                            tool_panel_section_id=tool_panel_section_id,
                                                            new_tool_panel_section_label=new_tool_panel_section_label )
                     tool_shed_repository = \
                         suc.create_or_update_tool_shed_repository( app=self.app,
                                                                    name=name,
                                                                    description=description,
                                                                    installed_changeset_revision=installed_changeset_revision,
                                                                    ctx_rev=ctx_rev,
                                                                    repository_clone_url=repository_clone_url,
                                                                    metadata_dict={},
                                                                    status=install_model.ToolShedRepository.installation_status.NEW,
                                                                    current_changeset_revision=changeset_revision,
                                                                    owner=repository_owner,
                                                                    dist_to_shed=False )
                     if tool_shed_repository not in all_created_or_updated_tool_shed_repositories:
                         all_created_or_updated_tool_shed_repositories.append( tool_shed_repository )
                     # Only append the tool shed repository to the list of created_or_updated_tool_shed_repositories if
                     # it is supposed to be installed.
                     if install_repository_dependencies or self.is_in_repo_info_dicts( repo_info_dict, repo_info_dicts ):
                         if tool_shed_repository not in created_or_updated_tool_shed_repositories:
                             # Keep the one-to-one mapping between items in 3 lists.
                             created_or_updated_tool_shed_repositories.append( tool_shed_repository )
                             tool_panel_section_keys.append( tool_panel_section_key )
                             filtered_repo_info_dicts.append( repo_info_dict )
     # Build repository dependency relationships even if the user chose to not install repository dependencies.
     self.build_repository_dependency_relationships( all_repo_info_dicts, all_created_or_updated_tool_shed_repositories )
     return created_or_updated_tool_shed_repositories, tool_panel_section_keys, all_repo_info_dicts, filtered_repo_info_dicts
 def get_required_repo_info_dicts( self, tool_shed_url, repo_info_dicts ):
     """
     Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of
     them to the list.  All repository_dependency entries in each of the received repo_info_dicts includes
     all required repositories, so only one pass through this method is required to retrieve all repository
     dependencies.
     """
     all_required_repo_info_dict = {}
     all_repo_info_dicts = []
     if repo_info_dicts:
         # We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool
         # shed to discover repository ids.
         required_repository_tups = []
         for repo_info_dict in repo_info_dicts:
             if repo_info_dict not in all_repo_info_dicts:
                 all_repo_info_dicts.append( repo_info_dict )
             for repository_name, repo_info_tup in repo_info_dict.items():
                 description, \
                     repository_clone_url, \
                     changeset_revision, \
                     ctx_rev, \
                     repository_owner, \
                     repository_dependencies, \
                     tool_dependencies = \
                     suc.get_repo_info_tuple_contents( repo_info_tup )
                 if repository_dependencies:
                     for key, val in repository_dependencies.items():
                         if key in [ 'root_key', 'description' ]:
                             continue
                         repository_components_tuple = container_util.get_components_from_key( key )
                         components_list = suc.extract_components_from_tuple( repository_components_tuple )
                         # Skip listing a repository dependency if it is required only to compile a tool dependency
                         # defined for the dependent repository since in this case, the repository dependency is really
                         # a dependency of the dependent repository's contained tool dependency, and only if that
                         # tool dependency requires compilation.
                         # For backward compatibility to the 12/20/12 Galaxy release.
                         only_if_compiling_contained_td = 'False'
                         if len( components_list ) == 4:
                             only_if_compiling_contained_td = 'False'
                         elif len( components_list ) == 5:
                             only_if_compiling_contained_td = 'False'
                         if not asbool( only_if_compiling_contained_td ):
                             if components_list not in required_repository_tups:
                                 required_repository_tups.append( components_list )
                         for components_list in val:
                             try:
                                 only_if_compiling_contained_td = components_list[ 5 ]
                             except:
                                 only_if_compiling_contained_td = 'False'
                             # Skip listing a repository dependency if it is required only to compile a tool dependency
                             # defined for the dependent repository (see above comment).
                             if not asbool( only_if_compiling_contained_td ):
                                 if components_list not in required_repository_tups:
                                     required_repository_tups.append( components_list )
                 else:
                     # We have a single repository with no dependencies.
                     components_list = [ tool_shed_url, repository_name, repository_owner, changeset_revision ]
                     required_repository_tups.append( components_list )
             if required_repository_tups:
                 # The value of required_repository_tups is a list of tuples, so we need to encode it.
                 encoded_required_repository_tups = []
                 for required_repository_tup in required_repository_tups:
                     # Convert every item in required_repository_tup to a string.
                     required_repository_tup = [ str( item ) for item in required_repository_tup ]
                     encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) )
                 encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups )
                 encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str )
                 if suc.is_tool_shed_client( self.app ):
                     # Handle secure / insecure Tool Shed URL protocol changes and port changes.
                     tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed_url )
                 pathspec = [ 'repository', 'get_required_repo_info_dict' ]
                 url = common_util.url_join( tool_shed_url, pathspec=pathspec )
                 # Fix for handling 307 redirect not being handled nicely by urllib2.urlopen when the urllib2.Request has data provided
                 url = urllib2.urlopen( urllib2.Request( url ) ).geturl()
                 request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) )
                 response = urllib2.urlopen( request ).read()
                 if response:
                     try:
                         required_repo_info_dict = json.loads( response )
                     except Exception, e:
                         log.exception( e )
                         return all_repo_info_dicts
                     required_repo_info_dicts = []
                     for k, v in required_repo_info_dict.items():
                         if k == 'repo_info_dicts':
                             encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ]
                             for encoded_dict_str in encoded_dict_strings:
                                 decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str )
                                 required_repo_info_dicts.append( decoded_dict )
                         else:
                             if k not in all_required_repo_info_dict:
                                 all_required_repo_info_dict[ k ] = v
                             else:
                                 if v and not all_required_repo_info_dict[ k ]:
                                     all_required_repo_info_dict[ k ] = v
                         if required_repo_info_dicts:
                             for required_repo_info_dict in required_repo_info_dicts:
                                 # Each required_repo_info_dict has a single entry, and all_repo_info_dicts is a list
                                 # of dictionaries, each of which has a single entry.  We'll check keys here rather than
                                 # the entire dictionary because a dictionary entry in all_repo_info_dicts will include
                                 # lists of discovered repository dependencies, but these lists will be empty in the
                                 # required_repo_info_dict since dependency discovery has not yet been performed for these
                                 # dictionaries.
                                 required_repo_info_dict_key = required_repo_info_dict.keys()[ 0 ]
                                 all_repo_info_dicts_keys = [ d.keys()[ 0 ] for d in all_repo_info_dicts ]
                                 if required_repo_info_dict_key not in all_repo_info_dicts_keys:
                                     all_repo_info_dicts.append( required_repo_info_dict )
                     all_required_repo_info_dict[ 'all_repo_info_dicts' ] = all_repo_info_dicts
 def get_required_repo_info_dicts( self, tool_shed_url, repo_info_dicts ):
     """
     Inspect the list of repo_info_dicts for repository dependencies and append a repo_info_dict for each of
     them to the list.  All repository_dependency entries in each of the received repo_info_dicts includes
     all required repositories, so only one pass through this method is required to retrieve all repository
     dependencies.
     """
     all_required_repo_info_dict = {}
     all_repo_info_dicts = []
     if repo_info_dicts:
         # We'll send tuples of ( tool_shed, repository_name, repository_owner, changeset_revision ) to the tool
         # shed to discover repository ids.
         required_repository_tups = []
         for repo_info_dict in repo_info_dicts:
             if repo_info_dict not in all_repo_info_dicts:
                 all_repo_info_dicts.append( repo_info_dict )
             for repository_name, repo_info_tup in repo_info_dict.items():
                 description, \
                     repository_clone_url, \
                     changeset_revision, \
                     ctx_rev, \
                     repository_owner, \
                     repository_dependencies, \
                     tool_dependencies = \
                     suc.get_repo_info_tuple_contents( repo_info_tup )
                 if repository_dependencies:
                     for key, val in repository_dependencies.items():
                         if key in [ 'root_key', 'description' ]:
                             continue
                         repository_components_tuple = container_util.get_components_from_key( key )
                         components_list = suc.extract_components_from_tuple( repository_components_tuple )
                         # Skip listing a repository dependency if it is required only to compile a tool dependency
                         # defined for the dependent repository since in this case, the repository dependency is really
                         # a dependency of the dependent repository's contained tool dependency, and only if that
                         # tool dependency requires compilation.
                         # For backward compatibility to the 12/20/12 Galaxy release.
                         prior_installation_required = 'False'
                         only_if_compiling_contained_td = 'False'
                         if len( components_list ) == 4:
                             prior_installation_required = 'False'
                             only_if_compiling_contained_td = 'False'
                         elif len( components_list ) == 5:
                             prior_installation_required = components_list[ 4 ]
                             only_if_compiling_contained_td = 'False'
                         if not asbool( only_if_compiling_contained_td ):
                             if components_list not in required_repository_tups:
                                 required_repository_tups.append( components_list )
                         for components_list in val:
                             try:
                                 only_if_compiling_contained_td = components_list[ 5 ]
                             except:
                                 only_if_compiling_contained_td = 'False'
                             # Skip listing a repository dependency if it is required only to compile a tool dependency
                             # defined for the dependent repository (see above comment).
                             if not asbool( only_if_compiling_contained_td ):
                                 if components_list not in required_repository_tups:
                                     required_repository_tups.append( components_list )
                 else:
                     # We have a single repository with no dependencies.
                     components_list = [ tool_shed_url, repository_name, repository_owner, changeset_revision ]
                     required_repository_tups.append( components_list )
             if required_repository_tups:
                 # The value of required_repository_tups is a list of tuples, so we need to encode it.
                 encoded_required_repository_tups = []
                 for required_repository_tup in required_repository_tups:
                     # Convert every item in required_repository_tup to a string.
                     required_repository_tup = [ str( item ) for item in required_repository_tup ]
                     encoded_required_repository_tups.append( encoding_util.encoding_sep.join( required_repository_tup ) )
                 encoded_required_repository_str = encoding_util.encoding_sep2.join( encoded_required_repository_tups )
                 encoded_required_repository_str = encoding_util.tool_shed_encode( encoded_required_repository_str )
                 if suc.is_tool_shed_client( self.app ):
                     # Handle secure / insecure Tool Shed URL protocol changes and port changes.
                     tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( self.app, tool_shed_url )
                 url = common_util.url_join( tool_shed_url, '/repository/get_required_repo_info_dict' )
                 # Fix for handling 307 redirect not being handled nicely by urllib2.urlopen when the urllib2.Request has data provided
                 url = urllib2.urlopen( urllib2.Request( url ) ).geturl()
                 request = urllib2.Request( url, data=urllib.urlencode( dict( encoded_str=encoded_required_repository_str ) ) )
                 response = urllib2.urlopen( request ).read()
                 if response:
                     try:
                         required_repo_info_dict = json.loads( response )
                     except Exception, e:
                         log.exception( e )
                         return all_repo_info_dicts
                     required_repo_info_dicts = []
                     for k, v in required_repo_info_dict.items():
                         if k == 'repo_info_dicts':
                             encoded_dict_strings = required_repo_info_dict[ 'repo_info_dicts' ]
                             for encoded_dict_str in encoded_dict_strings:
                                 decoded_dict = encoding_util.tool_shed_decode( encoded_dict_str )
                                 required_repo_info_dicts.append( decoded_dict )
                         else:
                             if k not in all_required_repo_info_dict:
                                 all_required_repo_info_dict[ k ] = v
                             else:
                                 if v and not all_required_repo_info_dict[ k ]:
                                     all_required_repo_info_dict[ k ] = v
                         if required_repo_info_dicts:
                             for required_repo_info_dict in required_repo_info_dicts:
                                 # Each required_repo_info_dict has a single entry, and all_repo_info_dicts is a list
                                 # of dictionaries, each of which has a single entry.  We'll check keys here rather than
                                 # the entire dictionary because a dictionary entry in all_repo_info_dicts will include
                                 # lists of discovered repository dependencies, but these lists will be empty in the
                                 # required_repo_info_dict since dependency discovery has not yet been performed for these
                                 # dictionaries.
                                 required_repo_info_dict_key = required_repo_info_dict.keys()[ 0 ]
                                 all_repo_info_dicts_keys = [ d.keys()[ 0 ] for d in all_repo_info_dicts ]
                                 if required_repo_info_dict_key not in all_repo_info_dicts_keys:
                                     all_repo_info_dicts.append( required_repo_info_dict )
                     all_required_repo_info_dict[ 'all_repo_info_dicts' ] = all_repo_info_dicts
Exemple #21
0
 def populate_tool_dependencies_dicts(
         self, tool_shed_url, tool_path,
         repository_installed_tool_dependencies,
         repository_missing_tool_dependencies, required_repo_info_dicts):
     """
     Return the populated installed_tool_dependencies and missing_tool_dependencies dictionaries
     for all repositories defined by entries in the received required_repo_info_dicts.
     """
     installed_tool_dependencies = None
     missing_tool_dependencies = None
     if repository_installed_tool_dependencies is None:
         repository_installed_tool_dependencies = {}
     else:
         # Add the install_dir attribute to the tool_dependencies.
         repository_installed_tool_dependencies = \
             self.add_installation_directories_to_tool_dependencies( repository_installed_tool_dependencies )
     if repository_missing_tool_dependencies is None:
         repository_missing_tool_dependencies = {}
     else:
         # Add the install_dir attribute to the tool_dependencies.
         repository_missing_tool_dependencies = \
             self.add_installation_directories_to_tool_dependencies( repository_missing_tool_dependencies )
     if required_repo_info_dicts:
         # Handle the tool dependencies defined for each of the repository's repository dependencies.
         for rid in required_repo_info_dicts:
             for name, repo_info_tuple in rid.items():
                 description, repository_clone_url, changeset_revision, \
                     ctx_rev, repository_owner, repository_dependencies, \
                     tool_dependencies = \
                     suc.get_repo_info_tuple_contents( repo_info_tuple )
                 if tool_dependencies:
                     # Add the install_dir attribute to the tool_dependencies.
                     tool_dependencies = self.add_installation_directories_to_tool_dependencies(
                         tool_dependencies)
                     # The required_repository may have been installed with a different changeset revision.
                     required_repository, installed_changeset_revision = \
                         suc.repository_was_previously_installed( self.app,
                                                                  tool_shed_url,
                                                                  name,
                                                                  repo_info_tuple,
                                                                  from_tip=False )
                     if required_repository:
                         required_repository_installed_tool_dependencies, required_repository_missing_tool_dependencies = \
                             self.get_installed_and_missing_tool_dependencies_for_installed_repository( required_repository,
                                                                                                        tool_dependencies )
                         if required_repository_installed_tool_dependencies:
                             # Add the install_dir attribute to the tool_dependencies.
                             required_repository_installed_tool_dependencies = \
                                 self.add_installation_directories_to_tool_dependencies( required_repository_installed_tool_dependencies )
                             for td_key, td_dict in required_repository_installed_tool_dependencies.items(
                             ):
                                 if td_key not in repository_installed_tool_dependencies:
                                     repository_installed_tool_dependencies[
                                         td_key] = td_dict
                         if required_repository_missing_tool_dependencies:
                             # Add the install_dir attribute to the tool_dependencies.
                             required_repository_missing_tool_dependencies = \
                                 self.add_installation_directories_to_tool_dependencies( required_repository_missing_tool_dependencies )
                             for td_key, td_dict in required_repository_missing_tool_dependencies.items(
                             ):
                                 if td_key not in repository_missing_tool_dependencies:
                                     repository_missing_tool_dependencies[
                                         td_key] = td_dict
     if repository_installed_tool_dependencies:
         installed_tool_dependencies = repository_installed_tool_dependencies
     if repository_missing_tool_dependencies:
         missing_tool_dependencies = repository_missing_tool_dependencies
     return installed_tool_dependencies, missing_tool_dependencies