Beispiel #1
0
def test_fetch_download_error(mocker):
    mock_fetcher = mocker.MagicMock(name='MockFetch')
    mock_fetcher.fetch.side_effect = exceptions.GalaxyDownloadError(
        url='http://example.invalid')

    repo_spec = RepositorySpec(namespace='some_namespace',
                               name='some_name',
                               version='86.75.30')
    find_results = {}

    with pytest.raises(exceptions.GalaxyDownloadError) as exc_info:
        install.fetch(mock_fetcher, repo_spec, find_results)
    log.debug("exc_info: %s", exc_info)
Beispiel #2
0
def test_fetch(mocker):
    mock_fetcher = mocker.MagicMock(name='MockFetch')
    mock_fetcher.fetch.return_value = {}
    repo_spec = RepositorySpec(namespace='some_namespace',
                               name='some_name',
                               version='86.75.30')
    find_results = {}

    res = install.fetch(mock_fetcher, repo_spec, find_results)

    log.debug('res: %s', res)
Beispiel #3
0
def install_repository(galaxy_context,
                       requirement_to_install,
                       display_callback=None,
                       # TODO: error handling callback ?
                       ignore_errors=False,
                       no_deps=False,
                       force_overwrite=False):
    '''This installs a single package by finding it, fetching it, verifying it and installing it.'''

    display_callback = display_callback or display.display_callback

    # INITIAL state
    # dep_requirements = []

    # TODO: we could do all the downloads first, then install them. Likely
    #       less error prone mid 'transaction'
    log.debug('Processing %r', requirement_to_install)

    repository_spec_to_install = requirement_to_install.requirement_spec
    requirement_spec_to_install = requirement_to_install.requirement_spec

    # else trans to ... FIND_FETCHER?

    # TODO: check if already installed and move to approriate state

    log.debug('About to find() requested requirement_spec_to_install: %s', requirement_spec_to_install)

    display_callback('', level='info')
    display_callback('Installing spec: %s' % requirement_spec_to_install.label, level='info')

    # We dont have anything that matches the RequirementSpec installed
    fetcher = fetch_factory.get(galaxy_context=galaxy_context,
                                requirement_spec=requirement_spec_to_install)

    # if we fail to get a fetcher here, then to... FIND_FETCHER_FAILURE ?
    # could also move some of the logic in fetcher_factory to be driven from here
    # and make the steps of mapping repository spec -> fetcher method part of the
    # state machine. That might be a good place to support multiple galaxy servers
    # or preferring local content to remote content, etc.

    # FIND state
    # See if we can find metadata and/or download the archive before we try to
    # remove an installed version...
    try:
        find_results = install.find(fetcher)
    except exceptions.GalaxyError as e:
        log.debug('requirement_to_install %s failed to be met: %s', requirement_to_install, e)
        log.warning('Unable to find metadata for %s: %s', requirement_spec_to_install.label, e)
        # FIXME: raise dep error exception?
        raise_without_ignore(ignore_errors, e)
        # continue
        return None

    # TODO: make sure repository_spec version is correct and set

    # TODO: state transition, if find_results -> INSTALL
    #       if not, then FIND_FAILED

    # TODO/FIXME: We give find() a RequirementSpec, but find_results should have enough
    #             info to create a concrete RepositorySpec

    # TODO: if we want client side content whitelist/blacklist, or pinned versions,
    #       or rules to only update within some semver range (ie, only 'patch' level),
    #       we could hook rule validation stuff here.

    # TODO: build a new repository_spec based on what we actually fetched to feed to
    #       install etc. The fetcher.fetch() could return a datastructure needed to build
    #       the new one instead of doing it in verify()
    found_repository_spec = install.repository_spec_from_find_results(find_results,
                                                                      requirement_spec_to_install)

    log.debug('found_repository_spec: %s', found_repository_spec)

    display_callback('  Found: %s (for spec %s)' % (found_repository_spec, requirement_spec_to_install.label))

    # See if the found collection spec is already installed and either warn or 'force_overwrite'
    # to remove existing first.

    # cheap 'update' is to consider anything already installed that matches the request repo_spec
    # as 'installed' and let force override that.

    # potential_repository_spec is a repo spec for the install candidate we potentially found.
    irdb = installed_repository_db.InstalledRepositoryDatabase(galaxy_context)
    # log.debug('Checking to see if %s is already installed', requirement_spec_to_install)
    log.debug('Checking to see if a collection named %s is already installed', found_repository_spec.label)

    repository_spec_match_filter = matchers.MatchRepositorySpecNamespaceName([found_repository_spec])

    # already_installed_iter = irdb.by_requirement_spec(requirement_spec_to_install)
    already_installed_iter = irdb.select(repository_spec_match_filter=repository_spec_match_filter)
    already_installed = sorted(list(already_installed_iter))

    log.debug('already_installed: %s', already_installed)

    # TODO: The already installed check above verifies that nothing that matches the requirement spec is installed,
    #       but just because the name+version required wasn't installed, that doesn't mean that name at a different
    #       version isn't installed.
    #       To catch that, also need to check if the irdb by name to see if anything with that name is installed.
    #

    repository_spec_to_install = found_repository_spec
    log.debug('About to download repository requested by %s: %s', requirement_spec_to_install, repository_spec_to_install)

    if find_results['custom'].get('collection_is_deprecated', False):
        display_callback("The collection '%s' is deprecated." % (found_repository_spec.label),
                         level='warning')

    # FETCH state
    try:
        fetch_results = install.fetch(fetcher,
                                      repository_spec=repository_spec_to_install,
                                      find_results=find_results)
        log.debug('fetch_results: %s', fetch_results)
        # fetch_results will include a 'archive_path' pointing to where the artifact
        # was saved to locally.
    except exceptions.GalaxyError as e:
        # fetch error probably should just go to a FAILED state, at least until
        # we have to implement retries
        log.warning('Unable to fetch %s: %s', repository_spec_to_install.name, e)
        raise_without_ignore(ignore_errors, e)
        # continue
        # FIXME: raise ?
        return None

    # FIXME: seems like we want to resolve deps before trying install
    #        We need the role (or other content) deps from meta before installing
    #        though, and sometimes (for galaxy case) we dont know that until we've downloaded
    #        the file, which we dont do until somewhere in the begin of content.install (fetch).
    #        We can get that from the galaxy API though.
    #
    # FIXME: exc handling

    # Remove the already installed version, via --force

    for already_installed_repository in already_installed:
        repo_label = '%s,%s' % (already_installed_repository.repository_spec.label,
                                already_installed_repository.repository_spec.version)

        # bail if we are not overwriting already installed content
        if not force_overwrite:
            display_callback('  %s is already installed at %s' %
                             (repo_label,
                              already_installed_repository.path),
                             level='warning')

            log.debug('A collection providing %s was already installed. In %s', requirement_spec_to_install, already_installed)

            return None

        display_callback('  Removing: %s (previously installed to %s)' %
                         (repo_label,
                          already_installed_repository.path),
                         level='info')

        log.debug('Removing already_installed %s', already_installed_repository)

        repository.remove(already_installed_repository)

    installed_repositories = []

    try:
        installed_repositories = install.install(galaxy_context,
                                                 fetcher,
                                                 fetch_results,
                                                 repository_spec=found_repository_spec,
                                                 force_overwrite=force_overwrite,
                                                 display_callback=display_callback)
    except exceptions.GalaxyError as e:
        msg = "- %s was NOT installed successfully: %s "
        display_callback(msg % (found_repository_spec, e), level='warning')
        log.warning(msg, found_repository_spec.label, str(e))
        raise_without_ignore(ignore_errors, e)
        return []

    if not installed_repositories:
        log.warning("- %s was NOT installed successfully.", found_repository_spec.label)
        raise_without_ignore(ignore_errors)

    return installed_repositories
Beispiel #4
0
def install_repository(
        galaxy_context,
        requirement_to_install,
        display_callback=None,
        # TODO: error handling callback ?
        ignore_errors=False,
        no_deps=False,
        force_overwrite=False):
    '''This installs a single package by finding it, fetching it, verifying it and installing it.'''

    display_callback = display_callback or display.display_callback

    # INITIAL state
    # dep_requirements = []

    # TODO: we could do all the downloads first, then install them. Likely
    #       less error prone mid 'transaction'
    log.debug('Processing %s', requirement_to_install)

    repository_spec_to_install = requirement_to_install.requirement_spec

    # else trans to ... FIND_FETCHER?

    # TODO: check if already installed and move to approriate state

    log.debug('About to find() requested repository_spec_to_install: %s',
              repository_spec_to_install)

    # potential_repository_spec is a repo spec for the install candidate we potentially found.
    irdb = installed_repository_db.InstalledRepositoryDatabase(galaxy_context)
    log.debug('Checking to see if %s is already installed',
              repository_spec_to_install)

    already_installed_iter = irdb.by_repository_spec(
        repository_spec_to_install)
    already_installed = sorted(list(already_installed_iter))

    log.debug('already_installed: %s', already_installed)

    if already_installed:
        for already_installed_repository in already_installed:
            display_callback(
                '%s is already installed at %s' %
                (already_installed_repository.repository_spec.label,
                 already_installed_repository.path),
                level='warning')
        log.debug('Stuff %s was already installed. In %s',
                  repository_spec_to_install, already_installed)

        return None

    fetcher = fetch_factory.get(galaxy_context=galaxy_context,
                                repository_spec=repository_spec_to_install)

    # if we fail to get a fetcher here, then to... FIND_FETCHER_FAILURE ?
    # could also move some of the logic in fetcher_factory to be driven from here
    # and make the steps of mapping repository spec -> fetcher method part of the
    # state machine. That might be a good place to support multiple galaxy servers
    # or preferring local content to remote content, etc.

    # FIND state
    # See if we can find metadata and/or download the archive before we try to
    # remove an installed version...
    try:
        find_results = install.find(fetcher)
    except exceptions.GalaxyError as e:
        log.warning('Unable to find metadata for %s: %s',
                    repository_spec_to_install.label, e)
        # FIXME: raise dep error exception?
        raise_without_ignore(ignore_errors, e)
        # continue
        return None

    # TODO: make sure repository_spec version is correct and set

    # TODO: state transition, if find_results -> INSTALL
    #       if not, then FIND_FAILED

    log.debug('About to download requested repository_spec_to_install: %s',
              repository_spec_to_install)

    # FETCH state
    try:
        fetch_results = install.fetch(
            fetcher,
            repository_spec=repository_spec_to_install,
            find_results=find_results)
        log.debug('fetch_results: %s', fetch_results)
        # fetch_results will include a 'archive_path' pointing to where the artifact
        # was saved to locally.
    except exceptions.GalaxyError as e:
        # fetch error probably should just go to a FAILED state, at least until
        # we have to implement retries
        log.warning('Unable to fetch %s: %s', repository_spec_to_install.name,
                    e)
        raise_without_ignore(ignore_errors, e)
        # continue
        # FIXME: raise ?
        return None

    # TODO: if we want client side content whitelist/blacklist, or pinned versions,
    #       or rules to only update within some semver range (ie, only 'patch' level),
    #       we could hook rule validation stuff here.

    # TODO: build a new repository_spec based on what we actually fetched to feed to
    #       install etc. The fetcher.fetch() could return a datastructure needed to build
    #       the new one instead of doing it in verify()
    fetched_repository_spec = install.update_repository_spec(
        fetch_results, repository_spec_to_install)

    log.debug('fetched_repository_spec: %s', fetched_repository_spec)

    # FIXME: seems like we want to resolve deps before trying install
    #        We need the role (or other content) deps from meta before installing
    #        though, and sometimes (for galaxy case) we dont know that until we've downloaded
    #        the file, which we dont do until somewhere in the begin of content.install (fetch).
    #        We can get that from the galaxy API though.
    #
    # FIXME: exc handling

    installed_repositories = []

    try:
        installed_repositories = install.install(
            galaxy_context,
            fetcher,
            fetch_results,
            repository_spec=fetched_repository_spec,
            force_overwrite=force_overwrite,
            display_callback=display_callback)
    except exceptions.GalaxyError as e:
        msg = "- %s was NOT installed successfully: %s "
        display_callback(msg % (fetched_repository_spec.label, e),
                         level='warning')
        log.warning(msg, fetched_repository_spec.label, str(e))
        raise_without_ignore(ignore_errors, e)
        return []

    if not installed_repositories:
        log.warning("- %s was NOT installed successfully.",
                    fetched_repository_spec.label)
        raise_without_ignore(ignore_errors)

    return installed_repositories