def test_fetch_download_error(mocker):
mock_fetcher = mocker.MagicMock(name='MockFetch')
mock_fetcher.fetch.side_effect = exceptions.GalaxyDownloadError(
url='http://example.invalid')
repo_spec = RepositorySpec(namespace='some_namespace',
name='some_name',
version='86.75.30')
find_results = {}
with pytest.raises(exceptions.GalaxyDownloadError) as exc_info:
install.fetch(mock_fetcher, repo_spec, find_results)
log.debug("exc_info: %s", exc_info)
def test_fetch(mocker):
mock_fetcher = mocker.MagicMock(name='MockFetch')
mock_fetcher.fetch.return_value = {}
repo_spec = RepositorySpec(namespace='some_namespace',
name='some_name',
version='86.75.30')
find_results = {}
res = install.fetch(mock_fetcher, repo_spec, find_results)
log.debug('res: %s', res)
def install_repository(galaxy_context,
requirement_to_install,
display_callback=None,
# TODO: error handling callback ?
ignore_errors=False,
no_deps=False,
force_overwrite=False):
'''This installs a single package by finding it, fetching it, verifying it and installing it.'''
display_callback = display_callback or display.display_callback
# INITIAL state
# dep_requirements = []
# TODO: we could do all the downloads first, then install them. Likely
# less error prone mid 'transaction'
log.debug('Processing %r', requirement_to_install)
repository_spec_to_install = requirement_to_install.requirement_spec
requirement_spec_to_install = requirement_to_install.requirement_spec
# else trans to ... FIND_FETCHER?
# TODO: check if already installed and move to approriate state
log.debug('About to find() requested requirement_spec_to_install: %s', requirement_spec_to_install)
display_callback('', level='info')
display_callback('Installing spec: %s' % requirement_spec_to_install.label, level='info')
# We dont have anything that matches the RequirementSpec installed
fetcher = fetch_factory.get(galaxy_context=galaxy_context,
requirement_spec=requirement_spec_to_install)
# if we fail to get a fetcher here, then to... FIND_FETCHER_FAILURE ?
# could also move some of the logic in fetcher_factory to be driven from here
# and make the steps of mapping repository spec -> fetcher method part of the
# state machine. That might be a good place to support multiple galaxy servers
# or preferring local content to remote content, etc.
# FIND state
# See if we can find metadata and/or download the archive before we try to
# remove an installed version...
try:
find_results = install.find(fetcher)
except exceptions.GalaxyError as e:
log.debug('requirement_to_install %s failed to be met: %s', requirement_to_install, e)
log.warning('Unable to find metadata for %s: %s', requirement_spec_to_install.label, e)
# FIXME: raise dep error exception?
raise_without_ignore(ignore_errors, e)
# continue
return None
# TODO: make sure repository_spec version is correct and set
# TODO: state transition, if find_results -> INSTALL
# if not, then FIND_FAILED
# TODO/FIXME: We give find() a RequirementSpec, but find_results should have enough
# info to create a concrete RepositorySpec
# TODO: if we want client side content whitelist/blacklist, or pinned versions,
# or rules to only update within some semver range (ie, only 'patch' level),
# we could hook rule validation stuff here.
# TODO: build a new repository_spec based on what we actually fetched to feed to
# install etc. The fetcher.fetch() could return a datastructure needed to build
# the new one instead of doing it in verify()
found_repository_spec = install.repository_spec_from_find_results(find_results,
requirement_spec_to_install)
log.debug('found_repository_spec: %s', found_repository_spec)
display_callback(' Found: %s (for spec %s)' % (found_repository_spec, requirement_spec_to_install.label))
# See if the found collection spec is already installed and either warn or 'force_overwrite'
# to remove existing first.
# cheap 'update' is to consider anything already installed that matches the request repo_spec
# as 'installed' and let force override that.
# potential_repository_spec is a repo spec for the install candidate we potentially found.
irdb = installed_repository_db.InstalledRepositoryDatabase(galaxy_context)
# log.debug('Checking to see if %s is already installed', requirement_spec_to_install)
log.debug('Checking to see if a collection named %s is already installed', found_repository_spec.label)
repository_spec_match_filter = matchers.MatchRepositorySpecNamespaceName([found_repository_spec])
# already_installed_iter = irdb.by_requirement_spec(requirement_spec_to_install)
already_installed_iter = irdb.select(repository_spec_match_filter=repository_spec_match_filter)
already_installed = sorted(list(already_installed_iter))
log.debug('already_installed: %s', already_installed)
# TODO: The already installed check above verifies that nothing that matches the requirement spec is installed,
# but just because the name+version required wasn't installed, that doesn't mean that name at a different
# version isn't installed.
# To catch that, also need to check if the irdb by name to see if anything with that name is installed.
#
repository_spec_to_install = found_repository_spec
log.debug('About to download repository requested by %s: %s', requirement_spec_to_install, repository_spec_to_install)
if find_results['custom'].get('collection_is_deprecated', False):
display_callback("The collection '%s' is deprecated." % (found_repository_spec.label),
level='warning')
# FETCH state
try:
fetch_results = install.fetch(fetcher,
repository_spec=repository_spec_to_install,
find_results=find_results)
log.debug('fetch_results: %s', fetch_results)
# fetch_results will include a 'archive_path' pointing to where the artifact
# was saved to locally.
except exceptions.GalaxyError as e:
# fetch error probably should just go to a FAILED state, at least until
# we have to implement retries
log.warning('Unable to fetch %s: %s', repository_spec_to_install.name, e)
raise_without_ignore(ignore_errors, e)
# continue
# FIXME: raise ?
return None
# FIXME: seems like we want to resolve deps before trying install
# We need the role (or other content) deps from meta before installing
# though, and sometimes (for galaxy case) we dont know that until we've downloaded
# the file, which we dont do until somewhere in the begin of content.install (fetch).
# We can get that from the galaxy API though.
#
# FIXME: exc handling
# Remove the already installed version, via --force
for already_installed_repository in already_installed:
repo_label = '%s,%s' % (already_installed_repository.repository_spec.label,
already_installed_repository.repository_spec.version)
# bail if we are not overwriting already installed content
if not force_overwrite:
display_callback(' %s is already installed at %s' %
(repo_label,
already_installed_repository.path),
level='warning')
log.debug('A collection providing %s was already installed. In %s', requirement_spec_to_install, already_installed)
return None
display_callback(' Removing: %s (previously installed to %s)' %
(repo_label,
already_installed_repository.path),
level='info')
log.debug('Removing already_installed %s', already_installed_repository)
repository.remove(already_installed_repository)
installed_repositories = []
try:
installed_repositories = install.install(galaxy_context,
fetcher,
fetch_results,
repository_spec=found_repository_spec,
force_overwrite=force_overwrite,
display_callback=display_callback)
except exceptions.GalaxyError as e:
msg = "- %s was NOT installed successfully: %s "
display_callback(msg % (found_repository_spec, e), level='warning')
log.warning(msg, found_repository_spec.label, str(e))
raise_without_ignore(ignore_errors, e)
return []
if not installed_repositories:
log.warning("- %s was NOT installed successfully.", found_repository_spec.label)
raise_without_ignore(ignore_errors)
return installed_repositories
def install_repository(
galaxy_context,
requirement_to_install,
display_callback=None,
# TODO: error handling callback ?
ignore_errors=False,
no_deps=False,
force_overwrite=False):
'''This installs a single package by finding it, fetching it, verifying it and installing it.'''
display_callback = display_callback or display.display_callback
# INITIAL state
# dep_requirements = []
# TODO: we could do all the downloads first, then install them. Likely
# less error prone mid 'transaction'
log.debug('Processing %s', requirement_to_install)
repository_spec_to_install = requirement_to_install.requirement_spec
# else trans to ... FIND_FETCHER?
# TODO: check if already installed and move to approriate state
log.debug('About to find() requested repository_spec_to_install: %s',
repository_spec_to_install)
# potential_repository_spec is a repo spec for the install candidate we potentially found.
irdb = installed_repository_db.InstalledRepositoryDatabase(galaxy_context)
log.debug('Checking to see if %s is already installed',
repository_spec_to_install)
already_installed_iter = irdb.by_repository_spec(
repository_spec_to_install)
already_installed = sorted(list(already_installed_iter))
log.debug('already_installed: %s', already_installed)
if already_installed:
for already_installed_repository in already_installed:
display_callback(
'%s is already installed at %s' %
(already_installed_repository.repository_spec.label,
already_installed_repository.path),
level='warning')
log.debug('Stuff %s was already installed. In %s',
repository_spec_to_install, already_installed)
return None
fetcher = fetch_factory.get(galaxy_context=galaxy_context,
repository_spec=repository_spec_to_install)
# if we fail to get a fetcher here, then to... FIND_FETCHER_FAILURE ?
# could also move some of the logic in fetcher_factory to be driven from here
# and make the steps of mapping repository spec -> fetcher method part of the
# state machine. That might be a good place to support multiple galaxy servers
# or preferring local content to remote content, etc.
# FIND state
# See if we can find metadata and/or download the archive before we try to
# remove an installed version...
try:
find_results = install.find(fetcher)
except exceptions.GalaxyError as e:
log.warning('Unable to find metadata for %s: %s',
repository_spec_to_install.label, e)
# FIXME: raise dep error exception?
raise_without_ignore(ignore_errors, e)
# continue
return None
# TODO: make sure repository_spec version is correct and set
# TODO: state transition, if find_results -> INSTALL
# if not, then FIND_FAILED
log.debug('About to download requested repository_spec_to_install: %s',
repository_spec_to_install)
# FETCH state
try:
fetch_results = install.fetch(
fetcher,
repository_spec=repository_spec_to_install,
find_results=find_results)
log.debug('fetch_results: %s', fetch_results)
# fetch_results will include a 'archive_path' pointing to where the artifact
# was saved to locally.
except exceptions.GalaxyError as e:
# fetch error probably should just go to a FAILED state, at least until
# we have to implement retries
log.warning('Unable to fetch %s: %s', repository_spec_to_install.name,
e)
raise_without_ignore(ignore_errors, e)
# continue
# FIXME: raise ?
return None
# TODO: if we want client side content whitelist/blacklist, or pinned versions,
# or rules to only update within some semver range (ie, only 'patch' level),
# we could hook rule validation stuff here.
# TODO: build a new repository_spec based on what we actually fetched to feed to
# install etc. The fetcher.fetch() could return a datastructure needed to build
# the new one instead of doing it in verify()
fetched_repository_spec = install.update_repository_spec(
fetch_results, repository_spec_to_install)
log.debug('fetched_repository_spec: %s', fetched_repository_spec)
# FIXME: seems like we want to resolve deps before trying install
# We need the role (or other content) deps from meta before installing
# though, and sometimes (for galaxy case) we dont know that until we've downloaded
# the file, which we dont do until somewhere in the begin of content.install (fetch).
# We can get that from the galaxy API though.
#
# FIXME: exc handling
installed_repositories = []
try:
installed_repositories = install.install(
galaxy_context,
fetcher,
fetch_results,
repository_spec=fetched_repository_spec,
force_overwrite=force_overwrite,
display_callback=display_callback)
except exceptions.GalaxyError as e:
msg = "- %s was NOT installed successfully: %s "
display_callback(msg % (fetched_repository_spec.label, e),
level='warning')
log.warning(msg, fetched_repository_spec.label, str(e))
raise_without_ignore(ignore_errors, e)
return []
if not installed_repositories:
log.warning("- %s was NOT installed successfully.",
fetched_repository_spec.label)
raise_without_ignore(ignore_errors)
return installed_repositories