def fetch(self, find_results=None):
find_results = find_results or {}
results = {}
download_url = find_results['custom']['download_url']
# download_url = _build_download_url(external_url=external_url, version=_content_version)
# TODO: error handling if there is no download_url
expected_filename = find_results.get('artifact',
{}).get('filename', None)
log.debug('repository_spec=%s', self.requirement_spec)
log.debug('download_url=%s', download_url)
log.debug('expected_filename=%s', expected_filename)
# for including in any error messages or logging for this fetch
self.remote_resource = download_url
# can raise GalaxyDownloadError
repository_archive_path = download.fetch_url(
download_url,
validate_certs=self.validate_certs,
filename=expected_filename)
self.local_path = repository_archive_path
log.debug('repository_archive_path=%s', repository_archive_path)
# validate the sha256sum of the downloaded artifact against the expected value
expected_chksum = find_results['artifact'].get('sha256')
collection_artifact.validate_artifact(self.local_path, expected_chksum)
# TODO: This is indication that a fetcher is wrong abstraction. A fetch
# can resolve a name/spec, find metadata about the content including avail versions,
# compare/sort versions, select matching versions, find a download uri, and finally
# actually fetch it.
# Ie, more of a RepositoryRepository (aiee) (RepositorySource? RepositoryChannel? RepositoryProvider?)
# that is a remote 'channel' with info and content itself.
results = {
'archive_path': repository_archive_path,
'fetch_method': self.fetch_method
}
# So fetch_results has the download url, if we follow redirects
# we could also add a 'final_download_url' or 'downloaded_url' so
# we know the original and the final url after redirects
results['custom'] = find_results['custom']
results['content'] = find_results['content']
results['artifact'] = find_results['artifact']
return results
def fetch(self):
'''Download the remote_url to a temp file
Can raise GalaxyDownloadError on any exception while downloadin remote_url and saving it.'''
# NOTE: could move download.fetch_url here instead of splitting it
content_archive_path = download.fetch_url(
self.remote_url, validate_certs=self.validate_certs)
self.local_path = content_archive_path
log.debug('content_archive_path=%s', content_archive_path)
return content_archive_path
def fetch(self, find_results=None):
find_results = find_results or {}
results = {}
download_url = get_download_url(
repo_data=find_results['custom']['repo_data'],
external_url=find_results['custom']['external_url'],
repoversion=find_results['custom']['repoversion'])
# download_url = _build_download_url(external_url=external_url, version=_content_version)
# TODO: error handling if there is no download_url
log.debug('repository_spec=%s', self.requirement_spec)
log.debug('download_url=%s', download_url)
# for including in any error messages or logging for this fetch
self.remote_resource = download_url
# can raise GalaxyDownloadError
repository_archive_path = download.fetch_url(
download_url, validate_certs=self.validate_certs)
self.local_path = repository_archive_path
log.debug('repository_archive_path=%s', repository_archive_path)
# TODO: This is indication that a fetcher is wrong abstraction. A fetch
# can resolve a name/spec, find metadata about the content including avail versions,
# compare/sort versions, select matching versions, find a download uri, and finally
# actually fetch it.
# Ie, more of a RepositoryRepository (aiee) (RepositorySource? RepositoryChannel? RepositoryProvider?)
# that is a remote 'channel' with info and content itself.
results = {
'archive_path': repository_archive_path,
'download_url': download_url,
'fetch_method': self.fetch_method
}
results['custom'] = {}
results['content'] = find_results['content']
results['content']['fetched_version'] = find_results['custom'][
'repoversion'].get('version')
return results
def fetch(self, find_results=None):
'''Download the remote_url to a temp file
Can raise GalaxyDownloadError on any exception while downloadin remote_url and saving it.'''
find_results = find_results or {}
# NOTE: could move download.fetch_url here instead of splitting it
repository_archive_path = download.fetch_url(self.remote_url, validate_certs=self.validate_certs)
self.local_path = repository_archive_path
log.debug('repository_archive_path=%s', repository_archive_path)
results = {'archive_path': repository_archive_path,
'fetch_method': self.fetch_method}
results['content'] = find_results['content']
results['custom'] = {'remote_url': self.remote_url,
'validate_certs': self.validate_certs}
return results
def fetch(self):
api = GalaxyAPI(self.galaxy_context)
# FIXME - Need to update our API calls once Galaxy has them implemented
content_username, repo_name, content_name = parse_content_name(
self.content_spec)
log.debug('content_username=%s', content_username)
log.debug('repo_name=%s', repo_name)
log.debug('content_name=%s', content_name)
# TODO: extract parsing of cli content sorta-url thing and add better tests
repo_name = repo_name or content_name
# FIXME: exception handling
repo_data = api.lookup_repo_by_name(content_username, repo_name)
if not repo_data:
raise exceptions.GalaxyClientError(
"- sorry, %s was not found on %s." %
(self.content_spec, api.api_server))
# FIXME: ?
# if repo_data.get('role_type') == 'APP#':
# Container Role
# self.display_callback("%s is a Container App role, and should only be installed using Ansible "
# "Container" % content_name, level='warning')
# FIXME - Need to update our API calls once Galaxy has them implemented
related = repo_data.get('related', {})
repo_versions_url = related.get('versions', None)
log.debug('related=%s', related)
# FIXME: exception handling
repo_versions = api.fetch_content_related(repo_versions_url)
log.debug('repo_versions: %s', repo_versions)
# related_repo_url = related.get('repository', None)
# log.debug('related_repo_url: %s', related_repo_url)
# related_content_url = related.get('content', None)
# log.debug('related_content_url: %s', related_content_url)
# content_repo = None
# if related_content_url:
# content_repo = api.fetch_content_related(related_content_url)
content_repo_versions = [
a.get('name') for a in repo_versions if a.get('name', None)
]
log.debug('content_repo_versions: %s', content_repo_versions)
# log.debug('content_repo: %s', content_repo)
# FIXME: mv to it's own method
# FIXME: pass these to fetch() if it really needs it
_content_version = content_version.get_content_version(
repo_data,
version=self.content_version,
content_versions=content_repo_versions,
content_content_name=content_name)
# FIXME: stop munging state
# self.content_meta.version = _content_version
external_url = repo_data.get('external_url', None)
if not external_url:
raise exceptions.GalaxyError(
'no external_url info on the Repository object from %s',
repo_name)
download_url = _build_download_url(external_url=external_url,
version=_content_version)
log.debug('content_spec=%s', self.content_spec)
log.debug('download_url=%s', download_url)
try:
content_archive_path = download.fetch_url(
download_url, validate_certs=self.validate_certs)
except exceptions.GalaxyDownloadError as e:
log.exception(e)
self.display_callback("failed to download the file: %s" % str(e))
return None
self.local_path = content_archive_path
log.debug('content_archive_path=%s', content_archive_path)
return content_archive_path
def install_repository_specs_loop(galaxy_context,
repository_spec_strings=None,
requirements_list=None,
collections_lockfile_path=None,
editable=False,
namespace_override=None,
display_callback=None,
# TODO: error handling callback ?
ignore_errors=False,
no_deps=False,
force_overwrite=False):
requirements_list = requirements_list or []
for repository_spec_string in repository_spec_strings:
fetch_method = \
repository_spec_parse.choose_repository_fetch_method(repository_spec_string,
editable=editable)
log.debug('fetch_method: %s', fetch_method)
if fetch_method == FetchMethods.LOCAL_FILE:
# Since we only know this is a local file we vaguely recognize, we have to
# open it up to get any more details. We _could_ attempt to parse the file
# name, but that rarely ends well. Filename could also be arbitrary for downloads
# from remote urls ('mazer install http://myci.example.com/somebuildjob/latest' etc)
spec_data = collection_artifact.load_data_from_collection_artifact(repository_spec_string)
spec_data['fetch_method'] = fetch_method
elif fetch_method == FetchMethods.REMOTE_URL:
# download the url
# hope it is a collection artifact and use load_data_from_collection_artifact() for the
# rest of the repo_spec data
log.debug('repository_spec_string: %s', repository_spec_string)
tmp_downloaded_path = download.fetch_url(repository_spec_string,
# Note: ignore_certs is meant for galaxy server,
# overloaded to apply for arbitrary http[s] downloads here
validate_certs=not galaxy_context.server['ignore_certs'])
spec_data = collection_artifact.load_data_from_collection_artifact(tmp_downloaded_path)
# pretend like this is a local_file install now
spec_data['fetch_method'] = FetchMethods.LOCAL_FILE
else:
spec_data = repository_spec_parse.spec_data_from_string(repository_spec_string,
namespace_override=namespace_override,
editable=editable)
spec_data['fetch_method'] = fetch_method
log.debug('spec_data: %s', spec_data)
req_spec = RequirementSpec.from_dict(spec_data)
req = Requirement(repository_spec=None, op=RequirementOps.EQ, requirement_spec=req_spec)
requirements_list.append(req)
log.debug('collections_lockfile_path: %s', collections_lockfile_path)
if collections_lockfile_path:
# load collections lockfile as if the 'dependencies' dict from a collection_info
collections_lockfile = load_collections_lockfile(collections_lockfile_path)
dependencies_list = requirements.from_dependencies_dict(collections_lockfile.dependencies)
# Create the CollectionsLock for the validators
collections_lock = CollectionsLock(dependencies=dependencies_list)
requirements_list.extend(collections_lock.dependencies)
log.debug('requirements_list: %s', requirements_list)
while True:
if not requirements_list:
break
display_callback('', level='info')
display_callback('Collection specs to install:', level='info')
for req in requirements_list:
if req.repository_spec:
msg = ' %s (required by %s)' % (req.requirement_spec.label, req.repository_spec)
else:
msg = ' %s' % req.requirement_spec.label
display_callback(msg, level='info')
just_installed_repositories = \
install_repositories_matching_repository_specs(galaxy_context,
requirements_list,
editable=editable,
namespace_override=namespace_override,
display_callback=display_callback,
ignore_errors=ignore_errors,
no_deps=no_deps,
force_overwrite=force_overwrite)
for just_installed_repo in just_installed_repositories:
display_callback(' Installed: %s (to %s)' %
(just_installed_repo.repository_spec,
just_installed_repo.path),
level='info')
# set the repository_specs to search for to whatever the install reported as being needed yet
# requirements_list = new_requirements_list
requirements_list = find_new_deps_from_installed(galaxy_context,
just_installed_repositories,
no_deps=no_deps)
# FIXME: what results to return?
return 0
