def from_dependencies_dict(dependencies_dict,
namespace_override=None,
editable=False,
repository_spec=None):
'''Build a list of Requirement objects from the 'dependencies' item in galaxy.yml'''
reqs = []
for req_label, req_version_spec in dependencies_dict.items():
req_spec_data = spec_data_from_string(
req_label,
namespace_override=namespace_override,
editable=editable)
req_spec_data['version_spec'] = req_version_spec
req_spec_data['req_spec_string'] = req_spec_data.pop(
'spec_string', None)
log.debug('req_spec_data: %s', req_spec_data)
req_spec = RequirementSpec.from_dict(req_spec_data)
log.debug('req_spec: %s', req_spec)
requirement = Requirement(repository_spec=repository_spec,
op=RequirementOps.EQ,
scope=RequirementScopes.INSTALL,
requirement_spec=req_spec)
log.debug('requirement: %s', requirement)
reqs.append(requirement)
return reqs
def from_dependency_spec_strings(dependency_spec_strings, namespace_override=None, editable=False):
deps = []
for dep_spec_string in dependency_spec_strings:
dep_spec_data = spec_data_from_string(dep_spec_string)
log.debug('dep_spec_data: %s', dep_spec_data)
dep_spec = RequirementSpec.from_dict(dep_spec_data)
log.debug('dep_spec: %s', dep_spec)
# Add a requirement, but with the 'RUNTIME' scope
requirement = Requirement(repository_spec=None, op=RequirementOps.EQ,
scope=RequirementScopes.RUNTIME,
requirement_spec=dep_spec)
deps.append(requirement)
return deps
def repository_spec_from_string(repository_spec_string,
namespace_override=None,
editable=False):
spec_data = repository_spec_parse.spec_data_from_string(
repository_spec_string,
namespace_override=namespace_override,
editable=editable)
log.debug('spec_data: %s', spec_data)
return RepositorySpec(
name=spec_data.get('name'),
namespace=spec_data.get('namespace'),
version=spec_data.get('version'),
# version=version,
scm=spec_data.get('scm'),
spec_string=spec_data.get('spec_string'),
fetch_method=spec_data.get('fetch_method'),
src=spec_data.get('src'))
def parse_repository_spec(repository_spec_string, resolver=None):
result = repository_spec_parse.spec_data_from_string(repository_spec_string,
resolver=resolver)
log.debug('result: %s', result)
return result
def yaml_parse(content, resolver=None):
"""parses the passed in yaml string and returns a dict with name/src/scm/version
Or... if the passed in 'content' is a dict, it either creates role or if not a role,
it copies the dict and sets name/src/scm/version in it"""
# FIXME: rm once we stop clobbering the passed in content, we can stop making a
# copy of orig_content as this is just for logging
# the original value
if isinstance(content, six.string_types):
log.debug('parsing content="%s" as a string', content)
# orig_content = copy.deepcopy(content)
res = repository_spec_parse.spec_data_from_string(content, resolver=resolver)
log.debug('parsed spec="%s" -> %s', content, res)
return res
# log.debug('content="%s" is not a string (it is a %s) so we are assuming it is a dict',
# content, type(content))
# orig_content = copy.deepcopy(content)
# Not sure what will/should happen if content is not a Mapping or a string
# FIXME: if content is not a string or a dict/map, throw a reasonable error.
# for ex, if a list of strings is passed in, the content.copy() below throws
# an attribute error
# FIXME: This isn't a 'parse' at all, if we want to convert a dict/map to the specific
# type of dict we expect, should be a different method
# FIXME: what is expected to happen if passed an empty dict?
if 'role' in content:
log.debug('content="%s" appears to be a role', content)
name = content['role']
if ',' in name:
# Old style: {role: "galaxy.role,version,name", other_vars: "here" }
# Maintained for backwards compat
content = role_spec_parse(content['role'])
else:
del content['role']
content['name'] = name
else:
# log.debug('content="%s" does not appear to be a role', content)
# FIXME: just use a new name already
# FIXME: this fails for objects with no dict attribute, like a list
content_copy = content.copy()
# log.debug('content_copy: %s', content_copy)
data = {'src': None, 'version': None, 'name': None, 'scm': None}
data.update(content_copy)
content = data
if data.get('src', None):
# valid_kw = ('src', 'version', 'name', 'scm')
new_data = repository_spec_parse.spec_data_from_string(data['src'])
# log.debug('new_data: %s', new_data)
for key in new_data:
if not data.get(key, None):
data[key] = new_data[key]
# New style: { src: 'galaxy.role,version,name', other_vars: "here" }
if 'github.com' in content["src"] and 'http' in content["src"] and '+' not in content["src"] and not content["src"].endswith('.tar.gz'):
content["src"] = "git+" + content["src"]
if 'version' not in content:
content['version'] = ''
if 'scm' not in content:
content['scm'] = None
for key in list(content.keys()):
# sub_name doesnt mean anything to role spec
if key not in VALID_ROLE_SPEC_KEYS and key not in ('sub_name', 'namespace', 'fetch_method'):
# log.debug('removing invalid key: %s', key)
content.pop(key)
# log.debug('"parsed" content="%s" into: %s', orig_content, content)
return content
def install_repository_specs_loop(galaxy_context,
repository_spec_strings=None,
requirements_list=None,
collections_lockfile_path=None,
editable=False,
namespace_override=None,
display_callback=None,
# TODO: error handling callback ?
ignore_errors=False,
no_deps=False,
force_overwrite=False):
requirements_list = requirements_list or []
for repository_spec_string in repository_spec_strings:
fetch_method = \
repository_spec_parse.choose_repository_fetch_method(repository_spec_string,
editable=editable)
log.debug('fetch_method: %s', fetch_method)
if fetch_method == FetchMethods.LOCAL_FILE:
# Since we only know this is a local file we vaguely recognize, we have to
# open it up to get any more details. We _could_ attempt to parse the file
# name, but that rarely ends well. Filename could also be arbitrary for downloads
# from remote urls ('mazer install http://myci.example.com/somebuildjob/latest' etc)
spec_data = collection_artifact.load_data_from_collection_artifact(repository_spec_string)
spec_data['fetch_method'] = fetch_method
elif fetch_method == FetchMethods.REMOTE_URL:
# download the url
# hope it is a collection artifact and use load_data_from_collection_artifact() for the
# rest of the repo_spec data
log.debug('repository_spec_string: %s', repository_spec_string)
tmp_downloaded_path = download.fetch_url(repository_spec_string,
# Note: ignore_certs is meant for galaxy server,
# overloaded to apply for arbitrary http[s] downloads here
validate_certs=not galaxy_context.server['ignore_certs'])
spec_data = collection_artifact.load_data_from_collection_artifact(tmp_downloaded_path)
# pretend like this is a local_file install now
spec_data['fetch_method'] = FetchMethods.LOCAL_FILE
else:
spec_data = repository_spec_parse.spec_data_from_string(repository_spec_string,
namespace_override=namespace_override,
editable=editable)
spec_data['fetch_method'] = fetch_method
log.debug('spec_data: %s', spec_data)
req_spec = RequirementSpec.from_dict(spec_data)
req = Requirement(repository_spec=None, op=RequirementOps.EQ, requirement_spec=req_spec)
requirements_list.append(req)
log.debug('collections_lockfile_path: %s', collections_lockfile_path)
if collections_lockfile_path:
# load collections lockfile as if the 'dependencies' dict from a collection_info
collections_lockfile = load_collections_lockfile(collections_lockfile_path)
dependencies_list = requirements.from_dependencies_dict(collections_lockfile.dependencies)
# Create the CollectionsLock for the validators
collections_lock = CollectionsLock(dependencies=dependencies_list)
requirements_list.extend(collections_lock.dependencies)
log.debug('requirements_list: %s', requirements_list)
while True:
if not requirements_list:
break
display_callback('', level='info')
display_callback('Collection specs to install:', level='info')
for req in requirements_list:
if req.repository_spec:
msg = ' %s (required by %s)' % (req.requirement_spec.label, req.repository_spec)
else:
msg = ' %s' % req.requirement_spec.label
display_callback(msg, level='info')
just_installed_repositories = \
install_repositories_matching_repository_specs(galaxy_context,
requirements_list,
editable=editable,
namespace_override=namespace_override,
display_callback=display_callback,
ignore_errors=ignore_errors,
no_deps=no_deps,
force_overwrite=force_overwrite)
for just_installed_repo in just_installed_repositories:
display_callback(' Installed: %s (to %s)' %
(just_installed_repo.repository_spec,
just_installed_repo.path),
level='info')
# set the repository_specs to search for to whatever the install reported as being needed yet
# requirements_list = new_requirements_list
requirements_list = find_new_deps_from_installed(galaxy_context,
just_installed_repositories,
no_deps=no_deps)
# FIXME: what results to return?
return 0