def load(data_or_file_object, repository_spec=None):
log.debug('START of load of requirements %s', data_or_file_object.name)
requirements_data = yaml.safe_load(data_or_file_object)
# log.debug('requirements_data: %s', pprint.pformat(requirements_data))
requirements_list = []
for req_data_item in requirements_data:
# log.debug('req_data_item: %s', req_data_item)
# log.debug('type(req_data_item): %s', type(req_data_item))
req_spec_data = yaml_parse.yaml_parse(req_data_item)
# log.debug('req_spec_data: %s', req_spec_data)
# name_info = content_name.parse_content_name(data_name)
# log.debug('data_name (after): %s', data_name)
# log.debug('name_info: %s', name_info)
req_spec = RepositorySpec.from_dict(req_spec_data)
# log.debug('req_spec: %s', req_spec)
req = Requirement(repository_spec=repository_spec,
op=RequirementOps.EQ,
requirement_spec=req_spec)
# log.debug('req: %s', req)
requirements_list.append(req)
log.debug('FINISH of load of requirements: %s: %s',
data_or_file_object.name, requirements_list)
return requirements_list
def from_dependencies_dict(dependencies_dict,
namespace_override=None,
editable=False,
repository_spec=None):
'''Build a list of Requirement objects from the 'dependencies' item in galaxy.yml'''
reqs = []
for req_label, req_version_spec in dependencies_dict.items():
req_spec_data = spec_data_from_string(
req_label,
namespace_override=namespace_override,
editable=editable)
req_spec_data['version_spec'] = req_version_spec
req_spec_data['req_spec_string'] = req_spec_data.pop(
'spec_string', None)
log.debug('req_spec_data: %s', req_spec_data)
req_spec = RequirementSpec.from_dict(req_spec_data)
log.debug('req_spec: %s', req_spec)
requirement = Requirement(repository_spec=repository_spec,
op=RequirementOps.EQ,
scope=RequirementScopes.INSTALL,
requirement_spec=req_spec)
log.debug('requirement: %s', requirement)
reqs.append(requirement)
return reqs
def load(data_or_file_object, repository_spec=None):
log.debug('START of load of requirements %s', data_or_file_object.name)
requirements_data = yaml.safe_load(data_or_file_object)
# log.debug('requirements_data: %s', pprint.pformat(requirements_data))
requirements_list = []
for req_data_item in requirements_data:
# log.debug('req_data_item: %s', req_data_item)
# log.debug('type(req_data_item): %s', type(req_data_item))
req_spec_data = yaml_parse.yaml_parse(req_data_item)
# FIXME: find a better place to do this
# create a version_spec from the 'version' field, a requiremenst.yml
# does not currently expect to have a 'version_spec' data in it
version_spec_str = '*'
version_str = req_spec_data.pop('version')
if version_str:
version_spec_str = '==%s' % version_str
version_spec = semantic_version.Spec(version_spec_str)
req_spec_data['version_spec'] = version_spec
# log.debug('data_name (after): %s', data_name)
# log.debug('name_info: %s', name_info)
req_spec = RequirementSpec.from_dict(req_spec_data)
# log.debug('req_spec: %s', req_spec)
req = Requirement(repository_spec=repository_spec,
op=RequirementOps.EQ,
requirement_spec=req_spec)
# log.debug('req: %s', req)
requirements_list.append(req)
log.debug('FINISH of load of requirements: %s: %s',
data_or_file_object.name, requirements_list)
return requirements_list
def from_dependency_spec_strings(dependency_spec_strings, namespace_override=None, editable=False):
deps = []
for dep_spec_string in dependency_spec_strings:
dep_spec_data = spec_data_from_string(dep_spec_string)
log.debug('dep_spec_data: %s', dep_spec_data)
dep_spec = RequirementSpec.from_dict(dep_spec_data)
log.debug('dep_spec: %s', dep_spec)
# Add a requirement, but with the 'RUNTIME' scope
requirement = Requirement(repository_spec=None, op=RequirementOps.EQ,
scope=RequirementScopes.RUNTIME,
requirement_spec=dep_spec)
deps.append(requirement)
return deps
def from_requirement_spec_strings(requirement_spec_strings,
namespace_override=None,
editable=False,
repository_spec=None):
reqs = []
for requirement_spec_string in requirement_spec_strings:
req_spec_data = spec_data_from_string(
requirement_spec_string,
namespace_override=namespace_override,
editable=editable)
req_spec = RequirementSpec.from_dict(req_spec_data)
req = Requirement(repository_spec=repository_spec,
op=RequirementOps.EQ,
requirement_spec=req_spec)
reqs.append(req)
return reqs
def test_find_new_deps_from_installed(galaxy_context):
repo_spec = RepositorySpec(namespace='some_namespace',
name='some_name',
version='4.3.2')
req_spec = RepositorySpec(namespace='some_required_namespace',
name='some_required_name',
version='1.0.0')
some_requirement = Requirement(repository_spec=repo_spec,
op=RequirementOps.EQ,
requirement_spec=req_spec)
installed_repo = Repository(repo_spec, requirements=[some_requirement, some_requirement])
res = install.find_new_deps_from_installed(galaxy_context, [installed_repo])
log.debug('res: %s', res)
assert isinstance(res, list)
assert isinstance(res[0], Requirement)
assert res[0].requirement_spec == req_spec
def install_repository_specs_loop(galaxy_context,
repository_spec_strings=None,
requirements_list=None,
collections_lockfile_path=None,
editable=False,
namespace_override=None,
display_callback=None,
# TODO: error handling callback ?
ignore_errors=False,
no_deps=False,
force_overwrite=False):
requirements_list = requirements_list or []
for repository_spec_string in repository_spec_strings:
fetch_method = \
repository_spec_parse.choose_repository_fetch_method(repository_spec_string,
editable=editable)
log.debug('fetch_method: %s', fetch_method)
if fetch_method == FetchMethods.LOCAL_FILE:
# Since we only know this is a local file we vaguely recognize, we have to
# open it up to get any more details. We _could_ attempt to parse the file
# name, but that rarely ends well. Filename could also be arbitrary for downloads
# from remote urls ('mazer install http://myci.example.com/somebuildjob/latest' etc)
spec_data = collection_artifact.load_data_from_collection_artifact(repository_spec_string)
spec_data['fetch_method'] = fetch_method
elif fetch_method == FetchMethods.REMOTE_URL:
# download the url
# hope it is a collection artifact and use load_data_from_collection_artifact() for the
# rest of the repo_spec data
log.debug('repository_spec_string: %s', repository_spec_string)
tmp_downloaded_path = download.fetch_url(repository_spec_string,
# Note: ignore_certs is meant for galaxy server,
# overloaded to apply for arbitrary http[s] downloads here
validate_certs=not galaxy_context.server['ignore_certs'])
spec_data = collection_artifact.load_data_from_collection_artifact(tmp_downloaded_path)
# pretend like this is a local_file install now
spec_data['fetch_method'] = FetchMethods.LOCAL_FILE
else:
spec_data = repository_spec_parse.spec_data_from_string(repository_spec_string,
namespace_override=namespace_override,
editable=editable)
spec_data['fetch_method'] = fetch_method
log.debug('spec_data: %s', spec_data)
req_spec = RequirementSpec.from_dict(spec_data)
req = Requirement(repository_spec=None, op=RequirementOps.EQ, requirement_spec=req_spec)
requirements_list.append(req)
log.debug('collections_lockfile_path: %s', collections_lockfile_path)
if collections_lockfile_path:
# load collections lockfile as if the 'dependencies' dict from a collection_info
collections_lockfile = load_collections_lockfile(collections_lockfile_path)
dependencies_list = requirements.from_dependencies_dict(collections_lockfile.dependencies)
# Create the CollectionsLock for the validators
collections_lock = CollectionsLock(dependencies=dependencies_list)
requirements_list.extend(collections_lock.dependencies)
log.debug('requirements_list: %s', requirements_list)
while True:
if not requirements_list:
break
display_callback('', level='info')
display_callback('Collection specs to install:', level='info')
for req in requirements_list:
if req.repository_spec:
msg = ' %s (required by %s)' % (req.requirement_spec.label, req.repository_spec)
else:
msg = ' %s' % req.requirement_spec.label
display_callback(msg, level='info')
just_installed_repositories = \
install_repositories_matching_repository_specs(galaxy_context,
requirements_list,
editable=editable,
namespace_override=namespace_override,
display_callback=display_callback,
ignore_errors=ignore_errors,
no_deps=no_deps,
force_overwrite=force_overwrite)
for just_installed_repo in just_installed_repositories:
display_callback(' Installed: %s (to %s)' %
(just_installed_repo.repository_spec,
just_installed_repo.path),
level='info')
# set the repository_specs to search for to whatever the install reported as being needed yet
# requirements_list = new_requirements_list
requirements_list = find_new_deps_from_installed(galaxy_context,
just_installed_repositories,
no_deps=no_deps)
# FIXME: what results to return?
return 0
def install_repository_specs_loop(
galaxy_context,
repository_spec_strings=None,
requirements_list=None,
editable=False,
namespace_override=None,
display_callback=None,
# TODO: error handling callback ?
ignore_errors=False,
no_deps=False,
force_overwrite=False):
requirements_list = requirements_list or []
for repository_spec_string in repository_spec_strings:
fetch_method = \
repository_spec.choose_repository_fetch_method(repository_spec_string,
editable=editable)
log.debug('fetch_method: %s', fetch_method)
if fetch_method == FetchMethods.LOCAL_FILE:
# Since only know this is a local file we vaguely recognize, we have to
# open it up to get any more details. We _could_ attempt to parse the file
# name, but that rarely ends well...
spec_data = collection_artifact.load_data_from_collection_artifact(
repository_spec_string)
spec_data['fetch_method'] = fetch_method
else:
spec_data = repository_spec.spec_data_from_string(
repository_spec_string,
namespace_override=namespace_override,
editable=editable)
spec_data['fetch_method'] = fetch_method
log.debug('spec_data: %s', spec_data)
req_spec = RepositorySpec.from_dict(spec_data)
req = Requirement(repository_spec=None,
op=RequirementOps.EQ,
requirement_spec=req_spec)
requirements_list.append(req)
log.debug('requirements_list: %s', requirements_list)
for req in requirements_list:
display_callback('Installing %s' % req.requirement_spec.label,
level='info')
while True:
if not requirements_list:
break
just_installed_repositories = \
install_repositories_matching_repository_specs(galaxy_context,
requirements_list,
editable=editable,
namespace_override=namespace_override,
display_callback=display_callback,
ignore_errors=ignore_errors,
no_deps=no_deps,
force_overwrite=force_overwrite)
# set the repository_specs to search for to whatever the install reported as being needed yet
# requirements_list = new_requirements_list
requirements_list = find_new_deps_from_installed(
galaxy_context, just_installed_repositories, no_deps=no_deps)
for req in requirements_list:
if req.repository_spec:
msg = 'Installing requirement %s (required by %s)' % (
req.requirement_spec.label, req.repository_spec.label)
else:
msg = 'Installing requirement %s' % req.requirement_spec.label
display_callback(msg, level='info')
# FIXME: what results to return?
return 0