def _get_provider(): # type () -> CollectionDependencyProviderBase
if RESOLVELIB_VERSION >= SemanticVersion("0.8.0"):
return CollectionDependencyProvider080
if RESOLVELIB_VERSION >= SemanticVersion("0.7.0"):
return CollectionDependencyProvider070
if RESOLVELIB_VERSION >= SemanticVersion("0.6.0"):
return CollectionDependencyProvider060
return CollectionDependencyProvider050
def version_added(v):
if 'version_added' in v:
version_added = v.get('version_added')
if isinstance(version_added, string_types):
# If it is not a string, schema validation will have already complained
# - or we have a float and we are in ansible/ansible, in which case we're
# also happy.
if v.get('version_added_collection') == 'ansible.builtin':
try:
version = StrictVersion()
version.parse(version_added)
except ValueError as exc:
raise _add_ansible_error_code(
Invalid(
'version_added (%r) is not a valid ansible-base version: '
'%s' % (version_added, exc)),
error_code='deprecation-either-date-or-version')
else:
try:
version = SemanticVersion()
version.parse(version_added)
except ValueError as exc:
raise _add_ansible_error_code(
Invalid(
'version_added (%r) is not a valid collection version '
'(see specification at https://semver.org/): '
'%s' % (version_added, exc)),
error_code='deprecation-either-date-or-version')
elif 'version_added_collection' in v:
# Must have been manual intervention, since version_added_collection is only
# added automatically when version_added is present
raise Invalid(
'version_added_collection cannot be specified without version_added'
)
return v
def is_pre_release(version):
# type: (str) -> bool
"""Figure out if a given version is a pre-release."""
try:
return SemanticVersion(version).is_prerelease
except ValueError:
return False
def removal_version(value, is_ansible, current_version=None, is_tombstone=False):
"""Validate a removal version string."""
msg = (
'Removal version must be a string' if is_ansible else
'Removal version must be a semantic version (https://semver.org/)'
)
if not isinstance(value, string_types):
raise Invalid(msg)
try:
if is_ansible:
version = StrictVersion()
version.parse(value)
version = LooseVersion(value) # We're storing Ansible's version as a LooseVersion
else:
version = SemanticVersion()
version.parse(value)
if version.major != 0 and (version.minor != 0 or version.patch != 0):
raise Invalid('removal_version (%r) must be a major release, not a minor or patch release '
'(see specification at https://semver.org/)' % (value, ))
if current_version is not None:
if is_tombstone:
# For a tombstone, the removal version must not be in the future
if version > current_version:
raise Invalid('The tombstone removal_version (%r) must not be after the '
'current version (%s)' % (value, current_version))
else:
# For a deprecation, the removal version must be in the future
if version <= current_version:
raise Invalid('The deprecation removal_version (%r) must be after the '
'current version (%s)' % (value, current_version))
except ValueError:
raise Invalid(msg)
return value
def check_removal_version(v, version_field, collection_name_field, error_code='invalid-removal-version'):
version = v.get(version_field)
collection_name = v.get(collection_name_field)
if not isinstance(version, string_types) or not isinstance(collection_name, string_types):
# If they are not strings, schema validation will have already complained.
return v
if collection_name == 'ansible.builtin':
try:
parsed_version = StrictVersion()
parsed_version.parse(version)
except ValueError as exc:
raise _add_ansible_error_code(
Invalid('%s (%r) is not a valid ansible-core version: %s' % (version_field, version, exc)),
error_code=error_code)
return v
try:
parsed_version = SemanticVersion()
parsed_version.parse(version)
if parsed_version.major != 0 and (parsed_version.minor != 0 or parsed_version.patch != 0):
raise _add_ansible_error_code(
Invalid('%s (%r) must be a major release, not a minor or patch release (see specification at '
'https://semver.org/)' % (version_field, version)),
error_code='removal-version-must-be-major')
except ValueError as exc:
raise _add_ansible_error_code(
Invalid('%s (%r) is not a valid collection version (see specification at https://semver.org/): '
'%s' % (version_field, version, exc)),
error_code=error_code)
return v
def set_option(self, optname, value, action=None, optdict=None):
super().set_option(optname, value, action, optdict)
if optname == 'collection-version' and value is not None:
self.collection_version = SemanticVersion(
self.config.collection_version)
if optname == 'collection-name' and value is not None:
self.collection_name = self.config.collection_name
def meets_requirements(version, requirements):
# type: (str, str) -> bool
"""Verify if a given version satisfies all the requirements.
Supported version identifiers are:
* '=='
* '!='
* '>'
* '>='
* '<'
* '<='
* '*'
Each requirement is delimited by ','.
"""
op_map = {
'!=': operator.ne,
'==': operator.eq,
'=': operator.eq,
'>=': operator.ge,
'>': operator.gt,
'<=': operator.le,
'<': operator.lt,
}
for req in requirements.split(','):
op_pos = 2 if len(req) > 1 and req[1] == '=' else 1
op = op_map.get(req[:op_pos])
requirement = req[op_pos:]
if not op:
requirement = req
op = operator.eq
if requirement == '*' or version == '*':
continue
if not op(
SemanticVersion(version),
SemanticVersion.from_loose_version(LooseVersion(requirement)),
):
break
else:
return True
# The loop was broken early, it does not meet all the requirements
return False
def semantic_version(v, error_code=None):
if not isinstance(v, string_types):
raise _add_ansible_error_code(
Invalid('Semantic version must be a string'), error_code
or 'collection-invalid-version')
try:
SemanticVersion(v)
except ValueError as e:
raise _add_ansible_error_code(
Invalid(str(e)), error_code or 'collection-invalid-version')
return v
def _check_version(self, node, version):
if not isinstance(version, str):
self.add_message('invalid-tagged-version',
node=node,
args=(version, ))
return
matcher = TAGGED_VERSION_RE.match(version)
if not matcher:
self.add_message('invalid-tagged-version',
node=node,
args=(version, ))
return
collection = matcher.group(1)
version_no = matcher.group(2)
if collection != (self.collection_name or 'ansible.builtin'):
self.add_message('wrong-collection-deprecated-version-tag',
node=node,
args=(version, ))
if collection == 'ansible.builtin':
# Ansible-base
try:
if not version_no:
raise ValueError('Version string should not be empty')
loose_version = LooseVersion(str(version_no))
if ANSIBLE_VERSION >= loose_version:
self.add_message('ansible-deprecated-version',
node=node,
args=(version, ))
except ValueError:
self.add_message('ansible-invalid-deprecated-version',
node=node,
args=(version, ))
else:
# Collections
try:
if not version_no:
raise ValueError('Version string should not be empty')
semantic_version = SemanticVersion(version_no)
if collection == self.collection_name and self.collection_version is not None:
if self.collection_version >= semantic_version:
self.add_message('collection-deprecated-version',
node=node,
args=(version, ))
except ValueError:
self.add_message('collection-invalid-deprecated-version',
node=node,
args=(version, ))
def _check_version(self, node, version, collection_name):
if not isinstance(version, (str, float)):
if collection_name == 'ansible.builtin':
symbol = 'ansible-invalid-deprecated-version'
else:
symbol = 'collection-invalid-deprecated-version'
self.add_message(symbol, node=node, args=(version, ))
return
version_no = str(version)
if collection_name == 'ansible.builtin':
# Ansible-base
try:
if not version_no:
raise ValueError('Version string should not be empty')
loose_version = LooseVersion(str(version_no))
if ANSIBLE_VERSION >= loose_version:
self.add_message('ansible-deprecated-version',
node=node,
args=(version, ))
except ValueError:
self.add_message('ansible-invalid-deprecated-version',
node=node,
args=(version, ))
elif collection_name:
# Collections
try:
if not version_no:
raise ValueError('Version string should not be empty')
semantic_version = SemanticVersion(version_no)
if collection_name == self.collection_name and self.collection_version is not None:
if self.collection_version >= semantic_version:
self.add_message('collection-deprecated-version',
node=node,
args=(version, ))
if semantic_version.major != 0 and (
semantic_version.minor != 0
or semantic_version.patch != 0):
self.add_message('removal-version-must-be-major',
node=node,
args=(version, ))
except ValueError:
self.add_message('collection-invalid-deprecated-version',
node=node,
args=(version, ))
def get_collection_version():
"""Return current collection version, or None if it is not available"""
import importlib.util
collection_detail_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'tools', 'collection_detail.py')
collection_detail_spec = importlib.util.spec_from_file_location('collection_detail', collection_detail_path)
collection_detail = importlib.util.module_from_spec(collection_detail_spec)
sys.modules['collection_detail'] = collection_detail
collection_detail_spec.loader.exec_module(collection_detail)
# noinspection PyBroadException
try:
result = collection_detail.read_manifest_json('.') or collection_detail.read_galaxy_yml('.')
return SemanticVersion(result['version'])
except Exception: # pylint: disable=broad-except
# We do not care why it fails, in case we cannot get the version
# just return None to indicate "we don't know".
return None
def version_added(v,
error_code='version-added-invalid',
accept_historical=False):
if 'version_added' in v:
version_added = v.get('version_added')
if isinstance(version_added, string_types):
# If it is not a string, schema validation will have already complained
# - or we have a float and we are in ansible/ansible, in which case we're
# also happy.
if v.get('version_added_collection') == 'ansible.builtin':
if version_added == 'historical' and accept_historical:
return v
try:
version = StrictVersion()
version.parse(version_added)
except ValueError as exc:
raise _add_ansible_error_code(Invalid(
'version_added (%r) is not a valid ansible-core version: '
'%s' % (version_added, exc)),
error_code=error_code)
else:
try:
version = SemanticVersion()
version.parse(version_added)
if version.major != 0 and version.patch != 0:
raise _add_ansible_error_code(
Invalid(
'version_added (%r) must be a major or minor release, '
'not a patch release (see specification at '
'https://semver.org/)' % (version_added, )),
error_code='version-added-must-be-major-or-minor')
except ValueError as exc:
raise _add_ansible_error_code(Invalid(
'version_added (%r) is not a valid collection version '
'(see specification at https://semver.org/): '
'%s' % (version_added, exc)),
error_code=error_code)
elif 'version_added_collection' in v:
# Must have been manual intervention, since version_added_collection is only
# added automatically when version_added is present
raise Invalid(
'version_added_collection cannot be specified without version_added'
)
return v
def removal_version(value, is_ansible):
"""Validate a removal version string."""
msg = (
'Removal version must be a string' if is_ansible else
'Removal version must be a semantic version (https://semver.org/)'
)
if not isinstance(value, string_types):
raise Invalid(msg)
try:
if is_ansible:
version = StrictVersion()
version.parse(value)
else:
version = SemanticVersion()
version.parse(value)
if version.major != 0 and (version.minor != 0 or version.patch != 0):
raise Invalid('removal_version (%r) must be a major release, not a minor or patch release '
'(see specification at https://semver.org/)' % (value, ))
except ValueError:
raise Invalid(msg)
return value
def test_from_loose_version(value, expected):
assert SemanticVersion.from_loose_version(value) == expected
def is_pre_release(version):
# type: (str) -> bool
"""Figure out if a given version is a pre-release."""
return SemanticVersion(version).is_prerelease
def collection_version(self) -> t.Optional[SemanticVersion]:
"""Return the collection version, or None if ansible-core is being tested."""
return SemanticVersion(
self.config.collection_version
) if self.config.collection_version is not None else None
('1.0.0', False),
]
STABLE = [
('1.0.0-alpha', False),
('1.0.0-alpha.1', False),
('1.0.0-0.3.7', False),
('1.0.0-x.7.z.92', False),
('0.1.2', False),
('0.1.2+bob', False),
('1.0.0', True),
('1.0.0+bob', True),
]
LOOSE_VERSION = [
(LooseVersion('1'), SemanticVersion('1.0.0')),
(LooseVersion('1-alpha'), SemanticVersion('1.0.0-alpha')),
(LooseVersion('1.0.0-alpha+build'), SemanticVersion('1.0.0-alpha+build')),
]
LOOSE_VERSION_INVALID = [
LooseVersion('1.a.3'),
LooseVersion(),
'bar',
StrictVersion('1.2.3'),
]
def test_semanticversion_none():
assert SemanticVersion().major is None
def find_matches(self, requirements):
# type: (list[Requirement]) -> list[Candidate]
r"""Find all possible candidates satisfying given requirements.
This tries to get candidates based on the requirements' types.
For concrete requirements (SCM, dir, namespace dir, local or
remote archives), the one-and-only match is returned
For a "named" requirement, Galaxy-compatible APIs are consulted
to find concrete candidates for this requirement. Of theres a
pre-installed candidate, it's prepended in front of others.
:param requirements: A collection of requirements which all of \
the returned candidates must match. \
All requirements are guaranteed to have \
the same identifier. \
The collection is never empty.
:returns: An iterable that orders candidates by preference, \
e.g. the most preferred candidate comes first.
"""
# FIXME: The first requirement may be a Git repo followed by
# FIXME: its cloned tmp dir. Using only the first one creates
# FIXME: loops that prevent any further dependency exploration.
# FIXME: We need to figure out how to prevent this.
first_req = requirements[0]
fqcn = first_req.fqcn
# The fqcn is guaranteed to be the same
version_req = "A SemVer-compliant version or '*' is required. See https://semver.org to learn how to compose it correctly. "
version_req += "This is an issue with the collection."
try:
coll_versions = self._api_proxy.get_collection_versions(first_req)
except TypeError as exc:
if first_req.is_concrete_artifact:
# Non hashable versions will cause a TypeError
raise ValueError(
f"Invalid version found for the collection '{first_req}'. {version_req}"
) from exc
# Unexpected error from a Galaxy server
raise
if first_req.is_concrete_artifact:
# FIXME: do we assume that all the following artifacts are also concrete?
# FIXME: does using fqcn==None cause us problems here?
# Ensure the version found in the concrete artifact is SemVer-compliant
for version, req_src in coll_versions:
version_err = f"Invalid version found for the collection '{first_req}': {version} ({type(version)}). {version_req}"
# NOTE: The known cases causing the version to be a non-string object come from
# NOTE: the differences in how the YAML parser normalizes ambiguous values and
# NOTE: how the end-users sometimes expect them to be parsed. Unless the users
# NOTE: explicitly use the double quotes of one of the multiline string syntaxes
# NOTE: in the collection metadata file, PyYAML will parse a value containing
# NOTE: two dot-separated integers as `float`, a single integer as `int`, and 3+
# NOTE: integers as a `str`. In some cases, they may also use an empty value
# NOTE: which is normalized as `null` and turned into `None` in the Python-land.
# NOTE: Another known mistake is setting a minor part of the SemVer notation
# NOTE: skipping the "patch" bit like "1.0" which is assumed non-compliant even
# NOTE: after the conversion to string.
if not isinstance(version, string_types):
raise ValueError(version_err)
elif version != '*':
try:
SemanticVersion(version)
except ValueError as ex:
raise ValueError(version_err) from ex
return [
Candidate(fqcn, version, _none_src_server, first_req.type,
None) for version, _none_src_server in coll_versions
]
latest_matches = []
signatures = []
extra_signature_sources = [] # type: list[str]
for version, src_server in coll_versions:
tmp_candidate = Candidate(fqcn, version, src_server, 'galaxy',
None)
unsatisfied = False
for requirement in requirements:
unsatisfied |= not self.is_satisfied_by(
requirement, tmp_candidate)
# FIXME
# unsatisfied |= not self.is_satisfied_by(requirement, tmp_candidate) or not (
# requirement.src is None or # if this is true for some candidates but not all it will break key param - Nonetype can't be compared to str
# or requirement.src == candidate.src
# )
if unsatisfied:
break
if not self._include_signatures:
continue
extra_signature_sources.extend(requirement.signature_sources
or [])
if not unsatisfied:
if self._include_signatures:
signatures = src_server.get_collection_signatures(
first_req.namespace, first_req.name, version)
for extra_source in extra_signature_sources:
signatures.append(
get_signature_from_source(extra_source))
latest_matches.append(
Candidate(fqcn, version, src_server, 'galaxy',
frozenset(signatures)))
latest_matches.sort(
key=lambda candidate: (
SemanticVersion(candidate.ver),
candidate.src,
),
reverse=True, # prefer newer versions over older ones
)
preinstalled_candidates = {
candidate
for candidate in self._preferred_candidates
if candidate.fqcn == fqcn and (
# check if an upgrade is necessary
all(
self.is_satisfied_by(requirement, candidate)
for requirement in requirements) and
(not self._upgrade or
# check if an upgrade is preferred
all(
SemanticVersion(latest.ver) <= SemanticVersion(
candidate.ver) for latest in latest_matches)))
}
return list(preinstalled_candidates) + latest_matches
from ansible.utils.version import SemanticVersion, LooseVersion
from collections.abc import Set
try:
from resolvelib import AbstractProvider
from resolvelib import __version__ as resolvelib_version
except ImportError:
class AbstractProvider: # type: ignore[no-redef]
pass
resolvelib_version = '0.0.0'
# TODO: add python requirements to ansible-test's ansible-core distribution info and remove the hardcoded lowerbound/upperbound fallback
RESOLVELIB_LOWERBOUND = SemanticVersion("0.5.3")
RESOLVELIB_UPPERBOUND = SemanticVersion("0.6.0")
RESOLVELIB_VERSION = SemanticVersion.from_loose_version(
LooseVersion(resolvelib_version))
class PinnedCandidateRequests(Set):
"""Custom set class to store Candidate objects. Excludes the 'signatures' attribute when determining if a Candidate instance is in the set."""
CANDIDATE_ATTRS = ('fqcn', 'ver', 'src', 'type')
def __init__(self, candidates):
self._candidates = set(candidates)
def __iter__(self):
return iter(self._candidates)
def find_matches(self, requirements):
# type: (List[Requirement]) -> List[Candidate]
r"""Find all possible candidates satisfying given requirements.
This tries to get candidates based on the requirements' types.
For concrete requirements (SCM, dir, namespace dir, local or
remote archives), the one-and-only match is returned
For a "named" requirement, Galaxy-compatible APIs are consulted
to find concrete candidates for this requirement. Of theres a
pre-installed candidate, it's prepended in front of others.
:param requirements: A collection of requirements which all of \
the returned candidates must match. \
All requirements are guaranteed to have \
the same identifier. \
The collection is never empty.
:returns: An iterable that orders candidates by preference, \
e.g. the most preferred candidate comes first.
"""
# FIXME: The first requirement may be a Git repo followed by
# FIXME: its cloned tmp dir. Using only the first one creates
# FIXME: loops that prevent any further dependency exploration.
# FIXME: We need to figure out how to prevent this.
first_req = requirements[0]
fqcn = first_req.fqcn
# The fqcn is guaranteed to be the same
coll_versions = self._api_proxy.get_collection_versions(first_req)
if first_req.is_concrete_artifact:
# FIXME: do we assume that all the following artifacts are also concrete?
# FIXME: does using fqcn==None cause us problems here?
return [
Candidate(fqcn, version, _none_src_server, first_req.type)
for version, _none_src_server in coll_versions
]
preinstalled_candidates = {
candidate for candidate in self._preferred_candidates
if candidate.fqcn == fqcn
}
return list(preinstalled_candidates) + sorted(
{
candidate for candidate in (
Candidate(fqcn, version, src_server, 'galaxy')
for version, src_server in coll_versions
)
if all(self.is_satisfied_by(requirement, candidate) for requirement in requirements)
# FIXME
# if all(self.is_satisfied_by(requirement, candidate) and (
# requirement.src is None or # if this is true for some candidates but not all it will break key param - Nonetype can't be compared to str
# requirement.src == candidate.src
# ))
},
key=lambda candidate: (
SemanticVersion(candidate.ver), candidate.src,
),
reverse=True, # prefer newer versions over older ones
)
def test_semanticversion_none():
assert SemanticVersion().major is None
def test_lt(left, right, expected):
assert (SemanticVersion(left) < SemanticVersion(right)) is expected
def test_ge(left, right, expected):
assert (SemanticVersion(left) >= SemanticVersion(right)) is expected
def test_valid(value):
SemanticVersion(value)
def test_comparison_with_string():
assert SemanticVersion('1.0.0') > '0.1.0'
def test_prerelease(value, expected):
assert SemanticVersion(value).is_prerelease is expected
def test_stable(value, expected):
assert SemanticVersion(value).is_stable is expected
