def _get_provider(): # type () -> CollectionDependencyProviderBase if RESOLVELIB_VERSION >= SemanticVersion("0.8.0"): return CollectionDependencyProvider080 if RESOLVELIB_VERSION >= SemanticVersion("0.7.0"): return CollectionDependencyProvider070 if RESOLVELIB_VERSION >= SemanticVersion("0.6.0"): return CollectionDependencyProvider060 return CollectionDependencyProvider050
def version_added(v): if 'version_added' in v: version_added = v.get('version_added') if isinstance(version_added, string_types): # If it is not a string, schema validation will have already complained # - or we have a float and we are in ansible/ansible, in which case we're # also happy. if v.get('version_added_collection') == 'ansible.builtin': try: version = StrictVersion() version.parse(version_added) except ValueError as exc: raise _add_ansible_error_code( Invalid( 'version_added (%r) is not a valid ansible-base version: ' '%s' % (version_added, exc)), error_code='deprecation-either-date-or-version') else: try: version = SemanticVersion() version.parse(version_added) except ValueError as exc: raise _add_ansible_error_code( Invalid( 'version_added (%r) is not a valid collection version ' '(see specification at https://semver.org/): ' '%s' % (version_added, exc)), error_code='deprecation-either-date-or-version') elif 'version_added_collection' in v: # Must have been manual intervention, since version_added_collection is only # added automatically when version_added is present raise Invalid( 'version_added_collection cannot be specified without version_added' ) return v
def is_pre_release(version): # type: (str) -> bool """Figure out if a given version is a pre-release.""" try: return SemanticVersion(version).is_prerelease except ValueError: return False
def removal_version(value, is_ansible, current_version=None, is_tombstone=False): """Validate a removal version string.""" msg = ( 'Removal version must be a string' if is_ansible else 'Removal version must be a semantic version (https://semver.org/)' ) if not isinstance(value, string_types): raise Invalid(msg) try: if is_ansible: version = StrictVersion() version.parse(value) version = LooseVersion(value) # We're storing Ansible's version as a LooseVersion else: version = SemanticVersion() version.parse(value) if version.major != 0 and (version.minor != 0 or version.patch != 0): raise Invalid('removal_version (%r) must be a major release, not a minor or patch release ' '(see specification at https://semver.org/)' % (value, )) if current_version is not None: if is_tombstone: # For a tombstone, the removal version must not be in the future if version > current_version: raise Invalid('The tombstone removal_version (%r) must not be after the ' 'current version (%s)' % (value, current_version)) else: # For a deprecation, the removal version must be in the future if version <= current_version: raise Invalid('The deprecation removal_version (%r) must be after the ' 'current version (%s)' % (value, current_version)) except ValueError: raise Invalid(msg) return value
def check_removal_version(v, version_field, collection_name_field, error_code='invalid-removal-version'): version = v.get(version_field) collection_name = v.get(collection_name_field) if not isinstance(version, string_types) or not isinstance(collection_name, string_types): # If they are not strings, schema validation will have already complained. return v if collection_name == 'ansible.builtin': try: parsed_version = StrictVersion() parsed_version.parse(version) except ValueError as exc: raise _add_ansible_error_code( Invalid('%s (%r) is not a valid ansible-core version: %s' % (version_field, version, exc)), error_code=error_code) return v try: parsed_version = SemanticVersion() parsed_version.parse(version) if parsed_version.major != 0 and (parsed_version.minor != 0 or parsed_version.patch != 0): raise _add_ansible_error_code( Invalid('%s (%r) must be a major release, not a minor or patch release (see specification at ' 'https://semver.org/)' % (version_field, version)), error_code='removal-version-must-be-major') except ValueError as exc: raise _add_ansible_error_code( Invalid('%s (%r) is not a valid collection version (see specification at https://semver.org/): ' '%s' % (version_field, version, exc)), error_code=error_code) return v
def set_option(self, optname, value, action=None, optdict=None): super().set_option(optname, value, action, optdict) if optname == 'collection-version' and value is not None: self.collection_version = SemanticVersion( self.config.collection_version) if optname == 'collection-name' and value is not None: self.collection_name = self.config.collection_name
def meets_requirements(version, requirements): # type: (str, str) -> bool """Verify if a given version satisfies all the requirements. Supported version identifiers are: * '==' * '!=' * '>' * '>=' * '<' * '<=' * '*' Each requirement is delimited by ','. """ op_map = { '!=': operator.ne, '==': operator.eq, '=': operator.eq, '>=': operator.ge, '>': operator.gt, '<=': operator.le, '<': operator.lt, } for req in requirements.split(','): op_pos = 2 if len(req) > 1 and req[1] == '=' else 1 op = op_map.get(req[:op_pos]) requirement = req[op_pos:] if not op: requirement = req op = operator.eq if requirement == '*' or version == '*': continue if not op( SemanticVersion(version), SemanticVersion.from_loose_version(LooseVersion(requirement)), ): break else: return True # The loop was broken early, it does not meet all the requirements return False
def semantic_version(v, error_code=None): if not isinstance(v, string_types): raise _add_ansible_error_code( Invalid('Semantic version must be a string'), error_code or 'collection-invalid-version') try: SemanticVersion(v) except ValueError as e: raise _add_ansible_error_code( Invalid(str(e)), error_code or 'collection-invalid-version') return v
def _check_version(self, node, version): if not isinstance(version, str): self.add_message('invalid-tagged-version', node=node, args=(version, )) return matcher = TAGGED_VERSION_RE.match(version) if not matcher: self.add_message('invalid-tagged-version', node=node, args=(version, )) return collection = matcher.group(1) version_no = matcher.group(2) if collection != (self.collection_name or 'ansible.builtin'): self.add_message('wrong-collection-deprecated-version-tag', node=node, args=(version, )) if collection == 'ansible.builtin': # Ansible-base try: if not version_no: raise ValueError('Version string should not be empty') loose_version = LooseVersion(str(version_no)) if ANSIBLE_VERSION >= loose_version: self.add_message('ansible-deprecated-version', node=node, args=(version, )) except ValueError: self.add_message('ansible-invalid-deprecated-version', node=node, args=(version, )) else: # Collections try: if not version_no: raise ValueError('Version string should not be empty') semantic_version = SemanticVersion(version_no) if collection == self.collection_name and self.collection_version is not None: if self.collection_version >= semantic_version: self.add_message('collection-deprecated-version', node=node, args=(version, )) except ValueError: self.add_message('collection-invalid-deprecated-version', node=node, args=(version, ))
def _check_version(self, node, version, collection_name): if not isinstance(version, (str, float)): if collection_name == 'ansible.builtin': symbol = 'ansible-invalid-deprecated-version' else: symbol = 'collection-invalid-deprecated-version' self.add_message(symbol, node=node, args=(version, )) return version_no = str(version) if collection_name == 'ansible.builtin': # Ansible-base try: if not version_no: raise ValueError('Version string should not be empty') loose_version = LooseVersion(str(version_no)) if ANSIBLE_VERSION >= loose_version: self.add_message('ansible-deprecated-version', node=node, args=(version, )) except ValueError: self.add_message('ansible-invalid-deprecated-version', node=node, args=(version, )) elif collection_name: # Collections try: if not version_no: raise ValueError('Version string should not be empty') semantic_version = SemanticVersion(version_no) if collection_name == self.collection_name and self.collection_version is not None: if self.collection_version >= semantic_version: self.add_message('collection-deprecated-version', node=node, args=(version, )) if semantic_version.major != 0 and ( semantic_version.minor != 0 or semantic_version.patch != 0): self.add_message('removal-version-must-be-major', node=node, args=(version, )) except ValueError: self.add_message('collection-invalid-deprecated-version', node=node, args=(version, ))
def get_collection_version(): """Return current collection version, or None if it is not available""" import importlib.util collection_detail_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'tools', 'collection_detail.py') collection_detail_spec = importlib.util.spec_from_file_location('collection_detail', collection_detail_path) collection_detail = importlib.util.module_from_spec(collection_detail_spec) sys.modules['collection_detail'] = collection_detail collection_detail_spec.loader.exec_module(collection_detail) # noinspection PyBroadException try: result = collection_detail.read_manifest_json('.') or collection_detail.read_galaxy_yml('.') return SemanticVersion(result['version']) except Exception: # pylint: disable=broad-except # We do not care why it fails, in case we cannot get the version # just return None to indicate "we don't know". return None
def version_added(v, error_code='version-added-invalid', accept_historical=False): if 'version_added' in v: version_added = v.get('version_added') if isinstance(version_added, string_types): # If it is not a string, schema validation will have already complained # - or we have a float and we are in ansible/ansible, in which case we're # also happy. if v.get('version_added_collection') == 'ansible.builtin': if version_added == 'historical' and accept_historical: return v try: version = StrictVersion() version.parse(version_added) except ValueError as exc: raise _add_ansible_error_code(Invalid( 'version_added (%r) is not a valid ansible-core version: ' '%s' % (version_added, exc)), error_code=error_code) else: try: version = SemanticVersion() version.parse(version_added) if version.major != 0 and version.patch != 0: raise _add_ansible_error_code( Invalid( 'version_added (%r) must be a major or minor release, ' 'not a patch release (see specification at ' 'https://semver.org/)' % (version_added, )), error_code='version-added-must-be-major-or-minor') except ValueError as exc: raise _add_ansible_error_code(Invalid( 'version_added (%r) is not a valid collection version ' '(see specification at https://semver.org/): ' '%s' % (version_added, exc)), error_code=error_code) elif 'version_added_collection' in v: # Must have been manual intervention, since version_added_collection is only # added automatically when version_added is present raise Invalid( 'version_added_collection cannot be specified without version_added' ) return v
def removal_version(value, is_ansible): """Validate a removal version string.""" msg = ( 'Removal version must be a string' if is_ansible else 'Removal version must be a semantic version (https://semver.org/)' ) if not isinstance(value, string_types): raise Invalid(msg) try: if is_ansible: version = StrictVersion() version.parse(value) else: version = SemanticVersion() version.parse(value) if version.major != 0 and (version.minor != 0 or version.patch != 0): raise Invalid('removal_version (%r) must be a major release, not a minor or patch release ' '(see specification at https://semver.org/)' % (value, )) except ValueError: raise Invalid(msg) return value
def test_from_loose_version(value, expected): assert SemanticVersion.from_loose_version(value) == expected
def is_pre_release(version): # type: (str) -> bool """Figure out if a given version is a pre-release.""" return SemanticVersion(version).is_prerelease
def collection_version(self) -> t.Optional[SemanticVersion]: """Return the collection version, or None if ansible-core is being tested.""" return SemanticVersion( self.config.collection_version ) if self.config.collection_version is not None else None
('1.0.0', False), ] STABLE = [ ('1.0.0-alpha', False), ('1.0.0-alpha.1', False), ('1.0.0-0.3.7', False), ('1.0.0-x.7.z.92', False), ('0.1.2', False), ('0.1.2+bob', False), ('1.0.0', True), ('1.0.0+bob', True), ] LOOSE_VERSION = [ (LooseVersion('1'), SemanticVersion('1.0.0')), (LooseVersion('1-alpha'), SemanticVersion('1.0.0-alpha')), (LooseVersion('1.0.0-alpha+build'), SemanticVersion('1.0.0-alpha+build')), ] LOOSE_VERSION_INVALID = [ LooseVersion('1.a.3'), LooseVersion(), 'bar', StrictVersion('1.2.3'), ] def test_semanticversion_none(): assert SemanticVersion().major is None
def find_matches(self, requirements): # type: (list[Requirement]) -> list[Candidate] r"""Find all possible candidates satisfying given requirements. This tries to get candidates based on the requirements' types. For concrete requirements (SCM, dir, namespace dir, local or remote archives), the one-and-only match is returned For a "named" requirement, Galaxy-compatible APIs are consulted to find concrete candidates for this requirement. Of theres a pre-installed candidate, it's prepended in front of others. :param requirements: A collection of requirements which all of \ the returned candidates must match. \ All requirements are guaranteed to have \ the same identifier. \ The collection is never empty. :returns: An iterable that orders candidates by preference, \ e.g. the most preferred candidate comes first. """ # FIXME: The first requirement may be a Git repo followed by # FIXME: its cloned tmp dir. Using only the first one creates # FIXME: loops that prevent any further dependency exploration. # FIXME: We need to figure out how to prevent this. first_req = requirements[0] fqcn = first_req.fqcn # The fqcn is guaranteed to be the same version_req = "A SemVer-compliant version or '*' is required. See https://semver.org to learn how to compose it correctly. " version_req += "This is an issue with the collection." try: coll_versions = self._api_proxy.get_collection_versions(first_req) except TypeError as exc: if first_req.is_concrete_artifact: # Non hashable versions will cause a TypeError raise ValueError( f"Invalid version found for the collection '{first_req}'. {version_req}" ) from exc # Unexpected error from a Galaxy server raise if first_req.is_concrete_artifact: # FIXME: do we assume that all the following artifacts are also concrete? # FIXME: does using fqcn==None cause us problems here? # Ensure the version found in the concrete artifact is SemVer-compliant for version, req_src in coll_versions: version_err = f"Invalid version found for the collection '{first_req}': {version} ({type(version)}). {version_req}" # NOTE: The known cases causing the version to be a non-string object come from # NOTE: the differences in how the YAML parser normalizes ambiguous values and # NOTE: how the end-users sometimes expect them to be parsed. Unless the users # NOTE: explicitly use the double quotes of one of the multiline string syntaxes # NOTE: in the collection metadata file, PyYAML will parse a value containing # NOTE: two dot-separated integers as `float`, a single integer as `int`, and 3+ # NOTE: integers as a `str`. In some cases, they may also use an empty value # NOTE: which is normalized as `null` and turned into `None` in the Python-land. # NOTE: Another known mistake is setting a minor part of the SemVer notation # NOTE: skipping the "patch" bit like "1.0" which is assumed non-compliant even # NOTE: after the conversion to string. if not isinstance(version, string_types): raise ValueError(version_err) elif version != '*': try: SemanticVersion(version) except ValueError as ex: raise ValueError(version_err) from ex return [ Candidate(fqcn, version, _none_src_server, first_req.type, None) for version, _none_src_server in coll_versions ] latest_matches = [] signatures = [] extra_signature_sources = [] # type: list[str] for version, src_server in coll_versions: tmp_candidate = Candidate(fqcn, version, src_server, 'galaxy', None) unsatisfied = False for requirement in requirements: unsatisfied |= not self.is_satisfied_by( requirement, tmp_candidate) # FIXME # unsatisfied |= not self.is_satisfied_by(requirement, tmp_candidate) or not ( # requirement.src is None or # if this is true for some candidates but not all it will break key param - Nonetype can't be compared to str # or requirement.src == candidate.src # ) if unsatisfied: break if not self._include_signatures: continue extra_signature_sources.extend(requirement.signature_sources or []) if not unsatisfied: if self._include_signatures: signatures = src_server.get_collection_signatures( first_req.namespace, first_req.name, version) for extra_source in extra_signature_sources: signatures.append( get_signature_from_source(extra_source)) latest_matches.append( Candidate(fqcn, version, src_server, 'galaxy', frozenset(signatures))) latest_matches.sort( key=lambda candidate: ( SemanticVersion(candidate.ver), candidate.src, ), reverse=True, # prefer newer versions over older ones ) preinstalled_candidates = { candidate for candidate in self._preferred_candidates if candidate.fqcn == fqcn and ( # check if an upgrade is necessary all( self.is_satisfied_by(requirement, candidate) for requirement in requirements) and (not self._upgrade or # check if an upgrade is preferred all( SemanticVersion(latest.ver) <= SemanticVersion( candidate.ver) for latest in latest_matches))) } return list(preinstalled_candidates) + latest_matches
from ansible.utils.version import SemanticVersion, LooseVersion from collections.abc import Set try: from resolvelib import AbstractProvider from resolvelib import __version__ as resolvelib_version except ImportError: class AbstractProvider: # type: ignore[no-redef] pass resolvelib_version = '0.0.0' # TODO: add python requirements to ansible-test's ansible-core distribution info and remove the hardcoded lowerbound/upperbound fallback RESOLVELIB_LOWERBOUND = SemanticVersion("0.5.3") RESOLVELIB_UPPERBOUND = SemanticVersion("0.6.0") RESOLVELIB_VERSION = SemanticVersion.from_loose_version( LooseVersion(resolvelib_version)) class PinnedCandidateRequests(Set): """Custom set class to store Candidate objects. Excludes the 'signatures' attribute when determining if a Candidate instance is in the set.""" CANDIDATE_ATTRS = ('fqcn', 'ver', 'src', 'type') def __init__(self, candidates): self._candidates = set(candidates) def __iter__(self): return iter(self._candidates)
def find_matches(self, requirements): # type: (List[Requirement]) -> List[Candidate] r"""Find all possible candidates satisfying given requirements. This tries to get candidates based on the requirements' types. For concrete requirements (SCM, dir, namespace dir, local or remote archives), the one-and-only match is returned For a "named" requirement, Galaxy-compatible APIs are consulted to find concrete candidates for this requirement. Of theres a pre-installed candidate, it's prepended in front of others. :param requirements: A collection of requirements which all of \ the returned candidates must match. \ All requirements are guaranteed to have \ the same identifier. \ The collection is never empty. :returns: An iterable that orders candidates by preference, \ e.g. the most preferred candidate comes first. """ # FIXME: The first requirement may be a Git repo followed by # FIXME: its cloned tmp dir. Using only the first one creates # FIXME: loops that prevent any further dependency exploration. # FIXME: We need to figure out how to prevent this. first_req = requirements[0] fqcn = first_req.fqcn # The fqcn is guaranteed to be the same coll_versions = self._api_proxy.get_collection_versions(first_req) if first_req.is_concrete_artifact: # FIXME: do we assume that all the following artifacts are also concrete? # FIXME: does using fqcn==None cause us problems here? return [ Candidate(fqcn, version, _none_src_server, first_req.type) for version, _none_src_server in coll_versions ] preinstalled_candidates = { candidate for candidate in self._preferred_candidates if candidate.fqcn == fqcn } return list(preinstalled_candidates) + sorted( { candidate for candidate in ( Candidate(fqcn, version, src_server, 'galaxy') for version, src_server in coll_versions ) if all(self.is_satisfied_by(requirement, candidate) for requirement in requirements) # FIXME # if all(self.is_satisfied_by(requirement, candidate) and ( # requirement.src is None or # if this is true for some candidates but not all it will break key param - Nonetype can't be compared to str # requirement.src == candidate.src # )) }, key=lambda candidate: ( SemanticVersion(candidate.ver), candidate.src, ), reverse=True, # prefer newer versions over older ones )
def test_semanticversion_none(): assert SemanticVersion().major is None
def test_lt(left, right, expected): assert (SemanticVersion(left) < SemanticVersion(right)) is expected
def test_ge(left, right, expected): assert (SemanticVersion(left) >= SemanticVersion(right)) is expected
def test_valid(value): SemanticVersion(value)
def test_comparison_with_string(): assert SemanticVersion('1.0.0') > '0.1.0'
def test_prerelease(value, expected): assert SemanticVersion(value).is_prerelease is expected
def test_stable(value, expected): assert SemanticVersion(value).is_stable is expected