def test_install_installed_collection(monkeypatch, tmp_path_factory, galaxy_server): mock_installed_collections = MagicMock( return_value=[Candidate('namespace.collection', '1.2.3', None, 'dir')]) monkeypatch.setattr(collection, 'find_existing_collections', mock_installed_collections) test_dir = to_text(tmp_path_factory.mktemp('test-ÅÑŚÌβŁÈ Collections')) concrete_artifact_cm = collection.concrete_artifact_manager.ConcreteArtifactsManager( test_dir, validate_certs=False) mock_display = MagicMock() monkeypatch.setattr(Display, 'display', mock_display) mock_get_info = MagicMock() mock_get_info.return_value = api.CollectionVersionMetadata( 'namespace', 'collection', '1.2.3', None, None, {}) monkeypatch.setattr(galaxy_server, 'get_collection_version_metadata', mock_get_info) mock_get_versions = MagicMock(return_value=['1.2.3', '1.3.0']) monkeypatch.setattr(galaxy_server, 'get_collection_versions', mock_get_versions) cli = GalaxyCLI(args=[ 'ansible-galaxy', 'collection', 'install', 'namespace.collection' ]) cli.run() expected = "Nothing to do. All requested collections are already installed. If you want to reinstall them, consider using `--force`." assert mock_display.mock_calls[1][1][0] == expected
def _is_user_requested(self, candidate): # type: (Candidate) -> bool """Check if the candidate is requested by the user.""" if candidate in self._pinned_candidate_requests: return True if candidate.is_online_index_pointer and candidate.src is not None: # NOTE: Candidate is a namedtuple, it has a source server set # NOTE: to a specific GalaxyAPI instance or `None`. When the # NOTE: user runs # NOTE: # NOTE: $ ansible-galaxy collection install ns.coll # NOTE: # NOTE: then it's saved in `self._pinned_candidate_requests` # NOTE: as `('ns.coll', '*', None, 'galaxy')` but then # NOTE: `self.find_matches()` calls `self.is_satisfied_by()` # NOTE: with Candidate instances bound to each specific # NOTE: server available, those look like # NOTE: `('ns.coll', '*', GalaxyAPI(...), 'galaxy')` and # NOTE: wouldn't match the user requests saved in # NOTE: `self._pinned_candidate_requests`. This is why we # NOTE: normalize the collection to have `src=None` and try # NOTE: again. # NOTE: # NOTE: When the user request comes from `requirements.yml` # NOTE: with the `source:` set, it'll match the first check # NOTE: but it still can have entries with `src=None` so this # NOTE: normalized check is still necessary. # NOTE: # NOTE: User-provided signatures are supplemental, so signatures # NOTE: are not used to determine if a candidate is user-requested return Candidate(candidate.fqcn, candidate.ver, None, candidate.type, None) in self._pinned_candidate_requests return False
def test_install_collection_with_download(galaxy_server, collection_artifact, monkeypatch): collection_path, collection_tar = collection_artifact shutil.rmtree(collection_path) collections_dir = ('%s' % os.path.sep).join(to_text(collection_path).split('%s' % os.path.sep)[:-2]) temp_path = os.path.join(os.path.split(collection_tar)[0], b'temp') os.makedirs(temp_path) mock_display = MagicMock() monkeypatch.setattr(Display, 'display', mock_display) concrete_artifact_cm = collection.concrete_artifact_manager.ConcreteArtifactsManager(temp_path, validate_certs=False) mock_download = MagicMock() mock_download.return_value = collection_tar monkeypatch.setattr(concrete_artifact_cm, 'get_galaxy_artifact_path', mock_download) req = Candidate('ansible_namespace.collection', '0.1.0', 'https://downloadme.com', 'galaxy', None) collection.install(req, to_text(collections_dir), concrete_artifact_cm) actual_files = os.listdir(collection_path) actual_files.sort() assert actual_files == [b'FILES.json', b'MANIFEST.json', b'README.md', b'docs', b'playbooks', b'plugins', b'roles', b'runme.sh'] assert mock_display.call_count == 2 assert mock_display.mock_calls[0][1][0] == "Installing 'ansible_namespace.collection:0.1.0' to '%s'" \ % to_text(collection_path) assert mock_display.mock_calls[1][1][0] == "ansible_namespace.collection:0.1.0 was installed successfully" assert mock_download.call_count == 1 assert mock_download.mock_calls[0][1][0].src == 'https://downloadme.com' assert mock_download.mock_calls[0][1][0].type == 'galaxy'
def test_install_collection(collection_artifact, monkeypatch): mock_display = MagicMock() monkeypatch.setattr(Display, 'display', mock_display) collection_tar = collection_artifact[1] temp_path = os.path.join(os.path.split(collection_tar)[0], b'temp') os.makedirs(temp_path) concrete_artifact_cm = collection.concrete_artifact_manager.ConcreteArtifactsManager(temp_path, validate_certs=False) output_path = os.path.join(os.path.split(collection_tar)[0]) collection_path = os.path.join(output_path, b'ansible_namespace', b'collection') os.makedirs(os.path.join(collection_path, b'delete_me')) # Create a folder to verify the install cleans out the dir candidate = Candidate('ansible_namespace.collection', '0.1.0', to_text(collection_tar), 'file', None) collection.install(candidate, to_text(output_path), concrete_artifact_cm) # Ensure the temp directory is empty, nothing is left behind assert os.listdir(temp_path) == [] actual_files = os.listdir(collection_path) actual_files.sort() assert actual_files == [b'FILES.json', b'MANIFEST.json', b'README.md', b'docs', b'playbooks', b'plugins', b'roles', b'runme.sh'] assert stat.S_IMODE(os.stat(os.path.join(collection_path, b'plugins')).st_mode) == 0o0755 assert stat.S_IMODE(os.stat(os.path.join(collection_path, b'README.md')).st_mode) == 0o0644 assert stat.S_IMODE(os.stat(os.path.join(collection_path, b'runme.sh')).st_mode) == 0o0755 assert mock_display.call_count == 2 assert mock_display.mock_calls[0][1][0] == "Installing 'ansible_namespace.collection:0.1.0' to '%s'" \ % to_text(collection_path) assert mock_display.mock_calls[1][1][0] == "ansible_namespace.collection:0.1.0 was installed successfully"
def __init__( self, # type: CollectionDependencyProvider apis, # type: MultiGalaxyAPIProxy concrete_artifacts_manager=None, # type: ConcreteArtifactsManager user_requirements=None, # type: Iterable[Requirement] preferred_candidates=None, # type: Iterable[Candidate] with_deps=True, # type: bool with_pre_releases=False, # type: bool upgrade=False, # type: bool include_signatures=True, # type: bool ): # type: (...) -> None r"""Initialize helper attributes. :param api: An instance of the multiple Galaxy APIs wrapper. :param concrete_artifacts_manager: An instance of the caching \ concrete artifacts manager. :param with_deps: A flag specifying whether the resolver \ should attempt to pull-in the deps of the \ requested requirements. On by default. :param with_pre_releases: A flag specifying whether the \ resolver should skip pre-releases. \ Off by default. :param upgrade: A flag specifying whether the resolver should \ skip matching versions that are not upgrades. \ Off by default. :param include_signatures: A flag to determine whether to retrieve \ signatures from the Galaxy APIs and \ include signatures in matching Candidates. \ On by default. """ self._api_proxy = apis self._make_req_from_dict = functools.partial( Requirement.from_requirement_dict, art_mgr=concrete_artifacts_manager, ) self._pinned_candidate_requests = PinnedCandidateRequests( # NOTE: User-provided signatures are supplemental, so signatures # NOTE: are not used to determine if a candidate is user-requested Candidate(req.fqcn, req.ver, req.src, req.type, None) for req in (user_requirements or ()) if req.is_concrete_artifact or ( req.ver != '*' and not req.ver.startswith(('<', '>', '!=')) ) ) self._preferred_candidates = set(preferred_candidates or ()) self._with_deps = with_deps self._with_pre_releases = with_pre_releases self._upgrade = upgrade self._include_signatures = include_signatures
def __init__( self, # type: CollectionDependencyProvider apis, # type: MultiGalaxyAPIProxy concrete_artifacts_manager=None, # type: ConcreteArtifactsManager user_requirements=None, # type: Iterable[Requirement] preferred_candidates=None, # type: Iterable[Candidate] with_deps=True, # type: bool with_pre_releases=False, # type: bool ): # type: (...) -> None r"""Initialize helper attributes. :param api: An instance of the multiple Galaxy APIs wrapper. :param concrete_artifacts_manager: An instance of the caching \ concrete artifacts manager. :param with_deps: A flag specifying whether the resolver \ should attempt to pull-in the deps of the \ requested requirements. On by default. :param with_pre_releases: A flag specifying whether the \ resolver should skip pre-releases. \ Off by default. """ self._api_proxy = apis self._make_req_from_dict = functools.partial( Requirement.from_requirement_dict, art_mgr=concrete_artifacts_manager, ) self._pinned_candidate_requests = set( Candidate(req.fqcn, req.ver, req.src, req.type) for req in (user_requirements or ()) if req.is_concrete_artifact or ( req.ver != '*' and not req.ver.startswith(('<', '>', '!=')) ) ) self._preferred_candidates = set(preferred_candidates or ()) self._with_deps = with_deps self._with_pre_releases = with_pre_releases
def find_matches(self, requirements): # type: (list[Requirement]) -> list[Candidate] r"""Find all possible candidates satisfying given requirements. This tries to get candidates based on the requirements' types. For concrete requirements (SCM, dir, namespace dir, local or remote archives), the one-and-only match is returned For a "named" requirement, Galaxy-compatible APIs are consulted to find concrete candidates for this requirement. Of theres a pre-installed candidate, it's prepended in front of others. :param requirements: A collection of requirements which all of \ the returned candidates must match. \ All requirements are guaranteed to have \ the same identifier. \ The collection is never empty. :returns: An iterable that orders candidates by preference, \ e.g. the most preferred candidate comes first. """ # FIXME: The first requirement may be a Git repo followed by # FIXME: its cloned tmp dir. Using only the first one creates # FIXME: loops that prevent any further dependency exploration. # FIXME: We need to figure out how to prevent this. first_req = requirements[0] fqcn = first_req.fqcn # The fqcn is guaranteed to be the same version_req = "A SemVer-compliant version or '*' is required. See https://semver.org to learn how to compose it correctly. " version_req += "This is an issue with the collection." try: coll_versions = self._api_proxy.get_collection_versions(first_req) except TypeError as exc: if first_req.is_concrete_artifact: # Non hashable versions will cause a TypeError raise ValueError( f"Invalid version found for the collection '{first_req}'. {version_req}" ) from exc # Unexpected error from a Galaxy server raise if first_req.is_concrete_artifact: # FIXME: do we assume that all the following artifacts are also concrete? # FIXME: does using fqcn==None cause us problems here? # Ensure the version found in the concrete artifact is SemVer-compliant for version, req_src in coll_versions: version_err = f"Invalid version found for the collection '{first_req}': {version} ({type(version)}). {version_req}" # NOTE: The known cases causing the version to be a non-string object come from # NOTE: the differences in how the YAML parser normalizes ambiguous values and # NOTE: how the end-users sometimes expect them to be parsed. Unless the users # NOTE: explicitly use the double quotes of one of the multiline string syntaxes # NOTE: in the collection metadata file, PyYAML will parse a value containing # NOTE: two dot-separated integers as `float`, a single integer as `int`, and 3+ # NOTE: integers as a `str`. In some cases, they may also use an empty value # NOTE: which is normalized as `null` and turned into `None` in the Python-land. # NOTE: Another known mistake is setting a minor part of the SemVer notation # NOTE: skipping the "patch" bit like "1.0" which is assumed non-compliant even # NOTE: after the conversion to string. if not isinstance(version, string_types): raise ValueError(version_err) elif version != '*': try: SemanticVersion(version) except ValueError as ex: raise ValueError(version_err) from ex return [ Candidate(fqcn, version, _none_src_server, first_req.type, None) for version, _none_src_server in coll_versions ] latest_matches = [] signatures = [] extra_signature_sources = [] # type: list[str] for version, src_server in coll_versions: tmp_candidate = Candidate(fqcn, version, src_server, 'galaxy', None) unsatisfied = False for requirement in requirements: unsatisfied |= not self.is_satisfied_by( requirement, tmp_candidate) # FIXME # unsatisfied |= not self.is_satisfied_by(requirement, tmp_candidate) or not ( # requirement.src is None or # if this is true for some candidates but not all it will break key param - Nonetype can't be compared to str # or requirement.src == candidate.src # ) if unsatisfied: break if not self._include_signatures: continue extra_signature_sources.extend(requirement.signature_sources or []) if not unsatisfied: if self._include_signatures: signatures = src_server.get_collection_signatures( first_req.namespace, first_req.name, version) for extra_source in extra_signature_sources: signatures.append( get_signature_from_source(extra_source)) latest_matches.append( Candidate(fqcn, version, src_server, 'galaxy', frozenset(signatures))) latest_matches.sort( key=lambda candidate: ( SemanticVersion(candidate.ver), candidate.src, ), reverse=True, # prefer newer versions over older ones ) preinstalled_candidates = { candidate for candidate in self._preferred_candidates if candidate.fqcn == fqcn and ( # check if an upgrade is necessary all( self.is_satisfied_by(requirement, candidate) for requirement in requirements) and (not self._upgrade or # check if an upgrade is preferred all( SemanticVersion(latest.ver) <= SemanticVersion( candidate.ver) for latest in latest_matches))) } return list(preinstalled_candidates) + latest_matches
def find_matches(self, requirements): # type: (List[Requirement]) -> List[Candidate] r"""Find all possible candidates satisfying given requirements. This tries to get candidates based on the requirements' types. For concrete requirements (SCM, dir, namespace dir, local or remote archives), the one-and-only match is returned For a "named" requirement, Galaxy-compatible APIs are consulted to find concrete candidates for this requirement. Of theres a pre-installed candidate, it's prepended in front of others. :param requirements: A collection of requirements which all of \ the returned candidates must match. \ All requirements are guaranteed to have \ the same identifier. \ The collection is never empty. :returns: An iterable that orders candidates by preference, \ e.g. the most preferred candidate comes first. """ # FIXME: The first requirement may be a Git repo followed by # FIXME: its cloned tmp dir. Using only the first one creates # FIXME: loops that prevent any further dependency exploration. # FIXME: We need to figure out how to prevent this. first_req = requirements[0] fqcn = first_req.fqcn # The fqcn is guaranteed to be the same coll_versions = self._api_proxy.get_collection_versions(first_req) if first_req.is_concrete_artifact: # FIXME: do we assume that all the following artifacts are also concrete? # FIXME: does using fqcn==None cause us problems here? return [ Candidate(fqcn, version, _none_src_server, first_req.type) for version, _none_src_server in coll_versions ] preinstalled_candidates = { candidate for candidate in self._preferred_candidates if candidate.fqcn == fqcn } return list(preinstalled_candidates) + sorted( { candidate for candidate in ( Candidate(fqcn, version, src_server, 'galaxy') for version, src_server in coll_versions ) if all(self.is_satisfied_by(requirement, candidate) for requirement in requirements) # FIXME # if all(self.is_satisfied_by(requirement, candidate) and ( # requirement.src is None or # if this is true for some candidates but not all it will break key param - Nonetype can't be compared to str # requirement.src == candidate.src # )) }, key=lambda candidate: ( SemanticVersion(candidate.ver), candidate.src, ), reverse=True, # prefer newer versions over older ones )