async def download_changelog( self, base_downloader: t.Callable[[str], t.Awaitable[str]]): changelog = await self._get_changelog_file(self.latest, base_downloader) if changelog is None: return changelog.changes.prune_versions(versions_after=None, versions_until=str(self.latest)) changelogs = [changelog] ancestor = changelog.changes.ancestor while ancestor is not None: ancestor_ver = PypiVer(ancestor) if ancestor_ver < self.earliest: break changelog = await self._get_changelog_file(ancestor_ver, base_downloader) if changelog is None: break changelog.changes.prune_versions(versions_after=None, versions_until=ancestor) changelogs.append(changelog) ancestor = changelog.changes.ancestor self.changelog = ChangelogData.concatenate(changelogs)
def test_build_file_write(tmpdir, dependencies, file_contents): filename = tmpdir / 'test.build' bf = BuildFile(filename) bf.write(PypiVer('4.0.0'), '2.11.0rc1', dependencies) with open(filename) as f: assert f.read() == file_contents
def build_multiple_command() -> int: app_ctx = app_context.app_ctx.get() build_filename = os.path.join(app_ctx.extra['data_dir'], app_ctx.extra['build_file']) build_file = BuildFile(build_filename) build_ansible_version, ansible_base_version, deps = build_file.parse() ansible_base_version = PypiVer(ansible_base_version) # TODO: implement --feature-frozen support if not str(app_ctx.extra['ansible_version']).startswith(build_ansible_version): print(f'{build_filename} is for version {build_ansible_version} but we need' f' {app_ctx.extra["ansible_version"].major}' f'.{app_ctx.extra["ansible_version"].minor}') return 1 with tempfile.TemporaryDirectory() as tmp_dir: download_dir = os.path.join(tmp_dir, 'collections') os.mkdir(download_dir, mode=0o700) included_versions = asyncio.run(get_collection_versions(deps, app_ctx.galaxy_url)) asyncio.run( download_collections(included_versions, app_ctx.galaxy_url, download_dir, app_ctx.extra['collection_cache'])) # TODO: PY3.8: # collections_to_install = [p for f in os.listdir(download_dir) # if os.path.isfile(p := os.path.join(download_dir, f))] collections_to_install = [] for collection in os.listdir(download_dir): path = os.path.join(download_dir, collection) if os.path.isfile(path): collections_to_install.append(path) collection_dirs = asyncio.run(install_separately(collections_to_install, download_dir)) asyncio.run(make_collection_dists(app_ctx.extra['sdist_dir'], collection_dirs)) # Create the ansible package that deps on the collections we just wrote package_dir = os.path.join(tmp_dir, f'ansible-{app_ctx.extra["ansible_version"]}') os.mkdir(package_dir, mode=0o700) ansible_collections_dir = os.path.join(package_dir, 'ansible_collections') os.mkdir(ansible_collections_dir, mode=0o700) # Construct the list of dependent collection packages collection_deps = [] for collection, version in sorted(included_versions.items()): collection_deps.append(f" '{collection}>={version},<{version.next_major()}'") collection_deps = '\n' + ',\n'.join(collection_deps) write_build_script(app_ctx.extra['ansible_version'], ansible_base_version, package_dir) write_python_build_files(app_ctx.extra['ansible_version'], ansible_base_version, collection_deps, package_dir) make_dist(package_dir, app_ctx.extra['sdist_dir']) # Write the deps file deps_filename = os.path.join(app_ctx.extra['dest_data_dir'], app_ctx.extra['deps_file']) deps_file = DepsFile(deps_filename) deps_file.write(app_ctx.extra['ansible_version'], ansible_base_version, included_versions) return 0
def build_single_command() -> int: app_ctx = app_context.app_ctx.get() build_filename = os.path.join(app_ctx.extra['data_dir'], app_ctx.extra['build_file']) build_file = BuildFile(build_filename) build_ansible_version, ansible_base_version, deps = build_file.parse() ansible_base_version = PypiVer(ansible_base_version) # If we're building a feature frozen release (betas and rcs) then we need to # change the upper version limit to not include new features. if app_ctx.extra['feature_frozen']: old_deps, deps = deps, {} # For each collection that's listed... for collection_name, spec in old_deps.items(): spec = SemVerSpec(spec) new_clauses = [] min_version = None # Look at each clause of the version specification for clause in spec.clause.clauses: if clause.operator in ('<', '<='): # Omit the upper bound as we're replacing it continue if clause.operator == '>=': # Save the lower bound so we can write out a new compatible version min_version = clause.target new_clauses.append(str(clause)) if min_version is None: raise ValueError(f'No minimum version specified for {collection_name}: {spec}') new_clauses.append(f'<{min_version.major}.{min_version.minor + 1}.0') deps[collection_name] = ','.join(new_clauses) included_versions = asyncio.run(get_collection_versions(deps, app_ctx.galaxy_url)) if not str(app_ctx.extra['ansible_version']).startswith(build_ansible_version): print(f'{build_filename} is for version {build_ansible_version} but we need' f' {app_ctx.extra["ansible_version"].major}' f'.{app_ctx.extra["ansible_version"].minor}') return 1 dependency_data = DependencyFileData( str(app_ctx.extra['ansible_version']), str(ansible_base_version), {collection: str(version) for collection, version in included_versions.items()}) build_single_impl(dependency_data) deps_filename = os.path.join(app_ctx.extra['dest_data_dir'], app_ctx.extra['deps_file']) deps_file = DepsFile(deps_filename) deps_file.write( dependency_data.ansible_version, dependency_data.ansible_base_version, dependency_data.deps) return 0
async def get_versions(self) -> t.List[PypiVer]: """ Get the versions of the ansible-base package on pypi. :returns: A list of :pypkg:obj:`packaging.versioning.Version`s for all the versions on pypi, including prereleases. """ pkg_info = await self.get_info() versions = [PypiVer(r) for r in pkg_info['releases']] versions.sort(reverse=True) return versions
def get_ansible_core_version( venv: t.Union['VenvRunner', 'FakeVenvRunner'], env: t.Dict[str, str], ) -> PypiVer: venv_python = venv.get_command('python') ansible_version_cmd = venv_python( '-c', 'import ansible.release; print(ansible.release.__version__)', _env=env) output = ansible_version_cmd.stdout.decode( 'utf-8', errors='surrogateescape').strip() return PypiVer(output)
async def get_ansible_base(aio_session: 'aiohttp.client.ClientSession', ansible_base_version: str, tmpdir: str, ansible_base_source: t.Optional[str] = None) -> str: """ Create an ansible-base directory of the requested version. :arg aio_session: :obj:`aiohttp.client.ClientSession` to make http requests with. :arg ansible_base_version: Version of ansible-base to retrieve. If it is the special string ``@devel``, then we will retrieve ansible-base from its git repository. If it is the special string ``@latest``, then we will retrieve the latest version from pypi. :arg tmpdir: Temporary directory use as a scratch area for downloading to and the place that the ansible-base directory should be placed in. :kwarg ansible_base_source: If given, a path to an ansible-base checkout or expanded sdist. This will be used instead of downloading an ansible-base package if the version matches with ``ansible_base_version``. """ if ansible_base_version == '@devel': # is the source usable? if source_is_devel(ansible_base_source): # source_is_devel() protects against this. This assert is to inform the type checker assert ansible_base_source is not None source_location: str = ansible_base_source else: source_location = await checkout_from_git(tmpdir) # Create an sdist from the source that can be installed install_file = await create_sdist(source_location, tmpdir) else: pypi_client = AnsibleBasePyPiClient(aio_session) if ansible_base_version == '@latest': ansible_base_version: PypiVer = await pypi_client.get_latest_version( ) else: ansible_base_version: PypiVer = PypiVer(ansible_base_version) # is the source the asked for version? if source_is_correct_version(ansible_base_source, ansible_base_version): assert ansible_base_source is not None # Create an sdist from the source that can be installed install_file = await create_sdist(ansible_base_source, tmpdir) else: install_file = await pypi_client.retrieve( ansible_base_version.public, tmpdir) return install_file
def get_ansible_core_package_name( ansible_base_version: t.Union[str, PypiVer]) -> str: """ Returns the name of the minimal ansible package. :arg ansible_base_version: The version of the minimal ansible package to retrieve the name for. :returns: 'ansible-core' when the version is 2.11 or higher. Otherwise 'ansible-base'. """ if not isinstance(ansible_base_version, PypiVer): ansible_base_version = PypiVer(ansible_base_version) if ansible_base_version.major <= 2 and ansible_base_version.minor <= 10: return 'ansible-base' return 'ansible-core'
async def get_versions(self) -> t.List[PypiVer]: """ Get the versions of the ansible-base package on pypi. :returns: A list of :pypkg:obj:`packaging.versioning.Version`s for all the versions on pypi, including prereleases. """ flog = mlog.fields(func='AnsibleBasePyPiClient.get_versions') flog.debug('Enter') release_info = await self.get_release_info() versions = [PypiVer(r) for r in release_info] versions.sort(reverse=True) flog.fields( versions=versions).info('sorted list of ansible-core versions') flog.debug('Leave') return versions
def _get_source_version(ansible_base_source: str) -> PypiVer: with open(os.path.join(ansible_base_source, 'lib', 'ansible', 'release.py')) as f: root = ast.parse(f.read()) # Find the version of the source source_version = None # Iterate backwards in case __version__ is assigned to multiple times for node in reversed(root.body): if isinstance(node, ast.Assign): for name in node.targets: # These attributes are dynamic so pyre cannot check them if name.id == '__version__': # pyre-ignore[16] source_version = node.value.s # pyre-ignore[16] break if source_version: break if not source_version: raise ValueError('Version was not found') return PypiVer(source_version)
def build_single_impl(dependency_data: DependencyFileData, add_release: bool = True) -> None: app_ctx = app_context.app_ctx.get() # Determine included collection versions ansible_base_version = PypiVer(dependency_data.ansible_base_version) included_versions = { collection: SemVer(version) for collection, version in dependency_data.deps.items() } with tempfile.TemporaryDirectory() as tmp_dir: download_dir = os.path.join(tmp_dir, 'collections') os.mkdir(download_dir, mode=0o700) # Download included collections asyncio.run(download_collections(included_versions, app_ctx.galaxy_url, download_dir, app_ctx.extra['collection_cache'])) # Get Ansible changelog, add new release ansible_changelog = ChangelogData.ansible( app_ctx.extra['data_dir'], app_ctx.extra['dest_data_dir']) if add_release: date = datetime.date.today() ansible_changelog.add_ansible_release( str(app_ctx.extra['ansible_version']), date, f'Release Date: {date}' f'\n\n' f'`Porting Guide <https://docs.ansible.com/ansible/devel/porting_guides.html>`_') # Get changelog and porting guide data changelog = get_changelog( app_ctx.extra['ansible_version'], deps_dir=app_ctx.extra['data_dir'], deps_data=[dependency_data], collection_cache=app_ctx.extra['collection_cache'], ansible_changelog=ansible_changelog) # Create package and collections directories package_dir = os.path.join(tmp_dir, f'ansible-{app_ctx.extra["ansible_version"]}') os.mkdir(package_dir, mode=0o700) ansible_collections_dir = os.path.join(package_dir, 'ansible_collections') os.mkdir(ansible_collections_dir, mode=0o700) # Write the ansible release info to the collections dir write_release_py(app_ctx.extra['ansible_version'], ansible_collections_dir) # Install collections # TODO: PY3.8: # collections_to_install = [p for f in os.listdir(download_dir) # if os.path.isfile(p := os.path.join(download_dir, f))] collections_to_install = [] for collection in os.listdir(download_dir): path = os.path.join(download_dir, collection) if os.path.isfile(path): collections_to_install.append(path) asyncio.run(install_together(collections_to_install, ansible_collections_dir)) # Compose and write release notes release_notes = ReleaseNotes.build(changelog) release_notes.write_changelog_to(package_dir) release_notes.write_porting_guide_to(package_dir) # Write build scripts and files write_build_script(app_ctx.extra['ansible_version'], ansible_base_version, package_dir) write_python_build_files(app_ctx.extra['ansible_version'], ansible_base_version, '', package_dir, release_notes, app_ctx.extra['debian']) if app_ctx.extra['debian']: write_debian_directory(app_ctx.extra['ansible_version'], ansible_base_version, package_dir) make_dist(package_dir, app_ctx.extra['sdist_dir']) # Write changelog and porting guide also to destination directory release_notes.write_changelog_to(app_ctx.extra['dest_data_dir']) release_notes.write_porting_guide_to(app_ctx.extra['dest_data_dir']) if add_release: ansible_changelog.changes.save()
def get_changelog( ansible_version: PypiVer, deps_dir: t.Optional[str], deps_data: t.Optional[t.List[DependencyFileData]] = None, collection_cache: t.Optional[str] = None, ansible_changelog: t.Optional[ChangelogData] = None) -> Changelog: dependencies: t.Dict[str, DependencyFileData] = {} ansible_changelog = ansible_changelog or ChangelogData.ansible( directory=deps_dir) ansible_ancestor_version_str = ansible_changelog.changes.ancestor ansible_ancestor_version = (PypiVer(ansible_ancestor_version_str) if ansible_ancestor_version_str else None) collection_metadata = CollectionsMetadata(deps_dir) if deps_dir is not None: for path in glob.glob(os.path.join(deps_dir, '*.deps'), recursive=False): deps_file = DepsFile(path) deps = deps_file.parse() version = PypiVer(deps.ansible_version) if version > ansible_version: print(f"Ignoring {path}, since {deps.ansible_version}" f" is newer than {ansible_version}") continue dependencies[deps.ansible_version] = deps if deps_data: for deps in deps_data: dependencies[deps.ansible_version] = deps base_versions: t.Dict[PypiVer, str] = dict() versions: t.Dict[str, t.Tuple[PypiVer, DependencyFileData]] = dict() versions_per_collection: t.Dict[str, t.Dict[PypiVer, str]] = defaultdict(dict) for deps in dependencies.values(): version = PypiVer(deps.ansible_version) versions[deps.ansible_version] = (version, deps) base_versions[version] = deps.ansible_base_version for collection_name, collection_version in deps.deps.items(): versions_per_collection[collection_name][ version] = collection_version base_collector = AnsibleBaseChangelogCollector(base_versions.values()) collectors = [ CollectionChangelogCollector( collection, versions_per_collection[collection].values()) for collection in sorted(versions_per_collection.keys()) ] asyncio.run( collect_changelogs(collectors, base_collector, collection_cache)) changelog = [] sorted_versions = collect_versions(versions, ansible_changelog.config) for index, (version_str, dummy) in enumerate(sorted_versions): version, deps = versions[version_str] prev_version = None if index + 1 < len(sorted_versions): prev_version = versions[sorted_versions[index + 1][0]][0] changelog.append( ChangelogEntry(version, version_str, prev_version, ansible_ancestor_version, base_versions, versions_per_collection, base_collector, ansible_changelog, collectors)) return Changelog(ansible_version, ansible_ancestor_version, changelog, base_collector, ansible_changelog, collectors, collection_metadata)
def __init__(self, versions: t.ValuesView[str]): self.versions = sorted(PypiVer(version) for version in versions) self.earliest = self.versions[0] self.latest = self.versions[-1] self.changelog = None self.porting_guide = None
async def get_ansible_plugin_info( venv: t.Union['VenvRunner', 'FakeVenvRunner'], collection_dir: t.Optional[str], collection_names: t.Optional[t.List[str]] = None ) -> t.Tuple[t.Mapping[str, t.Mapping[str, t.Any]], t.Mapping[ str, AnsibleCollectionMetadata]]: """ Retrieve information about all of the Ansible Plugins. :arg venv: A VenvRunner into which Ansible has been installed. :arg collection_dir: Directory in which the collections have been installed. If ``None``, the collections are assumed to be in the current search path for Ansible. :arg collection_names: Optional list of collections. If specified, will only collect information for plugins in these collections. :returns: An tuple. The first component is a nested directory structure that looks like: plugin_type: plugin_name: # Includes namespace and collection. {information from ansible-doc --json. See the ansible-doc documentation for more info.} The second component is a Mapping of collection names to metadata. """ flog = mlog.fields(func='get_ansible_plugin_info') flog.debug('Enter') env = _get_environment(collection_dir) ansible_core_version = get_ansible_core_version(venv, env) # Setup an sh.Command to run ansible-doc from the venv with only the collections we # found as providers of extra plugins. venv_ansible_doc = venv.get_command('ansible-doc') venv_ansible_doc = venv_ansible_doc.bake('-vvv', _env=env) # We invoke _get_plugin_info once for each documentable plugin type. Within _get_plugin_info, # new threads are spawned to handle waiting for ansible-doc to parse files and give us results. # To keep ourselves under thread_max, we need to divide the number of threads we're allowed over # each call of _get_plugin_info. # Why use thread_max instead of process_max? Even though this ultimately invokes separate # ansible-doc processes, the limiting factor is IO as ansible-doc reads from disk. So it makes # sense to scale up to thread_max instead of process_max. # Allocate more for modules because the vast majority of plugins are modules lib_ctx = app_context.lib_ctx.get() module_workers = max(int(.7 * lib_ctx.thread_max), 1) other_workers = int((lib_ctx.thread_max - module_workers) / (len(DOCUMENTABLE_PLUGINS) - 1)) if other_workers < 1: other_workers = 1 plugin_map = {} extractors = {} for plugin_type in DOCUMENTABLE_PLUGINS: min_ver = DOCUMENTABLE_PLUGINS_MIN_VERSION.get(plugin_type) if min_ver is not None and PypiVer(min_ver) > ansible_core_version: plugin_map[plugin_type] = {} continue if plugin_type == 'module': max_workers = module_workers else: max_workers = other_workers extractors[plugin_type] = create_task( _get_plugin_info(plugin_type, venv_ansible_doc, max_workers, collection_names)) results = await asyncio.gather(*extractors.values(), return_exceptions=True) err_msg = [] an_exception = None for plugin_type, extraction_result in zip(extractors, results): if isinstance(extraction_result, Exception): an_exception = extraction_result formatted_exception = traceback.format_exception( None, extraction_result, extraction_result.__traceback__) err_msg.append( f'Exception while parsing documentation for {plugin_type} plugins' ) err_msg.append(f'Exception:\n{"".join(formatted_exception)}') # Note: Exception will also be True. if isinstance(extraction_result, sh.ErrorReturnCode): stdout = extraction_result.stdout.decode("utf-8", errors="surrogateescape") stderr = extraction_result.stderr.decode("utf-8", errors="surrogateescape") err_msg.append(f'Full process stdout:\n{stdout}') err_msg.append(f'Full process stderr:\n{stderr}') if err_msg: sys.stderr.write('\n'.join(err_msg)) sys.stderr.write('\n') continue plugin_map[plugin_type] = extraction_result if an_exception: # We wanted to print out all of the exceptions raised by parsing the output but once we've # done so, we want to then fail by raising one of the exceptions. raise ParsingError('Parsing of plugins failed') flog.debug('Retrieving collection metadata') collection_metadata = get_collection_metadata(venv, env, collection_names) flog.debug('Leave') return (plugin_map, collection_metadata)