async def download_changelog(
self, base_downloader: t.Callable[[str], t.Awaitable[str]]):
changelog = await self._get_changelog_file(self.latest,
base_downloader)
if changelog is None:
return
changelog.changes.prune_versions(versions_after=None,
versions_until=str(self.latest))
changelogs = [changelog]
ancestor = changelog.changes.ancestor
while ancestor is not None:
ancestor_ver = PypiVer(ancestor)
if ancestor_ver < self.earliest:
break
changelog = await self._get_changelog_file(ancestor_ver,
base_downloader)
if changelog is None:
break
changelog.changes.prune_versions(versions_after=None,
versions_until=ancestor)
changelogs.append(changelog)
ancestor = changelog.changes.ancestor
self.changelog = ChangelogData.concatenate(changelogs)
def test_build_file_write(tmpdir, dependencies, file_contents):
filename = tmpdir / 'test.build'
bf = BuildFile(filename)
bf.write(PypiVer('4.0.0'), '2.11.0rc1', dependencies)
with open(filename) as f:
assert f.read() == file_contents
def build_multiple_command() -> int:
app_ctx = app_context.app_ctx.get()
build_filename = os.path.join(app_ctx.extra['data_dir'], app_ctx.extra['build_file'])
build_file = BuildFile(build_filename)
build_ansible_version, ansible_base_version, deps = build_file.parse()
ansible_base_version = PypiVer(ansible_base_version)
# TODO: implement --feature-frozen support
if not str(app_ctx.extra['ansible_version']).startswith(build_ansible_version):
print(f'{build_filename} is for version {build_ansible_version} but we need'
f' {app_ctx.extra["ansible_version"].major}'
f'.{app_ctx.extra["ansible_version"].minor}')
return 1
with tempfile.TemporaryDirectory() as tmp_dir:
download_dir = os.path.join(tmp_dir, 'collections')
os.mkdir(download_dir, mode=0o700)
included_versions = asyncio.run(get_collection_versions(deps, app_ctx.galaxy_url))
asyncio.run(
download_collections(included_versions, app_ctx.galaxy_url, download_dir,
app_ctx.extra['collection_cache']))
# TODO: PY3.8:
# collections_to_install = [p for f in os.listdir(download_dir)
# if os.path.isfile(p := os.path.join(download_dir, f))]
collections_to_install = []
for collection in os.listdir(download_dir):
path = os.path.join(download_dir, collection)
if os.path.isfile(path):
collections_to_install.append(path)
collection_dirs = asyncio.run(install_separately(collections_to_install, download_dir))
asyncio.run(make_collection_dists(app_ctx.extra['sdist_dir'], collection_dirs))
# Create the ansible package that deps on the collections we just wrote
package_dir = os.path.join(tmp_dir, f'ansible-{app_ctx.extra["ansible_version"]}')
os.mkdir(package_dir, mode=0o700)
ansible_collections_dir = os.path.join(package_dir, 'ansible_collections')
os.mkdir(ansible_collections_dir, mode=0o700)
# Construct the list of dependent collection packages
collection_deps = []
for collection, version in sorted(included_versions.items()):
collection_deps.append(f" '{collection}>={version},<{version.next_major()}'")
collection_deps = '\n' + ',\n'.join(collection_deps)
write_build_script(app_ctx.extra['ansible_version'], ansible_base_version, package_dir)
write_python_build_files(app_ctx.extra['ansible_version'], ansible_base_version,
collection_deps, package_dir)
make_dist(package_dir, app_ctx.extra['sdist_dir'])
# Write the deps file
deps_filename = os.path.join(app_ctx.extra['dest_data_dir'], app_ctx.extra['deps_file'])
deps_file = DepsFile(deps_filename)
deps_file.write(app_ctx.extra['ansible_version'], ansible_base_version, included_versions)
return 0
def build_single_command() -> int:
app_ctx = app_context.app_ctx.get()
build_filename = os.path.join(app_ctx.extra['data_dir'], app_ctx.extra['build_file'])
build_file = BuildFile(build_filename)
build_ansible_version, ansible_base_version, deps = build_file.parse()
ansible_base_version = PypiVer(ansible_base_version)
# If we're building a feature frozen release (betas and rcs) then we need to
# change the upper version limit to not include new features.
if app_ctx.extra['feature_frozen']:
old_deps, deps = deps, {}
# For each collection that's listed...
for collection_name, spec in old_deps.items():
spec = SemVerSpec(spec)
new_clauses = []
min_version = None
# Look at each clause of the version specification
for clause in spec.clause.clauses:
if clause.operator in ('<', '<='):
# Omit the upper bound as we're replacing it
continue
if clause.operator == '>=':
# Save the lower bound so we can write out a new compatible version
min_version = clause.target
new_clauses.append(str(clause))
if min_version is None:
raise ValueError(f'No minimum version specified for {collection_name}: {spec}')
new_clauses.append(f'<{min_version.major}.{min_version.minor + 1}.0')
deps[collection_name] = ','.join(new_clauses)
included_versions = asyncio.run(get_collection_versions(deps, app_ctx.galaxy_url))
if not str(app_ctx.extra['ansible_version']).startswith(build_ansible_version):
print(f'{build_filename} is for version {build_ansible_version} but we need'
f' {app_ctx.extra["ansible_version"].major}'
f'.{app_ctx.extra["ansible_version"].minor}')
return 1
dependency_data = DependencyFileData(
str(app_ctx.extra['ansible_version']),
str(ansible_base_version),
{collection: str(version) for collection, version in included_versions.items()})
build_single_impl(dependency_data)
deps_filename = os.path.join(app_ctx.extra['dest_data_dir'], app_ctx.extra['deps_file'])
deps_file = DepsFile(deps_filename)
deps_file.write(
dependency_data.ansible_version,
dependency_data.ansible_base_version,
dependency_data.deps)
return 0
async def get_versions(self) -> t.List[PypiVer]:
"""
Get the versions of the ansible-base package on pypi.
:returns: A list of :pypkg:obj:`packaging.versioning.Version`s
for all the versions on pypi, including prereleases.
"""
pkg_info = await self.get_info()
versions = [PypiVer(r) for r in pkg_info['releases']]
versions.sort(reverse=True)
return versions
def get_ansible_core_version(
venv: t.Union['VenvRunner', 'FakeVenvRunner'],
env: t.Dict[str, str],
) -> PypiVer:
venv_python = venv.get_command('python')
ansible_version_cmd = venv_python(
'-c',
'import ansible.release; print(ansible.release.__version__)',
_env=env)
output = ansible_version_cmd.stdout.decode(
'utf-8', errors='surrogateescape').strip()
return PypiVer(output)
async def get_ansible_base(aio_session: 'aiohttp.client.ClientSession',
ansible_base_version: str,
tmpdir: str,
ansible_base_source: t.Optional[str] = None) -> str:
"""
Create an ansible-base directory of the requested version.
:arg aio_session: :obj:`aiohttp.client.ClientSession` to make http requests with.
:arg ansible_base_version: Version of ansible-base to retrieve. If it is the special string
``@devel``, then we will retrieve ansible-base from its git repository. If it is the
special string ``@latest``, then we will retrieve the latest version from pypi.
:arg tmpdir: Temporary directory use as a scratch area for downloading to and the place that the
ansible-base directory should be placed in.
:kwarg ansible_base_source: If given, a path to an ansible-base checkout or expanded sdist.
This will be used instead of downloading an ansible-base package if the version matches
with ``ansible_base_version``.
"""
if ansible_base_version == '@devel':
# is the source usable?
if source_is_devel(ansible_base_source):
# source_is_devel() protects against this. This assert is to inform the type checker
assert ansible_base_source is not None
source_location: str = ansible_base_source
else:
source_location = await checkout_from_git(tmpdir)
# Create an sdist from the source that can be installed
install_file = await create_sdist(source_location, tmpdir)
else:
pypi_client = AnsibleBasePyPiClient(aio_session)
if ansible_base_version == '@latest':
ansible_base_version: PypiVer = await pypi_client.get_latest_version(
)
else:
ansible_base_version: PypiVer = PypiVer(ansible_base_version)
# is the source the asked for version?
if source_is_correct_version(ansible_base_source,
ansible_base_version):
assert ansible_base_source is not None
# Create an sdist from the source that can be installed
install_file = await create_sdist(ansible_base_source, tmpdir)
else:
install_file = await pypi_client.retrieve(
ansible_base_version.public, tmpdir)
return install_file
def get_ansible_core_package_name(
ansible_base_version: t.Union[str, PypiVer]) -> str:
"""
Returns the name of the minimal ansible package.
:arg ansible_base_version: The version of the minimal ansible package to retrieve the
name for.
:returns: 'ansible-core' when the version is 2.11 or higher. Otherwise 'ansible-base'.
"""
if not isinstance(ansible_base_version, PypiVer):
ansible_base_version = PypiVer(ansible_base_version)
if ansible_base_version.major <= 2 and ansible_base_version.minor <= 10:
return 'ansible-base'
return 'ansible-core'
async def get_versions(self) -> t.List[PypiVer]:
"""
Get the versions of the ansible-base package on pypi.
:returns: A list of :pypkg:obj:`packaging.versioning.Version`s
for all the versions on pypi, including prereleases.
"""
flog = mlog.fields(func='AnsibleBasePyPiClient.get_versions')
flog.debug('Enter')
release_info = await self.get_release_info()
versions = [PypiVer(r) for r in release_info]
versions.sort(reverse=True)
flog.fields(
versions=versions).info('sorted list of ansible-core versions')
flog.debug('Leave')
return versions
def _get_source_version(ansible_base_source: str) -> PypiVer:
with open(os.path.join(ansible_base_source, 'lib', 'ansible',
'release.py')) as f:
root = ast.parse(f.read())
# Find the version of the source
source_version = None
# Iterate backwards in case __version__ is assigned to multiple times
for node in reversed(root.body):
if isinstance(node, ast.Assign):
for name in node.targets:
# These attributes are dynamic so pyre cannot check them
if name.id == '__version__': # pyre-ignore[16]
source_version = node.value.s # pyre-ignore[16]
break
if source_version:
break
if not source_version:
raise ValueError('Version was not found')
return PypiVer(source_version)
def build_single_impl(dependency_data: DependencyFileData, add_release: bool = True) -> None:
app_ctx = app_context.app_ctx.get()
# Determine included collection versions
ansible_base_version = PypiVer(dependency_data.ansible_base_version)
included_versions = {
collection: SemVer(version)
for collection, version in dependency_data.deps.items()
}
with tempfile.TemporaryDirectory() as tmp_dir:
download_dir = os.path.join(tmp_dir, 'collections')
os.mkdir(download_dir, mode=0o700)
# Download included collections
asyncio.run(download_collections(included_versions, app_ctx.galaxy_url,
download_dir, app_ctx.extra['collection_cache']))
# Get Ansible changelog, add new release
ansible_changelog = ChangelogData.ansible(
app_ctx.extra['data_dir'], app_ctx.extra['dest_data_dir'])
if add_release:
date = datetime.date.today()
ansible_changelog.add_ansible_release(
str(app_ctx.extra['ansible_version']),
date,
f'Release Date: {date}'
f'\n\n'
f'`Porting Guide <https://docs.ansible.com/ansible/devel/porting_guides.html>`_')
# Get changelog and porting guide data
changelog = get_changelog(
app_ctx.extra['ansible_version'],
deps_dir=app_ctx.extra['data_dir'],
deps_data=[dependency_data],
collection_cache=app_ctx.extra['collection_cache'],
ansible_changelog=ansible_changelog)
# Create package and collections directories
package_dir = os.path.join(tmp_dir, f'ansible-{app_ctx.extra["ansible_version"]}')
os.mkdir(package_dir, mode=0o700)
ansible_collections_dir = os.path.join(package_dir, 'ansible_collections')
os.mkdir(ansible_collections_dir, mode=0o700)
# Write the ansible release info to the collections dir
write_release_py(app_ctx.extra['ansible_version'], ansible_collections_dir)
# Install collections
# TODO: PY3.8:
# collections_to_install = [p for f in os.listdir(download_dir)
# if os.path.isfile(p := os.path.join(download_dir, f))]
collections_to_install = []
for collection in os.listdir(download_dir):
path = os.path.join(download_dir, collection)
if os.path.isfile(path):
collections_to_install.append(path)
asyncio.run(install_together(collections_to_install, ansible_collections_dir))
# Compose and write release notes
release_notes = ReleaseNotes.build(changelog)
release_notes.write_changelog_to(package_dir)
release_notes.write_porting_guide_to(package_dir)
# Write build scripts and files
write_build_script(app_ctx.extra['ansible_version'], ansible_base_version, package_dir)
write_python_build_files(app_ctx.extra['ansible_version'], ansible_base_version, '',
package_dir, release_notes, app_ctx.extra['debian'])
if app_ctx.extra['debian']:
write_debian_directory(app_ctx.extra['ansible_version'], ansible_base_version,
package_dir)
make_dist(package_dir, app_ctx.extra['sdist_dir'])
# Write changelog and porting guide also to destination directory
release_notes.write_changelog_to(app_ctx.extra['dest_data_dir'])
release_notes.write_porting_guide_to(app_ctx.extra['dest_data_dir'])
if add_release:
ansible_changelog.changes.save()
def get_changelog(
ansible_version: PypiVer,
deps_dir: t.Optional[str],
deps_data: t.Optional[t.List[DependencyFileData]] = None,
collection_cache: t.Optional[str] = None,
ansible_changelog: t.Optional[ChangelogData] = None) -> Changelog:
dependencies: t.Dict[str, DependencyFileData] = {}
ansible_changelog = ansible_changelog or ChangelogData.ansible(
directory=deps_dir)
ansible_ancestor_version_str = ansible_changelog.changes.ancestor
ansible_ancestor_version = (PypiVer(ansible_ancestor_version_str)
if ansible_ancestor_version_str else None)
collection_metadata = CollectionsMetadata(deps_dir)
if deps_dir is not None:
for path in glob.glob(os.path.join(deps_dir, '*.deps'),
recursive=False):
deps_file = DepsFile(path)
deps = deps_file.parse()
version = PypiVer(deps.ansible_version)
if version > ansible_version:
print(f"Ignoring {path}, since {deps.ansible_version}"
f" is newer than {ansible_version}")
continue
dependencies[deps.ansible_version] = deps
if deps_data:
for deps in deps_data:
dependencies[deps.ansible_version] = deps
base_versions: t.Dict[PypiVer, str] = dict()
versions: t.Dict[str, t.Tuple[PypiVer, DependencyFileData]] = dict()
versions_per_collection: t.Dict[str, t.Dict[PypiVer,
str]] = defaultdict(dict)
for deps in dependencies.values():
version = PypiVer(deps.ansible_version)
versions[deps.ansible_version] = (version, deps)
base_versions[version] = deps.ansible_base_version
for collection_name, collection_version in deps.deps.items():
versions_per_collection[collection_name][
version] = collection_version
base_collector = AnsibleBaseChangelogCollector(base_versions.values())
collectors = [
CollectionChangelogCollector(
collection, versions_per_collection[collection].values())
for collection in sorted(versions_per_collection.keys())
]
asyncio.run(
collect_changelogs(collectors, base_collector, collection_cache))
changelog = []
sorted_versions = collect_versions(versions, ansible_changelog.config)
for index, (version_str, dummy) in enumerate(sorted_versions):
version, deps = versions[version_str]
prev_version = None
if index + 1 < len(sorted_versions):
prev_version = versions[sorted_versions[index + 1][0]][0]
changelog.append(
ChangelogEntry(version, version_str, prev_version,
ansible_ancestor_version, base_versions,
versions_per_collection, base_collector,
ansible_changelog, collectors))
return Changelog(ansible_version, ansible_ancestor_version, changelog,
base_collector, ansible_changelog, collectors,
collection_metadata)
def __init__(self, versions: t.ValuesView[str]):
self.versions = sorted(PypiVer(version) for version in versions)
self.earliest = self.versions[0]
self.latest = self.versions[-1]
self.changelog = None
self.porting_guide = None
async def get_ansible_plugin_info(
venv: t.Union['VenvRunner', 'FakeVenvRunner'],
collection_dir: t.Optional[str],
collection_names: t.Optional[t.List[str]] = None
) -> t.Tuple[t.Mapping[str, t.Mapping[str, t.Any]], t.Mapping[
str, AnsibleCollectionMetadata]]:
"""
Retrieve information about all of the Ansible Plugins.
:arg venv: A VenvRunner into which Ansible has been installed.
:arg collection_dir: Directory in which the collections have been installed.
If ``None``, the collections are assumed to be in the current
search path for Ansible.
:arg collection_names: Optional list of collections. If specified, will only collect
information for plugins in these collections.
:returns: An tuple. The first component is a nested directory structure that looks like:
plugin_type:
plugin_name: # Includes namespace and collection.
{information from ansible-doc --json. See the ansible-doc documentation
for more info.}
The second component is a Mapping of collection names to metadata.
"""
flog = mlog.fields(func='get_ansible_plugin_info')
flog.debug('Enter')
env = _get_environment(collection_dir)
ansible_core_version = get_ansible_core_version(venv, env)
# Setup an sh.Command to run ansible-doc from the venv with only the collections we
# found as providers of extra plugins.
venv_ansible_doc = venv.get_command('ansible-doc')
venv_ansible_doc = venv_ansible_doc.bake('-vvv', _env=env)
# We invoke _get_plugin_info once for each documentable plugin type. Within _get_plugin_info,
# new threads are spawned to handle waiting for ansible-doc to parse files and give us results.
# To keep ourselves under thread_max, we need to divide the number of threads we're allowed over
# each call of _get_plugin_info.
# Why use thread_max instead of process_max? Even though this ultimately invokes separate
# ansible-doc processes, the limiting factor is IO as ansible-doc reads from disk. So it makes
# sense to scale up to thread_max instead of process_max.
# Allocate more for modules because the vast majority of plugins are modules
lib_ctx = app_context.lib_ctx.get()
module_workers = max(int(.7 * lib_ctx.thread_max), 1)
other_workers = int((lib_ctx.thread_max - module_workers) /
(len(DOCUMENTABLE_PLUGINS) - 1))
if other_workers < 1:
other_workers = 1
plugin_map = {}
extractors = {}
for plugin_type in DOCUMENTABLE_PLUGINS:
min_ver = DOCUMENTABLE_PLUGINS_MIN_VERSION.get(plugin_type)
if min_ver is not None and PypiVer(min_ver) > ansible_core_version:
plugin_map[plugin_type] = {}
continue
if plugin_type == 'module':
max_workers = module_workers
else:
max_workers = other_workers
extractors[plugin_type] = create_task(
_get_plugin_info(plugin_type, venv_ansible_doc, max_workers,
collection_names))
results = await asyncio.gather(*extractors.values(),
return_exceptions=True)
err_msg = []
an_exception = None
for plugin_type, extraction_result in zip(extractors, results):
if isinstance(extraction_result, Exception):
an_exception = extraction_result
formatted_exception = traceback.format_exception(
None, extraction_result, extraction_result.__traceback__)
err_msg.append(
f'Exception while parsing documentation for {plugin_type} plugins'
)
err_msg.append(f'Exception:\n{"".join(formatted_exception)}')
# Note: Exception will also be True.
if isinstance(extraction_result, sh.ErrorReturnCode):
stdout = extraction_result.stdout.decode("utf-8",
errors="surrogateescape")
stderr = extraction_result.stderr.decode("utf-8",
errors="surrogateescape")
err_msg.append(f'Full process stdout:\n{stdout}')
err_msg.append(f'Full process stderr:\n{stderr}')
if err_msg:
sys.stderr.write('\n'.join(err_msg))
sys.stderr.write('\n')
continue
plugin_map[plugin_type] = extraction_result
if an_exception:
# We wanted to print out all of the exceptions raised by parsing the output but once we've
# done so, we want to then fail by raising one of the exceptions.
raise ParsingError('Parsing of plugins failed')
flog.debug('Retrieving collection metadata')
collection_metadata = get_collection_metadata(venv, env, collection_names)
flog.debug('Leave')
return (plugin_map, collection_metadata)
