def fetch_requirements(
requirements_file,
thirdparty_dir,
python_version,
operating_system,
with_sources,
with_about,
allow_unpinned,
only_sources,
):
"""
Fetch and save to THIRDPARTY_DIR all the required wheels for pinned
dependencies found in the `--requirement` FILE requirements file(s). Only
fetch wheels compatible with the provided `--python-version` and
`--operating-system`.
Also fetch the corresponding .ABOUT, .LICENSE and .NOTICE files together
with a virtualenv.pyz app.
Use exclusively our remote repository (and not PyPI).
"""
# fetch wheels
python_versions = python_version
operating_systems = operating_system
requirements_files = requirements_file
if not only_sources:
envs = itertools.product(python_versions, operating_systems)
envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os)
for pyv, os in envs)
for env, reqf in itertools.product(envs, requirements_files):
for package, error in utils_thirdparty.fetch_wheels(
environment=env,
requirements_file=reqf,
allow_unpinned=allow_unpinned,
dest_dir=thirdparty_dir,
):
if error:
print('Failed to fetch wheel:', package, ':', error)
# optionally fetch sources
if with_sources or only_sources:
for reqf in requirements_files:
for package, error in utils_thirdparty.fetch_sources(
requirements_file=reqf,
allow_unpinned=allow_unpinned,
dest_dir=thirdparty_dir,
):
if error:
print('Failed to fetch source:', package, ':', error)
if with_about:
utils_thirdparty.add_fetch_or_update_about_and_license_files(
dest_dir=thirdparty_dir)
utils_thirdparty.find_problems(
dest_dir=thirdparty_dir,
report_missing_sources=with_sources or only_sources,
report_missing_wheels=not only_sources,
)
def check_thirdparty_dir(
dest_dir,
wheels,
sdists,
):
"""
Check a thirdparty directory for problems and print these on screen.
"""
# check for problems
print(f"==> CHECK FOR PROBLEMS")
utils_thirdparty.find_problems(
dest_dir=dest_dir,
report_missing_sources=sdists,
report_missing_wheels=wheels,
)
def fix_thirdparty_dir(
thirdparty_dir,
build_wheels,
build_remotely,
):
"""
Fix a thirdparty directory of dependent package wheels and sdist.
Multiple fixes are applied:
- fetch or build missing binary wheels
- fetch missing source distributions
- derive, fetch or add missing ABOUT files
- fetch missing .LICENSE and .NOTICE files
- remove outdated package versions and the ABOUT, .LICENSE and .NOTICE files
Optionally build missing binary wheels for all supported OS and Python
version combos locally or remotely.
"""
print('***FETCH*** MISSING WHEELS')
package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(
dest_dir=thirdparty_dir)
print('***FETCH*** MISSING SOURCES')
src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(
dest_dir=thirdparty_dir)
package_envts_not_built = []
if build_wheels:
print('***BUILD*** MISSING WHEELS')
package_envts_not_built, _wheel_filenames_built = utils_thirdparty.build_missing_wheels(
packages_and_envts=package_envts_not_fetched,
build_remotely=build_remotely,
dest_dir=thirdparty_dir,
)
print('***ADD*** ABOUT AND LICENSES')
utils_thirdparty.add_fetch_or_update_about_and_license_files(
dest_dir=thirdparty_dir)
# report issues
for name, version in src_name_ver_not_fetched:
print(f'{name}=={version}: Failed to fetch source distribution.')
for package, envt in package_envts_not_built:
print(f'{package.name}=={package.version}: Failed to build wheel '
f'on {envt.operating_system} for Python {envt.python_version}')
print('***FIND PROBLEMS***')
utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
def check_thirdparty_dir(thirdparty_dir):
"""
Check a thirdparty directory for problems.
"""
utils_thirdparty.find_problems(dest_dir=thirdparty_dir)
def fetch_thirdparty(
requirements_files,
specifiers,
latest_version,
dest_dir,
python_versions,
operating_systems,
wheels,
sdists,
index_urls,
):
"""
Download to --dest-dir THIRDPARTY_DIR the PyPI wheels, source distributions,
and their ABOUT metadata, license and notices files.
Download the PyPI packages listed in the combination of:
- the pip requirements --requirements REQUIREMENT-FILE(s),
- the pip name==version --specifier SPECIFIER(s)
- any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR.
Download wheels with the --wheels option for the ``--python-version`` PYVER(s)
and ``--operating_system`` OS(s) combinations defaulting to all supported combinations.
Download sdists tarballs with the --sdists option.
Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels and sources fetched.
Download wheels and sdists the provided PyPI simple --index-url INDEX(s) URLs.
"""
print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}")
existing_packages_by_nv = {
(package.name, package.version): package
for package in utils_thirdparty.get_local_packages(directory=dest_dir)
}
required_name_versions = set(existing_packages_by_nv.keys())
for req_file in requirements_files:
nvs = utils_requirements.load_requirements(
requirements_file=req_file,
with_unpinned=latest_version,
)
required_name_versions.update(nvs)
for specifier in specifiers:
nv = utils_requirements.get_name_version(
requirement=specifier,
with_unpinned=latest_version,
)
required_name_versions.add(nv)
if not required_name_versions:
print("Error: no requirements requested.")
sys.exit(1)
if not os.listdir(dest_dir) and not (wheels or sdists):
print("Error: one or both of --wheels and --sdists is required.")
sys.exit(1)
if latest_version:
latest_name_versions = set()
names = set(name for name, _version in sorted(required_name_versions))
for name in sorted(names):
latests = utils_thirdparty.PypiPackage.sorted(
utils_thirdparty.get_package_versions(name=name,
version=None,
index_urls=index_urls))
if not latests:
print(f"No distribution found for: {name}")
continue
latest = latests[-1]
latest_name_versions.add((latest.name, latest.version))
required_name_versions = latest_name_versions
if TRACE:
print("required_name_versions:", required_name_versions)
if wheels:
# create the environments matrix we need for wheels
evts = itertools.product(python_versions, operating_systems)
environments = [
utils_thirdparty.Environment.from_pyver_and_os(pyv, os)
for pyv, os in evts
]
wheels_not_found = {}
sdists_not_found = {}
# iterate over requirements, one at a time
for name, version in sorted(required_name_versions):
nv = name, version
existing_package = existing_packages_by_nv.get(nv)
if wheels:
for environment in environments:
if existing_package:
existing_wheels = list(
existing_package.get_supported_wheels(
environment=environment))
else:
existing_wheels = None
if existing_wheels:
if TRACE:
print(
f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}"
)
if all(w.is_pure() for w in existing_wheels):
break
else:
continue
if TRACE:
print(
f"Fetching wheel for: {name}=={version} on: {environment}"
)
try:
(
fetched_wheel_filenames,
existing_wheel_filenames,
) = utils_thirdparty.download_wheel(
name=name,
version=version,
environment=environment,
dest_dir=dest_dir,
index_urls=index_urls,
)
if TRACE:
if existing_wheel_filenames:
print(
f" ====> Wheels already available: {name}=={version} on: {environment}"
)
for whl in existing_wheel_filenames:
print(f" {whl}")
if fetched_wheel_filenames:
print(
f" ====> Wheels fetched: {name}=={version} on: {environment}"
)
for whl in fetched_wheel_filenames:
print(f" {whl}")
fwfns = fetched_wheel_filenames + existing_wheel_filenames
if all(
utils_thirdparty.Wheel.from_filename(f).is_pure()
for f in fwfns):
break
except utils_thirdparty.DistributionNotFound as e:
wheels_not_found[f"{name}=={version}"] = str(e)
if sdists:
if existing_package and existing_package.sdist:
if TRACE:
print(
f" ====> Sdist already available: {name}=={version}: {existing_package.sdist!r}"
)
continue
if TRACE:
print(f" Fetching sdist for: {name}=={version}")
try:
fetched = utils_thirdparty.download_sdist(
name=name,
version=version,
dest_dir=dest_dir,
index_urls=index_urls,
)
if TRACE:
if not fetched:
print(
f" ====> Sdist already available: {name}=={version} on: {environment}"
)
else:
print(
f" ====> Sdist fetched: {fetched} for {name}=={version} on: {environment}"
)
except utils_thirdparty.DistributionNotFound as e:
sdists_not_found[f"{name}=={version}"] = str(e)
if wheels and wheels_not_found:
print(f"==> MISSING WHEELS")
for wh in wheels_not_found:
print(f" {wh}")
if sdists and sdists_not_found:
print(f"==> MISSING SDISTS")
for sd in sdists_not_found:
print(f" {sd}")
print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES")
utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir)
utils_thirdparty.clean_about_files(dest_dir=dest_dir)
# check for problems
print(f"==> CHECK FOR PROBLEMS")
utils_thirdparty.find_problems(
dest_dir=dest_dir,
report_missing_sources=sdists,
report_missing_wheels=wheels,
)
def bootstrap(
requirements_file,
thirdparty_dir,
python_version,
operating_system,
with_deps,
latest_version,
sync_dejacode,
build_remotely=False,
):
"""
Boostrap a thirdparty Python packages directory from pip requirements.
Fetch or build to THIRDPARTY_DIR all the wheels and source distributions for
the pip ``--requirement-file`` requirements FILE(s). Build wheels compatible
with all the provided ``--python-version`` PYVER(s) and ```--operating_system``
OS(s) defaulting to all supported combinations. Create or fetch .ABOUT and
.LICENSE files.
Optionally ignore version specifiers and use the ``--latest-version``
of everything.
Sources and wheels are fetched with attempts first from PyPI, then our remote repository.
If missing wheels are built as needed.
"""
# rename variables for clarity since these are lists
requirements_files = requirements_file
python_versions = python_version
operating_systems = operating_system
# create the environments we need
evts = itertools.product(python_versions, operating_systems)
environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts]
# collect all packages to process from requirements files
# this will fail with an exception if there are packages we cannot find
required_name_versions = set()
for req_file in requirements_files:
nvs = utils_thirdparty.load_requirements(requirements_file=req_file,
force_pinned=False)
required_name_versions.update(nvs)
if latest_version:
required_name_versions = set(
(name, None) for name, _ver in required_name_versions)
print(
f'PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES'
)
# fetch all available wheels, keep track of missing
# start with local, then remote, then PyPI
print('==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS')
# list of all the wheel filenames either pre-existing, fetched or built
# updated as we progress
available_wheel_filenames = []
local_packages_by_namever = {
(p.name, p.version): p
for p in utils_thirdparty.get_local_packages(directory=thirdparty_dir)
}
# list of (name, version, environment) not local and to fetch
name_version_envt_to_fetch = []
# start with a local check
for (name, version), envt in itertools.product(required_name_versions,
environments):
local_pack = local_packages_by_namever.get((
name,
version,
))
if local_pack:
supported_wheels = list(
local_pack.get_supported_wheels(environment=envt))
if supported_wheels:
available_wheel_filenames.extend(w.filename
for w in supported_wheels)
print(
f'====> No fetch or build needed. '
f'Local wheel already available for {name}=={version} '
f'on os: {envt.operating_system} for Python: {envt.python_version}'
)
continue
name_version_envt_to_fetch.append((
name,
version,
envt,
))
print(
f'==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS'
)
# list of (name, version, environment) not fetch and to build
name_version_envt_to_build = []
# then check if the wheel can be fetched without building from remote and Pypi
for name, version, envt in name_version_envt_to_fetch:
fetched_fwn = utils_thirdparty.fetch_package_wheel(
name=name,
version=version,
environment=envt,
dest_dir=thirdparty_dir,
)
if fetched_fwn:
available_wheel_filenames.append(fetched_fwn)
else:
name_version_envt_to_build.append((
name,
version,
envt,
))
# At this stage we have all the wheels we could obtain without building
for name, version, envt in name_version_envt_to_build:
print(f'====> Need to build wheels for {name}=={version} on os: '
f'{envt.operating_system} for Python: {envt.python_version}')
packages_and_envts_to_build = [
(PypiPackage(name, version), envt)
for name, version, envt in name_version_envt_to_build
]
print(f'==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS')
package_envts_not_built, wheel_filenames_built = utils_thirdparty.build_missing_wheels(
packages_and_envts=packages_and_envts_to_build,
build_remotely=build_remotely,
with_deps=with_deps,
dest_dir=thirdparty_dir,
)
if wheel_filenames_built:
available_wheel_filenames.extend(available_wheel_filenames)
for pack, envt in package_envts_not_built:
print(
f'====> FAILED to build any wheel for {pack.name}=={pack.version} '
f'on os: {envt.operating_system} for Python: {envt.python_version}'
)
print(f'==> FETCHING SOURCE DISTRIBUTIONS')
# fetch all sources, keep track of missing
# This is a list of (name, version)
utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir)
print(f'==> FETCHING ABOUT AND LICENSE FILES')
utils_thirdparty.add_fetch_or_update_about_and_license_files(
dest_dir=thirdparty_dir)
############################################################################
if sync_dejacode:
print(f'==> SYNC WITH DEJACODE')
# try to fetch from DejaCode any missing ABOUT
# create all missing DejaCode packages
pass
utils_thirdparty.find_problems(dest_dir=thirdparty_dir)