def generate_requirements(extras_require):
"""
Convert requirements from a setup()-style dictionary to ('Requires-Dist', 'requirement')
and ('Provides-Extra', 'extra') tuples.
extras_require is a dictionary of {extra: [requirements]} as passed to setup(),
using the empty extra {'': [requirements]} to hold install_requires.
"""
for extra, depends in extras_require.items():
condition = ""
extra = extra or ""
if ":" in extra: # setuptools extra:condition syntax
extra, condition = extra.split(":", 1)
extra = safe_extra(extra)
if extra:
yield "Provides-Extra", extra
if condition:
condition = "(" + condition + ") and "
condition += "extra == '%s'" % extra
for dependency in depends:
new_req = Requirement(dependency)
if condition:
if new_req.marker:
new_req.marker = "(%s) and %s" % (new_req.marker,
condition)
else:
new_req.marker = condition
yield "Requires-Dist", str(new_req)
def package_or_url_from_pep508(requirement: Requirement,
remove_version_specifiers=False) -> str:
requirement.marker = None
requirement.name = canonicalize_name(requirement.name)
if remove_version_specifiers:
requirement.specifier = SpecifierSet("")
return str(requirement)
def get_package_dependencies(self, extras=None) -> List[Requirement]:
self._ensure_meta_present()
if extras is None:
extras = set()
result = []
requires = self._distribution_meta.requires or []
for v in requires:
req = Requirement(v)
markers = getattr(req.marker, "_markers", ()) or ()
for _at, (m_key, op,
m_val) in ((j, i) for j, i in enumerate(markers)
if isinstance(i, tuple) and len(i) == 3):
if m_key.value == "extra" and op.value == "==":
extra = m_val.value
break
else:
extra, _at = None, None
if extra is None or extra in extras:
if _at is not None:
# noinspection PyProtectedMember
del markers[_at]
_at -= 1
if _at > 0 and markers[_at] in ("and", "or"):
del markers[_at]
# noinspection PyProtectedMember
if len(markers) == 0:
req.marker = None
result.append(req)
return result
def add_markers_to_dep(d, marker_str):
# type: (str, Union[str, Marker]) -> str
req = PackagingRequirement(d)
existing_marker = getattr(req, "marker", None)
if isinstance(marker_str, Marker):
marker_str = str(marker_str)
if existing_marker is not None:
marker_str = str(merge_markers(existing_marker, marker_str))
if marker_str:
marker_str = marker_str.replace("'", '"')
req.marker = Marker(marker_str)
return str(req)
def resolve_requirement_versions(package_versions):
"""
Resolves a list of requirements for the same package.
Given a list of package details in the form of `packaging.requirements.Requirement`
objects, combine the specifier, extras, url and marker information to create
a new requirement object.
"""
resolved = Requirement(str(package_versions[0]))
for package_version in package_versions[1:]:
resolved.specifier = resolved.specifier & package_version.specifier
resolved.extras = resolved.extras.union(package_version.extras)
resolved.url = resolved.url or package_version.url
if resolved.marker and package_version.marker:
resolved.marker = Marker(
f"{resolved.marker} or {package_version.marker}")
elif package_version.marker:
resolved.marker = package_version.marker
return resolved
def require(pkgname, version=None):
'''Ensure a package requirement is met. Install or update package as required
and print a warning message if this can't be done due to the local
environment, or when automatic package management is disabled by setting
the environment variable 'LIBTBX_DISABLE_UPDATES'.
:param pkgname: A string describing the package requirement. This will
generally just be a package name, but package features
can be specified in square brackets. Features are not
enforced, but will be requested during installation and
update.
:param version: An optional string describing version constraints. This
can be a minimum version, eg. '>=1.0', a maximum version,
eg. '<2', or both, eg. '>=4.5,<4.6'.
:return: True when the requirement is met, False otherwise.'''
if not pip:
_notice(
" WARNING: Can not verify python package requirements - pip/setuptools out of date",
" Please update pip and setuptools by running:", "",
" libtbx.python -m pip install pip setuptools --upgrade", "",
" or following the instructions at https://pip.pypa.io/en/stable/installing/"
)
return False
if not version:
version = ''
requirement = Requirement(pkgname + version)
# Check if we have an environment marker in the request
if requirement.marker and not requirement.marker.evaluate():
# We skip dependencies that don't match our current environment
return True
# Erase the marker from any further output
requirement.marker = None
# package name without feature specification
basepkgname = requirement.name
requirestring = str(requirement)
baserequirestring = requirement.name + str(requirement.specifier)
try:
try:
print("requires %s, has %s" %
(requirestring,
pkg_resources.require(requirestring)[0].version))
return True
except pkg_resources.UnknownExtra:
print("requires %s, has %s, but without features" %
(requirestring,
pkg_resources.require(baserequirestring)[0].version))
return True
except pkg_resources.DistributionNotFound:
currentversion = '(not determined)'
project_name = pkgname
action = 'install'
print("requirement %s is not currently met, package not installed" %
(requirestring))
except pkg_resources.VersionConflict:
currentversion = pkg_resources.require(basepkgname)[0].version
project_name = pkg_resources.require(basepkgname)[0].project_name
action = 'update'
print("requirement %s is not currently met, current version %s" %
(requirestring, currentversion))
# Check if package can be updated
for path_item in sys.path:
egg_link = os.path.join(path_item, project_name + '.egg-link')
if os.path.isfile(egg_link):
with open(egg_link, 'r') as fh:
location = fh.readline().strip()
_notice(
" WARNING: Can not update package {package} automatically.",
"",
"It is installed as editable package for development purposes. The currently",
"installed version, {currentversion}, is too old. The required version is {requirement}.",
"Please update the package manually in its installed location:",
"",
" {location}",
package=pkgname,
currentversion=currentversion,
requirement=version,
location=location)
return False
if not os.path.isdir(libtbx.env.under_base('.')):
_notice(
" WARNING: Can not {action} package {package} automatically.",
"",
"You are running in a base-less installation, which disables automatic package",
"management by convention, see https://github.com/cctbx/cctbx_project/issues/151",
"",
"Please {action} the package manually.",
package=pkgname,
currentversion=currentversion,
requirement=version,
action=action)
return False
if os.getenv('LIBTBX_DISABLE_UPDATES') and os.getenv(
'LIBTBX_DISABLE_UPDATES').strip() not in ('0', ''):
_notice(
" WARNING: Can not {action} package {package} automatically.",
"",
"Environment variable LIBTBX_DISABLE_UPDATES is set. Please {action} manually.",
package=pkgname,
currentversion=currentversion,
requirement=version,
action=action)
return False
print("attempting {action} of {package}...".format(action=action,
package=pkgname))
has_req_tracker = os.environ.get('PIP_REQ_TRACKER')
exit_code = pip_main(['install', requirestring])
if not has_req_tracker:
# clean up environment after pip call for next invocation
os.environ.pop('PIP_REQ_TRACKER', None)
if exit_code == 0:
print("{action} successful".format(action=action))
return True
else:
print("{action} failed. please check manually".format(action=action))
return False
def updates(ctx, sync_dependencies, include_security_deps, batch_size):
ignore_deps = set(IGNORED_DEPS)
if not include_security_deps:
ignore_deps.update(SECURITY_DEPS)
ignore_deps = {normalize_project_name(d) for d in ignore_deps}
dependencies, errors = read_agent_dependencies()
if errors:
for error in errors:
echo_failure(error)
abort()
api_urls = [
f'https://pypi.org/pypi/{package}/json' for package in dependencies
]
package_data = asyncio.run(scrape_version_data(api_urls))
package_data = {
normalize_project_name(package_name): versions
for package_name, versions in package_data.items()
}
new_dependencies = copy.deepcopy(dependencies)
version_updates = defaultdict(lambda: defaultdict(set))
updated_packages = set()
for name, python_versions in sorted(new_dependencies.items()):
if name in ignore_deps:
continue
elif batch_size is not None and len(updated_packages) >= batch_size:
break
new_python_versions = package_data[name]
dropped_py2 = len(set(new_python_versions.values())) > 1
for python_version, package_version in new_python_versions.items():
dependency_definitions = python_versions[python_version]
if not dependency_definitions or package_version is None:
continue
dependency_definition, checks = dependency_definitions.popitem()
requirement = Requirement(dependency_definition)
requirement.specifier = SpecifierSet(f'=={package_version}')
if dropped_py2 and 'python_version' not in dependency_definition:
python_marker = f'python_version {"<" if python_version == "py2" else ">"} "3.0"'
if not requirement.marker:
requirement.marker = Marker(python_marker)
else:
requirement.marker = Marker(
f'{requirement.marker} and {python_marker}')
new_dependency_definition = get_normalized_dependency(requirement)
dependency_definitions[new_dependency_definition] = checks
if dependency_definition != new_dependency_definition:
version_updates[name][package_version].add(python_version)
updated_packages.add(name)
if sync_dependencies:
if updated_packages:
update_agent_dependencies(new_dependencies)
ctx.invoke(sync)
echo_info(f'Updated {len(updated_packages)} dependencies')
else:
if updated_packages:
echo_failure(
f"{len(updated_packages)} dependencies are out of sync:")
for name, versions in version_updates.items():
for package_version, python_versions in versions.items():
echo_failure(
f'{name} can be updated to version {package_version} '
f'on {" and ".join(sorted(python_versions))}')
abort()
else:
echo_info('All dependencies are up to date')