def generate_requirements(extras_require): """ Convert requirements from a setup()-style dictionary to ('Requires-Dist', 'requirement') and ('Provides-Extra', 'extra') tuples. extras_require is a dictionary of {extra: [requirements]} as passed to setup(), using the empty extra {'': [requirements]} to hold install_requires. """ for extra, depends in extras_require.items(): condition = "" extra = extra or "" if ":" in extra: # setuptools extra:condition syntax extra, condition = extra.split(":", 1) extra = safe_extra(extra) if extra: yield "Provides-Extra", extra if condition: condition = "(" + condition + ") and " condition += "extra == '%s'" % extra for dependency in depends: new_req = Requirement(dependency) if condition: if new_req.marker: new_req.marker = "(%s) and %s" % (new_req.marker, condition) else: new_req.marker = condition yield "Requires-Dist", str(new_req)
def package_or_url_from_pep508(requirement: Requirement, remove_version_specifiers=False) -> str: requirement.marker = None requirement.name = canonicalize_name(requirement.name) if remove_version_specifiers: requirement.specifier = SpecifierSet("") return str(requirement)
def get_package_dependencies(self, extras=None) -> List[Requirement]: self._ensure_meta_present() if extras is None: extras = set() result = [] requires = self._distribution_meta.requires or [] for v in requires: req = Requirement(v) markers = getattr(req.marker, "_markers", ()) or () for _at, (m_key, op, m_val) in ((j, i) for j, i in enumerate(markers) if isinstance(i, tuple) and len(i) == 3): if m_key.value == "extra" and op.value == "==": extra = m_val.value break else: extra, _at = None, None if extra is None or extra in extras: if _at is not None: # noinspection PyProtectedMember del markers[_at] _at -= 1 if _at > 0 and markers[_at] in ("and", "or"): del markers[_at] # noinspection PyProtectedMember if len(markers) == 0: req.marker = None result.append(req) return result
def add_markers_to_dep(d, marker_str): # type: (str, Union[str, Marker]) -> str req = PackagingRequirement(d) existing_marker = getattr(req, "marker", None) if isinstance(marker_str, Marker): marker_str = str(marker_str) if existing_marker is not None: marker_str = str(merge_markers(existing_marker, marker_str)) if marker_str: marker_str = marker_str.replace("'", '"') req.marker = Marker(marker_str) return str(req)
def resolve_requirement_versions(package_versions): """ Resolves a list of requirements for the same package. Given a list of package details in the form of `packaging.requirements.Requirement` objects, combine the specifier, extras, url and marker information to create a new requirement object. """ resolved = Requirement(str(package_versions[0])) for package_version in package_versions[1:]: resolved.specifier = resolved.specifier & package_version.specifier resolved.extras = resolved.extras.union(package_version.extras) resolved.url = resolved.url or package_version.url if resolved.marker and package_version.marker: resolved.marker = Marker( f"{resolved.marker} or {package_version.marker}") elif package_version.marker: resolved.marker = package_version.marker return resolved
def require(pkgname, version=None): '''Ensure a package requirement is met. Install or update package as required and print a warning message if this can't be done due to the local environment, or when automatic package management is disabled by setting the environment variable 'LIBTBX_DISABLE_UPDATES'. :param pkgname: A string describing the package requirement. This will generally just be a package name, but package features can be specified in square brackets. Features are not enforced, but will be requested during installation and update. :param version: An optional string describing version constraints. This can be a minimum version, eg. '>=1.0', a maximum version, eg. '<2', or both, eg. '>=4.5,<4.6'. :return: True when the requirement is met, False otherwise.''' if not pip: _notice( " WARNING: Can not verify python package requirements - pip/setuptools out of date", " Please update pip and setuptools by running:", "", " libtbx.python -m pip install pip setuptools --upgrade", "", " or following the instructions at https://pip.pypa.io/en/stable/installing/" ) return False if not version: version = '' requirement = Requirement(pkgname + version) # Check if we have an environment marker in the request if requirement.marker and not requirement.marker.evaluate(): # We skip dependencies that don't match our current environment return True # Erase the marker from any further output requirement.marker = None # package name without feature specification basepkgname = requirement.name requirestring = str(requirement) baserequirestring = requirement.name + str(requirement.specifier) try: try: print("requires %s, has %s" % (requirestring, pkg_resources.require(requirestring)[0].version)) return True except pkg_resources.UnknownExtra: print("requires %s, has %s, but without features" % (requirestring, pkg_resources.require(baserequirestring)[0].version)) return True except pkg_resources.DistributionNotFound: currentversion = '(not determined)' project_name = pkgname action = 'install' print("requirement %s is not currently met, package not installed" % (requirestring)) except pkg_resources.VersionConflict: currentversion = pkg_resources.require(basepkgname)[0].version project_name = pkg_resources.require(basepkgname)[0].project_name action = 'update' print("requirement %s is not currently met, current version %s" % (requirestring, currentversion)) # Check if package can be updated for path_item in sys.path: egg_link = os.path.join(path_item, project_name + '.egg-link') if os.path.isfile(egg_link): with open(egg_link, 'r') as fh: location = fh.readline().strip() _notice( " WARNING: Can not update package {package} automatically.", "", "It is installed as editable package for development purposes. The currently", "installed version, {currentversion}, is too old. The required version is {requirement}.", "Please update the package manually in its installed location:", "", " {location}", package=pkgname, currentversion=currentversion, requirement=version, location=location) return False if not os.path.isdir(libtbx.env.under_base('.')): _notice( " WARNING: Can not {action} package {package} automatically.", "", "You are running in a base-less installation, which disables automatic package", "management by convention, see https://github.com/cctbx/cctbx_project/issues/151", "", "Please {action} the package manually.", package=pkgname, currentversion=currentversion, requirement=version, action=action) return False if os.getenv('LIBTBX_DISABLE_UPDATES') and os.getenv( 'LIBTBX_DISABLE_UPDATES').strip() not in ('0', ''): _notice( " WARNING: Can not {action} package {package} automatically.", "", "Environment variable LIBTBX_DISABLE_UPDATES is set. Please {action} manually.", package=pkgname, currentversion=currentversion, requirement=version, action=action) return False print("attempting {action} of {package}...".format(action=action, package=pkgname)) has_req_tracker = os.environ.get('PIP_REQ_TRACKER') exit_code = pip_main(['install', requirestring]) if not has_req_tracker: # clean up environment after pip call for next invocation os.environ.pop('PIP_REQ_TRACKER', None) if exit_code == 0: print("{action} successful".format(action=action)) return True else: print("{action} failed. please check manually".format(action=action)) return False
def updates(ctx, sync_dependencies, include_security_deps, batch_size): ignore_deps = set(IGNORED_DEPS) if not include_security_deps: ignore_deps.update(SECURITY_DEPS) ignore_deps = {normalize_project_name(d) for d in ignore_deps} dependencies, errors = read_agent_dependencies() if errors: for error in errors: echo_failure(error) abort() api_urls = [ f'https://pypi.org/pypi/{package}/json' for package in dependencies ] package_data = asyncio.run(scrape_version_data(api_urls)) package_data = { normalize_project_name(package_name): versions for package_name, versions in package_data.items() } new_dependencies = copy.deepcopy(dependencies) version_updates = defaultdict(lambda: defaultdict(set)) updated_packages = set() for name, python_versions in sorted(new_dependencies.items()): if name in ignore_deps: continue elif batch_size is not None and len(updated_packages) >= batch_size: break new_python_versions = package_data[name] dropped_py2 = len(set(new_python_versions.values())) > 1 for python_version, package_version in new_python_versions.items(): dependency_definitions = python_versions[python_version] if not dependency_definitions or package_version is None: continue dependency_definition, checks = dependency_definitions.popitem() requirement = Requirement(dependency_definition) requirement.specifier = SpecifierSet(f'=={package_version}') if dropped_py2 and 'python_version' not in dependency_definition: python_marker = f'python_version {"<" if python_version == "py2" else ">"} "3.0"' if not requirement.marker: requirement.marker = Marker(python_marker) else: requirement.marker = Marker( f'{requirement.marker} and {python_marker}') new_dependency_definition = get_normalized_dependency(requirement) dependency_definitions[new_dependency_definition] = checks if dependency_definition != new_dependency_definition: version_updates[name][package_version].add(python_version) updated_packages.add(name) if sync_dependencies: if updated_packages: update_agent_dependencies(new_dependencies) ctx.invoke(sync) echo_info(f'Updated {len(updated_packages)} dependencies') else: if updated_packages: echo_failure( f"{len(updated_packages)} dependencies are out of sync:") for name, versions in version_updates.items(): for package_version, python_versions in versions.items(): echo_failure( f'{name} can be updated to version {package_version} ' f'on {" and ".join(sorted(python_versions))}') abort() else: echo_info('All dependencies are up to date')