def _create_isolated_env_venv(path): # type: (str) -> Tuple[str, str] """ On Python 3 we use the venv package from the standard library. :param path: The path where to create the isolated build environment :return: The Python executable and script folder """ import venv venv.EnvBuilder(with_pip=True).create(path) executable, script_dir, purelib = _find_executable_and_scripts(path) # Get the version of pip in the environment pip_distribution = next( iter(metadata.distributions(name='pip', path=[purelib]))) pip_version = packaging.version.Version(pip_distribution.version) # Currently upgrade if Pip 19.1+ not available, since Pip 19 is the first # one to officially support PEP 517, and 19.1 supports manylinux1. if pip_version < packaging.version.Version('19.1'): subprocess.check_call( [executable, '-m', 'pip', 'install', '-U', 'pip']) # Avoid the setuptools from ensurepip to break the isolation subprocess.check_call( [executable, '-m', 'pip', 'uninstall', 'setuptools', '-y']) return executable, script_dir
def entry_points_compat(**params): """Return EntryPoint objects for all installed packages. Pass selection parameters (group or name) to filter the result to entry points matching those properties (see EntryPoints.select()). For compatibility, returns ``SelectableGroups`` object unless selection parameters are supplied. In the future, this function will return ``EntryPoints`` instead of ``SelectableGroups`` even when no selection parameters are supplied. For maximum future compatibility, pass selection parameters or invoke ``.select`` with parameters on the result. :return: EntryPoints or SelectableGroups for all installed packages. """ def dist_name(dist): return dist.metadata['Name'] unique = functools.partial(unique_everseen, key=dist_name) eps = itertools.chain.from_iterable( dist.entry_points for dist in unique(distributions()) ) return SelectableGroups.load(eps).select(**params)
def _create_isolated_env_venv(path): # type: (str) -> Tuple[str, str] """ On Python 3 we use the venv package from the standard library. :param path: The path where to create the isolated build environment :return: The Python executable and script folder """ import venv venv.EnvBuilder(with_pip=True, symlinks=_fs_supports_symlink()).create(path) executable, script_dir, purelib = _find_executable_and_scripts(path) # Get the version of pip in the environment pip_distribution = next(iter(metadata.distributions(name='pip', path=[purelib]))) current_pip_version = packaging.version.Version(pip_distribution.version) if platform.system() == 'Darwin' and int(platform.mac_ver()[0].split('.')[0]) >= 11: # macOS 11+ name scheme change requires 20.3. Intel macOS 11.0 can be told to report 10.16 for backwards # compatibility; but that also fixes earlier versions of pip so this is only needed for 11+. is_apple_silicon_python = sys.version_info >= (3, 6) and platform.machine() != 'x86_64' minimum_pip_version = '21.0.1' if is_apple_silicon_python else '20.3.0' else: # PEP-517 and manylinux1 was first implemented in 19.1 minimum_pip_version = '19.1.0' if current_pip_version < packaging.version.Version(minimum_pip_version): subprocess.check_call([executable, '-m', 'pip', 'install', 'pip>={}'.format(minimum_pip_version)]) # Avoid the setuptools from ensurepip to break the isolation subprocess.check_call([executable, '-m', 'pip', 'uninstall', 'setuptools', '-y']) return executable, script_dir
def setup_plugins(): import sys import os import importlib_metadata import importlib # noqa from importlib.util import module_from_spec, spec_from_file_location # noqa: E402 from indy_common.config_util import getConfigOnce # noqa: E402 import indy_node.server.plugin # noqa: E402 def find_and_load_plugin(plugin_name, plugin_root, installed_packages): if plugin_name in installed_packages: # TODO: Need a test for installed packages plugin_name = plugin_name.replace('-', '_') plugin = importlib.import_module(plugin_name) else: plugin_path = os.path.join(plugin_root.__path__[0], plugin_name, '__init__.py') spec = spec_from_file_location('__init__.py', plugin_path) plugin = module_from_spec(spec) spec.loader.exec_module(plugin) return plugin # TODO: Refactor to use plenum's setup_plugins # TODO: Should have a check to make sure no plugin defines any conflicting ledger id or request field global PLUGIN_LEDGER_IDS global PLUGIN_CLIENT_REQUEST_FIELDS global PLUGIN_CLIENT_REQ_OP_TYPES config = getConfigOnce(ignore_external_config_update_errors=True) plugin_root = config.PLUGIN_ROOT try: plugin_root = importlib.import_module(plugin_root) except ImportError: raise ImportError( 'Incorrect plugin root {}. No such package found'.format( plugin_root)) sys.path.insert(0, plugin_root.__path__[0]) enabled_plugins = config.ENABLED_PLUGINS installed_packages = set(p.metadata["Name"] for p in importlib_metadata.distributions()) for plugin_name in enabled_plugins: plugin = find_and_load_plugin(plugin_name, plugin_root, installed_packages) plugin_globals = plugin.__dict__ # The following lines are idempotent so loading the same plugin twice is not the problem. if 'LEDGER_IDS' in plugin_globals: PLUGIN_LEDGER_IDS.update(plugin_globals['LEDGER_IDS']) if 'CLIENT_REQUEST_FIELDS' in plugin_globals: PLUGIN_CLIENT_REQUEST_FIELDS.update( plugin_globals['CLIENT_REQUEST_FIELDS']) if 'REQ_OP_TYPES' in plugin_globals: PLUGIN_CLIENT_REQ_OP_TYPES.update(plugin_globals['REQ_OP_TYPES']) # Reloading message types since some some schemas would have been changed import indy_common.types importlib.reload(indy_common.types)
def load_entrypoints(self, group: Optional[str] = None): """Load plugins from setuptools entry_points""" group = group or self.project for dist in importlib_metadata.distributions(): for epoint in dist.entry_points: if epoint.group != group: continue plugin = epoint.load() self.register((plugin, epoint.name))
def test_unicode_dir_on_sys_path(self): """ Ensure a Unicode subdirectory of a directory on sys.path does not crash. """ fixtures.build_files( {self.unicode_filename(): {}}, prefix=self.site_dir, ) list(distributions())
def test_interleaved_discovery(self): """ When the search is cached, it is possible for searches to be interleaved, so make sure those use-cases are safe. Ref #293 """ dists = distributions() next(dists) version('importlib_metadata') next(dists)
def get_project_distribution() -> Union[Distribution, None]: ditr: Distribution for distr in importlib_metadata.distributions(): relative_path: Path try: relative_path = Path(__file__).relative_to(distr.locate_file("")) except ValueError: pass else: if relative_path in distr.files: return distr return None
def load(cls, env): # type: (Env) -> InstalledRepository """ Load installed packages. For now, it uses the pip "freeze" command. """ repo = cls() for distribution in sorted( distributions(path=env.sys_path), key=lambda d: str(d._path), ): metadata = distribution.metadata name = metadata["name"] version = metadata["version"] package = Package(name, version, version) package.description = metadata.get("summary", "") repo.add_package(package) path = Path(distribution._path) is_standard_package = True try: path.relative_to(env.site_packages) except ValueError: is_standard_package = False if is_standard_package: continue src_path = env.path / "src" # A VCS dependency should have been installed # in the src directory. If not, it's a path dependency try: path.relative_to(src_path) from poetry.vcs.git import Git git = Git() revision = git.rev_parse("HEAD", src_path / package.name).strip() url = git.remote_url(src_path / package.name) package.source_type = "git" package.source_url = url package.source_reference = revision except ValueError: package.source_type = "directory" package.source_url = str(path.parent) return repo
def __init__(self, linter): """Initialize the linter by loading all 'allowed' imports from package requirements""" super().__init__(linter) self.known_files = {} # type: Dict[str, _DistInfo] self.known_modules = defaultdict(set) # type: defaultdict[str, Set[_DistInfo]] if hasattr(isort, "place_module"): # isort >= v5 self._isort_place_module = isort.place_module # pylint: disable=no-member else: sorter = isort.SortImports(file_contents="") # pylint: disable=no-member self._isort_place_module = sorter.place_module all_loadable_distributions = set( importlib_metadata.distributions() ) # type: Set[Distribution] setup_result = run_setup("setup.py") self.first_party_packages = _filter_non_namespace_packages(setup_result.packages or []) self.allowed_distributions = { get_distribution(x).project_name for x in setup_result.install_requires } self.visited_distributions = set() self.dists_without_file_info = set() for dist in all_loadable_distributions: dist_name = dist.metadata["Name"] allowed = dist_name in self.allowed_distributions # Get a list of files created by the distribution distribution_files = dist.files or [] # Resolve the (relative) paths to absolute paths resolved_filepaths = {x.locate() for x in distribution_files} # in python3.4 dict.get() always returns None when passing a pathlib.Path as key distribution_file_info = { str(p): _DistInfo(dist, allowed) for p in resolved_filepaths } # Add them to the whitelist self.known_files.update(distribution_file_info) # Add distributions without files to candidate list for unmatched imports if not distribution_file_info: self.dists_without_file_info.add(dist_name) # Add source distributions to backup list if not dist.read_text("SOURCES.txt"): continue dist_modules_text = dist.read_text("top_level.txt") or "" dist_modules = dist_modules_text.split() for mod in dist_modules: self.known_modules[mod].add(_DistInfo(dist, allowed))
def _get_all_installed_packages(self) -> List[str]: """Get all installed packages. Used to check if dependency is installed or not. Returns: list of string names of packages that are installed """ if self._all_installed_packages is None: # Only retrieve once self._all_installed_packages = [ item.metadata.get("Name") for item in metadata.distributions() if item.metadata.get("Name") is not None ] return self._all_installed_packages
def entry_points_with_dist(group: str): """ Return EntryPoint objects of the given group, along with the distribution information. This is like the ``entry_points()`` function from importlib.metadata, except it also returns the distribution the entry_point was loaded from. :param group: Filter results to only this entrypoint group :return: Generator of (EntryPoint, Distribution) objects for the specified groups """ for dist in importlib_metadata.distributions(): for e in dist.entry_points: if e.group != group: continue yield e, dist
def _generate_software_info(self): os_version = self._prepare_for_json(platform.platform()) installed_packages = [ "{} {}".format(p.metadata["Name"], p.version) for p in importlib_metadata.distributions() ] output = self._run_external_cmd("dpkg-query --list | grep indy") indy_packages = output.split(os.linesep) return { "Software": { "OS_version": os_version, "Installed_packages": installed_packages, # TODO add this field "Indy_packages": self._prepare_for_json(indy_packages), } }
def get_requirements(config: Config): py_reqs = [] if config.graph: py_reqs = list( map(Record.from_distribution, filter(config.keep_dist, im.distributions()))) else: py_reqs = list(map(Record.from_name, config.includes)) py_reqs = list(filter(lambda r: r is not None, py_reqs)) js_reqs = get_js_requirements(config) reqs = [*py_reqs, *js_reqs] reqs.sort(key=lambda r: r.name) if config.verbose: for req in reqs: pprint(req.__dict__) return reqs
def perform(args): import importlib_metadata # type: ignore import jinja2 # type: ignore from packaging.requirements import Requirement # type: ignore from packaging.utils import canonicalize_name # type: ignore if args.python: output = check_output([ args.python, '-c', 'import json, sys; json.dump(sys.path, sys.stdout)' ]) sys_paths = json.loads(output) args.path = [p for p in sys_paths if path.isdir(p)] kdargs = dict() if args.path: kdargs.update(path=args.path) dists = list(importlib_metadata.distributions(**kdargs)) requires_extras = lru_cache(maxsize=None)(partial(_get_requires_extras, dists)) installed = lru_cache(maxsize=None)(partial(_installed, dists)) context = { 'distributions': dists, 'options': args, 'in_site': in_site, 'in_usersite': in_usersite, 'canonicalize_name': canonicalize_name, 'requires_extras': requires_extras, 'installed': installed, 'Requirement': Requirement, } if args.template: template = jinja2.Environment( loader=jinja2.FileSystemLoader(''), extensions=['jinja2.ext.do']).get_template(args.template) else: template = jinja2.Environment( loader=jinja2.PackageLoader(__package__), extensions=['jinja2.ext.do']).get_template('default.dot.j2') for s in template.generate(context): args.output.write(s) print('', file=args.output)
def entry_points_with_dist(group: str) -> Iterator[tuple[metadata.EntryPoint, metadata.Distribution]]: """Retrieve entry points of the given group. This is like the ``entry_points()`` function from importlib.metadata, except it also returns the distribution the entry_point was loaded from. :param group: Filter results to only this entrypoint group :return: Generator of (EntryPoint, Distribution) objects for the specified groups """ loaded: set[str] = set() for dist in metadata.distributions(): key = canonicalize_name(dist.metadata["Name"]) if key in loaded: continue loaded.add(key) for e in dist.entry_points: if e.group != group: continue yield e, dist
def get_all_entry_points(): """ Get all entry points related to ``ros2cli`` and any of its extensions. :returns: mapping of entry point names to ``EntryPoint`` instances :rtype: dict """ extension_points = get_entry_points(EXTENSION_POINT_GROUP_NAME) entry_points = defaultdict(dict) for dist in importlib_metadata.distributions(): for ep in dist.entry_points: # skip groups which are not registered as extension points if ep.group not in extension_points: continue entry_points[ep.group][ep.name] = (dist, ep) return entry_points
def log_packages_neptune(neptune_logger): """Uses the neptunecontrib.api to log the packages of the current python env.""" from neptunecontrib.api import log_table import pandas as pd import importlib_metadata dists = importlib_metadata.distributions() packages = { idx: (dist.metadata['Name'], dist.version) for idx, dist in enumerate(dists) } packages_df = pd.DataFrame.from_dict(packages, orient='index', columns=['package', 'version']) log_table(name='packages', table=packages_df, experiment=neptune_logger.experiment)
def _mark_plugins_for_rewrite(self, hook): """ Given an importhook, mark for rewrite any top-level modules or packages in the distribution package for all pytest plugins. """ self.pluginmanager.rewrite_hook = hook if os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"): # We don't autoload from setuptools entry points, no need to continue. return package_files = (str(file) for dist in importlib_metadata.distributions() if any(ep.group == "pytest11" for ep in dist.entry_points) for file in dist.files or []) for name in _iter_rewritable_modules(package_files): hook.mark_rewrite(name)
def load_setuptools_entrypoints(self, group, name=None): """ Load modules from querying the specified setuptools ``group``. :param str group: entry point group to load plugins :param str name: if given, loads only plugins with the given ``name``. :rtype: int :return: return the number of loaded plugins by this call. """ count = 0 for dist in importlib_metadata.distributions(): for ep in dist.entry_points: if (ep.group != group or (name is not None and ep.name != name) # already registered or self.get_plugin(ep.name) or self.is_blocked(ep.name)): continue plugin = ep.load() self.register(plugin, name=ep.name) self._plugin_distinfo.append((plugin, DistFacade(dist))) count += 1 return count
def _mark_plugins_for_rewrite(self, hook): """ Given an importhook, mark for rewrite any top-level modules or packages in the distribution package for all pytest plugins. """ self.pluginmanager.rewrite_hook = hook if os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"): # We don't autoload from setuptools entry points, no need to continue. return package_files = ( str(file) for dist in importlib_metadata.distributions() if any(ep.group == "pytest11" for ep in dist.entry_points) for file in dist.files ) for name in _iter_rewritable_modules(package_files): hook.mark_rewrite(name)
def list_extensions() -> List[ExtensionInfo]: """ List all installed Chaos Toolkit extensions in the current environment. Notice, for now we can only list extensions that start with `chaostoolkit-` in their package name. This is not as powerful and solid as we want it to be. The trick is that we can't rely on any metadata inside extensions to tell us they exist and what functionnality they provide either. Python has the concept of trove classifiers on packages but we can't extend them yet so they are of no use to us. In a future version, we will provide a mechanism from packages to support a better detection. """ infos = [] distros = importlib_metadata.distributions() seen = [] for dist in distros: info = dist.metadata name = info['Name'] if name == "chaostoolkit-lib": continue if name in seen: continue seen.append(name) if name.startswith("chaostoolkit-"): ext = ExtensionInfo(name=name, version=info["Version"], summary=info["Summary"], license=info["License"], author=info["Author"], url=info["Home-page"]) infos.append(ext) return infos
def load_setuptools_entrypoints(self, group, name=None): """ Load modules from querying the specified setuptools ``group``. :param str group: entry point group to load plugins :param str name: if given, loads only plugins with the given ``name``. :rtype: int :return: return the number of loaded plugins by this call. """ count = 0 for dist in importlib_metadata.distributions(): for ep in dist.entry_points: if ( ep.group != group or (name is not None and ep.name != name) # already registered or self.get_plugin(ep.name) or self.is_blocked(ep.name) ): continue plugin = ep.load() self.register(plugin, name=ep.name) self._plugin_distinfo.append((plugin, DistFacade(dist))) count += 1 return count
def get_requirements(excludes=excludes): licenses = get_licenses(excludes=excludes) predicate = lambda dist: dist.metadata['Name'] not in excludes py_reqs = list( map( lambda d: inflate(d, licenses), filter(predicate, im.distributions()) ) ) js_reqs_json = subprocess.run( ['node', cwd.joinpath('notice.js')], capture_output=True ).stdout.decode(encoding=encoding) js_reqs = list( map( lambda r: Record(**r), json.loads(js_reqs_json) ) ) custom_reqs = load_custom_requirements() reqs = [*py_reqs, *js_reqs, *custom_reqs] reqs.sort(key=lambda r: r.name) return reqs
def test_discovery(self): """ Discovering distributions should succeed even if there is an invalid path on sys.path. """ list(importlib_metadata.distributions())
def test_invalid_usage(self): with self.assertRaises(ValueError): list(distributions(context='something', name='else'))
def test_package_discovery(self): dists = list(distributions()) assert all(isinstance(dist, Distribution) for dist in dists) assert any(dist.metadata['Name'] == 'egginfo-pkg' for dist in dists) assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists)
from importlib_metadata import distributions for d in distributions(): print(d.metadata["Name"], d.version)
def get_installed_distributions(): return ([ Dist(project_name=it.metadata["Name"], version=it.version, check_version_conflict=(lambda: None)) for it in distributions() ])
def test_one_distribution(self): dists = list(distributions(path=sys.path[:1])) assert len(dists) == 1
def _find_local_distribution(self): dist, = importlib_metadata.distributions(path=['.', 'src']) return dist
def _findPlugins(self): return [pkg.metadata["Name"] for pkg in importlib_metadata.distributions() if pkg.metadata["Name"].startswith(PluginManager.prefix)]