def _create_isolated_env_venv(path): # type: (str) -> Tuple[str, str]
"""
On Python 3 we use the venv package from the standard library.
:param path: The path where to create the isolated build environment
:return: The Python executable and script folder
"""
import venv
venv.EnvBuilder(with_pip=True).create(path)
executable, script_dir, purelib = _find_executable_and_scripts(path)
# Get the version of pip in the environment
pip_distribution = next(
iter(metadata.distributions(name='pip', path=[purelib])))
pip_version = packaging.version.Version(pip_distribution.version)
# Currently upgrade if Pip 19.1+ not available, since Pip 19 is the first
# one to officially support PEP 517, and 19.1 supports manylinux1.
if pip_version < packaging.version.Version('19.1'):
subprocess.check_call(
[executable, '-m', 'pip', 'install', '-U', 'pip'])
# Avoid the setuptools from ensurepip to break the isolation
subprocess.check_call(
[executable, '-m', 'pip', 'uninstall', 'setuptools', '-y'])
return executable, script_dir
def entry_points_compat(**params):
"""Return EntryPoint objects for all installed packages.
Pass selection parameters (group or name) to filter the
result to entry points matching those properties (see
EntryPoints.select()).
For compatibility, returns ``SelectableGroups`` object unless
selection parameters are supplied. In the future, this function
will return ``EntryPoints`` instead of ``SelectableGroups``
even when no selection parameters are supplied.
For maximum future compatibility, pass selection parameters
or invoke ``.select`` with parameters on the result.
:return: EntryPoints or SelectableGroups for all installed packages.
"""
def dist_name(dist):
return dist.metadata['Name']
unique = functools.partial(unique_everseen, key=dist_name)
eps = itertools.chain.from_iterable(
dist.entry_points for dist in unique(distributions())
)
return SelectableGroups.load(eps).select(**params)
def _create_isolated_env_venv(path): # type: (str) -> Tuple[str, str]
"""
On Python 3 we use the venv package from the standard library.
:param path: The path where to create the isolated build environment
:return: The Python executable and script folder
"""
import venv
venv.EnvBuilder(with_pip=True, symlinks=_fs_supports_symlink()).create(path)
executable, script_dir, purelib = _find_executable_and_scripts(path)
# Get the version of pip in the environment
pip_distribution = next(iter(metadata.distributions(name='pip', path=[purelib])))
current_pip_version = packaging.version.Version(pip_distribution.version)
if platform.system() == 'Darwin' and int(platform.mac_ver()[0].split('.')[0]) >= 11:
# macOS 11+ name scheme change requires 20.3. Intel macOS 11.0 can be told to report 10.16 for backwards
# compatibility; but that also fixes earlier versions of pip so this is only needed for 11+.
is_apple_silicon_python = sys.version_info >= (3, 6) and platform.machine() != 'x86_64'
minimum_pip_version = '21.0.1' if is_apple_silicon_python else '20.3.0'
else:
# PEP-517 and manylinux1 was first implemented in 19.1
minimum_pip_version = '19.1.0'
if current_pip_version < packaging.version.Version(minimum_pip_version):
subprocess.check_call([executable, '-m', 'pip', 'install', 'pip>={}'.format(minimum_pip_version)])
# Avoid the setuptools from ensurepip to break the isolation
subprocess.check_call([executable, '-m', 'pip', 'uninstall', 'setuptools', '-y'])
return executable, script_dir
def setup_plugins():
import sys
import os
import importlib_metadata
import importlib # noqa
from importlib.util import module_from_spec, spec_from_file_location # noqa: E402
from indy_common.config_util import getConfigOnce # noqa: E402
import indy_node.server.plugin # noqa: E402
def find_and_load_plugin(plugin_name, plugin_root, installed_packages):
if plugin_name in installed_packages:
# TODO: Need a test for installed packages
plugin_name = plugin_name.replace('-', '_')
plugin = importlib.import_module(plugin_name)
else:
plugin_path = os.path.join(plugin_root.__path__[0], plugin_name,
'__init__.py')
spec = spec_from_file_location('__init__.py', plugin_path)
plugin = module_from_spec(spec)
spec.loader.exec_module(plugin)
return plugin
# TODO: Refactor to use plenum's setup_plugins
# TODO: Should have a check to make sure no plugin defines any conflicting ledger id or request field
global PLUGIN_LEDGER_IDS
global PLUGIN_CLIENT_REQUEST_FIELDS
global PLUGIN_CLIENT_REQ_OP_TYPES
config = getConfigOnce(ignore_external_config_update_errors=True)
plugin_root = config.PLUGIN_ROOT
try:
plugin_root = importlib.import_module(plugin_root)
except ImportError:
raise ImportError(
'Incorrect plugin root {}. No such package found'.format(
plugin_root))
sys.path.insert(0, plugin_root.__path__[0])
enabled_plugins = config.ENABLED_PLUGINS
installed_packages = set(p.metadata["Name"]
for p in importlib_metadata.distributions())
for plugin_name in enabled_plugins:
plugin = find_and_load_plugin(plugin_name, plugin_root,
installed_packages)
plugin_globals = plugin.__dict__
# The following lines are idempotent so loading the same plugin twice is not the problem.
if 'LEDGER_IDS' in plugin_globals:
PLUGIN_LEDGER_IDS.update(plugin_globals['LEDGER_IDS'])
if 'CLIENT_REQUEST_FIELDS' in plugin_globals:
PLUGIN_CLIENT_REQUEST_FIELDS.update(
plugin_globals['CLIENT_REQUEST_FIELDS'])
if 'REQ_OP_TYPES' in plugin_globals:
PLUGIN_CLIENT_REQ_OP_TYPES.update(plugin_globals['REQ_OP_TYPES'])
# Reloading message types since some some schemas would have been changed
import indy_common.types
importlib.reload(indy_common.types)
def load_entrypoints(self, group: Optional[str] = None):
"""Load plugins from setuptools entry_points"""
group = group or self.project
for dist in importlib_metadata.distributions():
for epoint in dist.entry_points:
if epoint.group != group:
continue
plugin = epoint.load()
self.register((plugin, epoint.name))
def test_unicode_dir_on_sys_path(self):
"""
Ensure a Unicode subdirectory of a directory on sys.path
does not crash.
"""
fixtures.build_files(
{self.unicode_filename(): {}},
prefix=self.site_dir,
)
list(distributions())
def test_interleaved_discovery(self):
"""
When the search is cached, it is
possible for searches to be interleaved, so make sure
those use-cases are safe.
Ref #293
"""
dists = distributions()
next(dists)
version('importlib_metadata')
next(dists)
def get_project_distribution() -> Union[Distribution, None]:
ditr: Distribution
for distr in importlib_metadata.distributions():
relative_path: Path
try:
relative_path = Path(__file__).relative_to(distr.locate_file(""))
except ValueError:
pass
else:
if relative_path in distr.files:
return distr
return None
def load(cls, env): # type: (Env) -> InstalledRepository
"""
Load installed packages.
For now, it uses the pip "freeze" command.
"""
repo = cls()
for distribution in sorted(
distributions(path=env.sys_path),
key=lambda d: str(d._path),
):
metadata = distribution.metadata
name = metadata["name"]
version = metadata["version"]
package = Package(name, version, version)
package.description = metadata.get("summary", "")
repo.add_package(package)
path = Path(distribution._path)
is_standard_package = True
try:
path.relative_to(env.site_packages)
except ValueError:
is_standard_package = False
if is_standard_package:
continue
src_path = env.path / "src"
# A VCS dependency should have been installed
# in the src directory. If not, it's a path dependency
try:
path.relative_to(src_path)
from poetry.vcs.git import Git
git = Git()
revision = git.rev_parse("HEAD",
src_path / package.name).strip()
url = git.remote_url(src_path / package.name)
package.source_type = "git"
package.source_url = url
package.source_reference = revision
except ValueError:
package.source_type = "directory"
package.source_url = str(path.parent)
return repo
def __init__(self, linter):
"""Initialize the linter by loading all 'allowed' imports from package requirements"""
super().__init__(linter)
self.known_files = {} # type: Dict[str, _DistInfo]
self.known_modules = defaultdict(set) # type: defaultdict[str, Set[_DistInfo]]
if hasattr(isort, "place_module"): # isort >= v5
self._isort_place_module = isort.place_module # pylint: disable=no-member
else:
sorter = isort.SortImports(file_contents="") # pylint: disable=no-member
self._isort_place_module = sorter.place_module
all_loadable_distributions = set(
importlib_metadata.distributions()
) # type: Set[Distribution]
setup_result = run_setup("setup.py")
self.first_party_packages = _filter_non_namespace_packages(setup_result.packages or [])
self.allowed_distributions = {
get_distribution(x).project_name for x in setup_result.install_requires
}
self.visited_distributions = set()
self.dists_without_file_info = set()
for dist in all_loadable_distributions:
dist_name = dist.metadata["Name"]
allowed = dist_name in self.allowed_distributions
# Get a list of files created by the distribution
distribution_files = dist.files or []
# Resolve the (relative) paths to absolute paths
resolved_filepaths = {x.locate() for x in distribution_files}
# in python3.4 dict.get() always returns None when passing a pathlib.Path as key
distribution_file_info = {
str(p): _DistInfo(dist, allowed) for p in resolved_filepaths
}
# Add them to the whitelist
self.known_files.update(distribution_file_info)
# Add distributions without files to candidate list for unmatched imports
if not distribution_file_info:
self.dists_without_file_info.add(dist_name)
# Add source distributions to backup list
if not dist.read_text("SOURCES.txt"):
continue
dist_modules_text = dist.read_text("top_level.txt") or ""
dist_modules = dist_modules_text.split()
for mod in dist_modules:
self.known_modules[mod].add(_DistInfo(dist, allowed))
def _get_all_installed_packages(self) -> List[str]:
"""Get all installed packages. Used to check if dependency is installed or not.
Returns:
list of string names of packages that are installed
"""
if self._all_installed_packages is None:
# Only retrieve once
self._all_installed_packages = [
item.metadata.get("Name") for item in metadata.distributions()
if item.metadata.get("Name") is not None
]
return self._all_installed_packages
def entry_points_with_dist(group: str):
"""
Return EntryPoint objects of the given group, along with the distribution information.
This is like the ``entry_points()`` function from importlib.metadata,
except it also returns the distribution the entry_point was loaded from.
:param group: Filter results to only this entrypoint group
:return: Generator of (EntryPoint, Distribution) objects for the specified groups
"""
for dist in importlib_metadata.distributions():
for e in dist.entry_points:
if e.group != group:
continue
yield e, dist
def _generate_software_info(self):
os_version = self._prepare_for_json(platform.platform())
installed_packages = [
"{} {}".format(p.metadata["Name"], p.version)
for p in importlib_metadata.distributions()
]
output = self._run_external_cmd("dpkg-query --list | grep indy")
indy_packages = output.split(os.linesep)
return {
"Software": {
"OS_version": os_version,
"Installed_packages": installed_packages,
# TODO add this field
"Indy_packages": self._prepare_for_json(indy_packages),
}
}
def get_requirements(config: Config):
py_reqs = []
if config.graph:
py_reqs = list(
map(Record.from_distribution,
filter(config.keep_dist, im.distributions())))
else:
py_reqs = list(map(Record.from_name, config.includes))
py_reqs = list(filter(lambda r: r is not None, py_reqs))
js_reqs = get_js_requirements(config)
reqs = [*py_reqs, *js_reqs]
reqs.sort(key=lambda r: r.name)
if config.verbose:
for req in reqs:
pprint(req.__dict__)
return reqs
def perform(args):
import importlib_metadata # type: ignore
import jinja2 # type: ignore
from packaging.requirements import Requirement # type: ignore
from packaging.utils import canonicalize_name # type: ignore
if args.python:
output = check_output([
args.python, '-c',
'import json, sys; json.dump(sys.path, sys.stdout)'
])
sys_paths = json.loads(output)
args.path = [p for p in sys_paths if path.isdir(p)]
kdargs = dict()
if args.path:
kdargs.update(path=args.path)
dists = list(importlib_metadata.distributions(**kdargs))
requires_extras = lru_cache(maxsize=None)(partial(_get_requires_extras,
dists))
installed = lru_cache(maxsize=None)(partial(_installed, dists))
context = {
'distributions': dists,
'options': args,
'in_site': in_site,
'in_usersite': in_usersite,
'canonicalize_name': canonicalize_name,
'requires_extras': requires_extras,
'installed': installed,
'Requirement': Requirement,
}
if args.template:
template = jinja2.Environment(
loader=jinja2.FileSystemLoader(''),
extensions=['jinja2.ext.do']).get_template(args.template)
else:
template = jinja2.Environment(
loader=jinja2.PackageLoader(__package__),
extensions=['jinja2.ext.do']).get_template('default.dot.j2')
for s in template.generate(context):
args.output.write(s)
print('', file=args.output)
def entry_points_with_dist(group: str) -> Iterator[tuple[metadata.EntryPoint, metadata.Distribution]]:
"""Retrieve entry points of the given group.
This is like the ``entry_points()`` function from importlib.metadata,
except it also returns the distribution the entry_point was loaded from.
:param group: Filter results to only this entrypoint group
:return: Generator of (EntryPoint, Distribution) objects for the specified groups
"""
loaded: set[str] = set()
for dist in metadata.distributions():
key = canonicalize_name(dist.metadata["Name"])
if key in loaded:
continue
loaded.add(key)
for e in dist.entry_points:
if e.group != group:
continue
yield e, dist
def get_all_entry_points():
"""
Get all entry points related to ``ros2cli`` and any of its extensions.
:returns: mapping of entry point names to ``EntryPoint`` instances
:rtype: dict
"""
extension_points = get_entry_points(EXTENSION_POINT_GROUP_NAME)
entry_points = defaultdict(dict)
for dist in importlib_metadata.distributions():
for ep in dist.entry_points:
# skip groups which are not registered as extension points
if ep.group not in extension_points:
continue
entry_points[ep.group][ep.name] = (dist, ep)
return entry_points
def log_packages_neptune(neptune_logger):
"""Uses the neptunecontrib.api to log the packages of the current python env."""
from neptunecontrib.api import log_table
import pandas as pd
import importlib_metadata
dists = importlib_metadata.distributions()
packages = {
idx: (dist.metadata['Name'], dist.version)
for idx, dist in enumerate(dists)
}
packages_df = pd.DataFrame.from_dict(packages,
orient='index',
columns=['package', 'version'])
log_table(name='packages',
table=packages_df,
experiment=neptune_logger.experiment)
def _mark_plugins_for_rewrite(self, hook):
"""
Given an importhook, mark for rewrite any top-level
modules or packages in the distribution package for
all pytest plugins.
"""
self.pluginmanager.rewrite_hook = hook
if os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"):
# We don't autoload from setuptools entry points, no need to continue.
return
package_files = (str(file)
for dist in importlib_metadata.distributions()
if any(ep.group == "pytest11"
for ep in dist.entry_points)
for file in dist.files or [])
for name in _iter_rewritable_modules(package_files):
hook.mark_rewrite(name)
def load_setuptools_entrypoints(self, group, name=None):
""" Load modules from querying the specified setuptools ``group``.
:param str group: entry point group to load plugins
:param str name: if given, loads only plugins with the given ``name``.
:rtype: int
:return: return the number of loaded plugins by this call.
"""
count = 0
for dist in importlib_metadata.distributions():
for ep in dist.entry_points:
if (ep.group != group or (name is not None and ep.name != name)
# already registered
or
self.get_plugin(ep.name) or self.is_blocked(ep.name)):
continue
plugin = ep.load()
self.register(plugin, name=ep.name)
self._plugin_distinfo.append((plugin, DistFacade(dist)))
count += 1
return count
def _mark_plugins_for_rewrite(self, hook):
"""
Given an importhook, mark for rewrite any top-level
modules or packages in the distribution package for
all pytest plugins.
"""
self.pluginmanager.rewrite_hook = hook
if os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"):
# We don't autoload from setuptools entry points, no need to continue.
return
package_files = (
str(file)
for dist in importlib_metadata.distributions()
if any(ep.group == "pytest11" for ep in dist.entry_points)
for file in dist.files
)
for name in _iter_rewritable_modules(package_files):
hook.mark_rewrite(name)
def list_extensions() -> List[ExtensionInfo]:
"""
List all installed Chaos Toolkit extensions in the current environment.
Notice, for now we can only list extensions that start with `chaostoolkit-`
in their package name.
This is not as powerful and solid as we want it to be. The trick is that we
can't rely on any metadata inside extensions to tell us they exist and
what functionnality they provide either. Python has the concept of trove
classifiers on packages but we can't extend them yet so they are of no use
to us.
In a future version, we will provide a mechanism from packages to support
a better detection.
"""
infos = []
distros = importlib_metadata.distributions()
seen = []
for dist in distros:
info = dist.metadata
name = info['Name']
if name == "chaostoolkit-lib":
continue
if name in seen:
continue
seen.append(name)
if name.startswith("chaostoolkit-"):
ext = ExtensionInfo(name=name,
version=info["Version"],
summary=info["Summary"],
license=info["License"],
author=info["Author"],
url=info["Home-page"])
infos.append(ext)
return infos
def load_setuptools_entrypoints(self, group, name=None):
""" Load modules from querying the specified setuptools ``group``.
:param str group: entry point group to load plugins
:param str name: if given, loads only plugins with the given ``name``.
:rtype: int
:return: return the number of loaded plugins by this call.
"""
count = 0
for dist in importlib_metadata.distributions():
for ep in dist.entry_points:
if (
ep.group != group
or (name is not None and ep.name != name)
# already registered
or self.get_plugin(ep.name)
or self.is_blocked(ep.name)
):
continue
plugin = ep.load()
self.register(plugin, name=ep.name)
self._plugin_distinfo.append((plugin, DistFacade(dist)))
count += 1
return count
def get_requirements(excludes=excludes):
licenses = get_licenses(excludes=excludes)
predicate = lambda dist: dist.metadata['Name'] not in excludes
py_reqs = list(
map(
lambda d: inflate(d, licenses),
filter(predicate, im.distributions())
)
)
js_reqs_json = subprocess.run(
['node', cwd.joinpath('notice.js')],
capture_output=True
).stdout.decode(encoding=encoding)
js_reqs = list(
map(
lambda r: Record(**r),
json.loads(js_reqs_json)
)
)
custom_reqs = load_custom_requirements()
reqs = [*py_reqs, *js_reqs, *custom_reqs]
reqs.sort(key=lambda r: r.name)
return reqs
def test_discovery(self):
"""
Discovering distributions should succeed even if
there is an invalid path on sys.path.
"""
list(importlib_metadata.distributions())
def test_invalid_usage(self):
with self.assertRaises(ValueError):
list(distributions(context='something', name='else'))
def test_package_discovery(self):
dists = list(distributions())
assert all(isinstance(dist, Distribution) for dist in dists)
assert any(dist.metadata['Name'] == 'egginfo-pkg' for dist in dists)
assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists)
from importlib_metadata import distributions
for d in distributions():
print(d.metadata["Name"], d.version)
def get_installed_distributions():
return ([
Dist(project_name=it.metadata["Name"],
version=it.version,
check_version_conflict=(lambda: None)) for it in distributions()
])
def test_one_distribution(self):
dists = list(distributions(path=sys.path[:1]))
assert len(dists) == 1
def _find_local_distribution(self):
dist, = importlib_metadata.distributions(path=['.', 'src'])
return dist
def _findPlugins(self):
return [pkg.metadata["Name"]
for pkg in importlib_metadata.distributions()
if pkg.metadata["Name"].startswith(PluginManager.prefix)]