def identify_distributions(self, files):
unknown_files = set(files)
found_package_count = 0
venv_paths = map(self._get_venv_path, files)
venv_paths = set(filter(None, venv_paths))
venvs = []
for venv_path in venv_paths:
package_details, file_to_pkg = self._get_package_details(venv_path)
local_pkgs = set(
piputils.get_pip_packages(self._session,
venv_path + "/bin/pip",
restriction="local"))
pkg_to_found_files = defaultdict(list)
for path in set(unknown_files): # Clone the set
# The supplied path may be relative or absolute, but
# file_to_pkg keys are absolute paths.
fullpath = os.path.abspath(path)
if fullpath in file_to_pkg:
unknown_files.remove(path)
pkg_to_found_files[file_to_pkg[fullpath]].append(
os.path.relpath(path, venv_path))
# Some files, like venvs/dev/lib/python2.7/abc.py could be
# symlinks populated by virtualenv itself during venv creation
# since it relies on system wide python environment. So we need
# to resolve those into filenames which could be associated with
# system wide installation of python
for path in unknown_files.copy():
if is_subpath(path, venv_path) and op.islink(path):
unknown_files.add(op.realpath(path))
unknown_files.remove(path)
packages = []
for name, details in iteritems(package_details):
location = details["location"]
packages.append(
VenvPackage(name=details["name"],
version=details["version"],
local=name in local_pkgs,
location=location,
editable=details["editable"],
files=pkg_to_found_files[name]))
if location and not is_subpath(location, venv_path):
unknown_files.add(location)
found_package_count += len(packages)
venvs.append(
VenvEnvironment(path=venv_path,
python_version=self._python_version(venv_path),
packages=packages))
if venvs:
yield (VenvDistribution(name="venv",
venv_version=self._venv_version(),
path=self._venv_exe_path(),
environments=venvs), unknown_files)
def identify_distributions(self, paths):
conda_paths = set()
root_to_envs = defaultdict(list)
# Start with all paths being set as unknown
unknown_files = set(paths)
# Track count of found packages and files
found_package_count = 0
total_file_count = len(unknown_files)
# First, loop through all the files and identify conda paths
for path in paths:
conda_path = self._get_conda_env_path(path)
if conda_path:
if conda_path not in conda_paths:
conda_paths.add(conda_path)
# Loop through conda_paths, find packages and create the
# environments
for idx, conda_path in enumerate(conda_paths):
# Start with an empty channels list
channels = []
found_channel_names = set()
# Find the root path for the environment
# TODO: cache/memoize for those paths which have been considered
# since will be asked again below
conda_info = self._get_conda_info(conda_path)
root_path = conda_info.get('root_prefix')
if not root_path:
lgr.warning(
"Could not find root path for conda environment %s" %
conda_path)
continue
# Retrieve the environment details
env_export = self._get_conda_env_export(root_path, conda_path)
(conda_package_details, file_to_pkg) = \
self._get_conda_package_details(conda_path)
(conda_pip_package_details, file_to_pip_pkg) = \
self._get_conda_pip_package_details(env_export, conda_path)
# Join our conda and pip packages
conda_package_details.update(conda_pip_package_details)
file_to_pkg.update(file_to_pip_pkg)
# Initialize a map from packages to files that defaults to []
pkg_to_found_files = defaultdict(list)
# Get the conda path prefix to calculate relative paths
path_prefix = conda_path + os.path.sep
# Loop through unknown files, assigning them to packages if found
for path in set(unknown_files): # Clone the set
if path in file_to_pkg:
# The file was found so remove from unknown file set
unknown_files.remove(path)
# Make relative paths if it is begins with the conda path
if path.startswith(path_prefix):
rel_path = path[len(path_prefix):]
else:
rel_path = path
# And add to the package
pkg_to_found_files[file_to_pkg[path]].append(rel_path)
packages = []
# Create the packages in the environment
for package_name in conda_package_details:
details = conda_package_details[package_name]
# Look up or create the conda channel for the environment list
channel_name = details.get("schannel")
if channel_name:
if channel_name not in found_channel_names:
# New channel for our environment, so add it
channel_url = details.get("channel")
found_channel_names.add(channel_name)
channels.append(
CondaChannel(name=channel_name, url=channel_url))
location = details.get("location")
# Create the package
package = CondaPackage(name=details.get("name"),
installer=details.get("installer"),
version=details.get("version"),
build=details.get("build"),
channel_name=channel_name,
size=details.get("size"),
md5=details.get("md5"),
url=details.get("url"),
location=location,
editable=details.get("editable"),
files=pkg_to_found_files[package_name])
packages.append(package)
# Make editable pip packages available to other tracers.
if location and not is_subpath(location, conda_path):
unknown_files.add(location)
# Give the distribution a name
# Determine name from path (Alt approach: use conda-env info)
if os.path.normpath(conda_path) == os.path.normpath(root_path):
env_name = "root"
else:
env_name = os.path.basename(os.path.normpath(conda_path))
# Keep track of found package count
found_package_count += len(packages)
# Create the conda environment (works with root environments too)
conda_env = CondaEnvironment(name=env_name,
path=conda_path,
packages=packages,
channels=channels)
root_to_envs[root_path].append(conda_env)
lgr.debug("%s: %d packages with %d files, and %d other files",
self.__class__.__name__, found_package_count,
total_file_count - len(unknown_files), len(unknown_files))
# Find all the identified conda_roots
conda_roots = root_to_envs.keys()
# Loop through conda_roots and create the distributions
for idx, root_path in enumerate(conda_roots):
# Retrieve distribution details
conda_info = self._get_conda_info(root_path)
# Give the distribution a name
if (len(conda_roots)) > 1:
dist_name = 'conda-%d' % idx
else:
dist_name = 'conda'
dist = CondaDistribution(
name=dist_name,
conda_version=conda_info.get("conda_version"),
python_version=conda_info.get("python_version"),
platform=conda_info.get("platform"),
path=root_path,
environments=root_to_envs[root_path])
yield dist, unknown_files
def identify_distributions(self, files):
unknown_files = set(files)
found_package_count = 0
venv_paths = map(self._get_venv_path, files)
venv_paths = set(filter(None, venv_paths))
venvs = []
for venv_path in venv_paths:
package_details, file_to_pkg = self._get_package_details(venv_path)
local_pkgs = set(
piputils.get_pip_packages(self._session,
venv_path + "/bin/pip",
restriction="local"))
pkg_to_found_files = defaultdict(list)
for path in set(unknown_files): # Clone the set
# The supplied path may be relative or absolute, but
# file_to_pkg keys are absolute paths.
fullpath = os.path.abspath(path)
if fullpath in file_to_pkg:
unknown_files.remove(path)
pkg_to_found_files[file_to_pkg[fullpath]].append(
os.path.relpath(path, venv_path))
# Some virtualenv files are links to system files. Files themselves
# may be linked or they may be in a linked directory. We need to
# resolve these links and pass them out as unknown files for other
# tracers to use.
for path in unknown_files.copy():
if is_subpath(path, venv_path):
rpath = op.realpath(path)
# ... but the resolved link may point to another path under
# the environment (e.g., bin/python -> bin/python3), and we
# don't want to pass that back out as unknown.
if not is_subpath(rpath, venv_path):
unknown_files.add(rpath)
unknown_files.remove(path)
packages = []
for name, details in package_details.items():
location = details["location"]
packages.append(
VenvPackage(name=details["name"],
version=details["version"],
local=name in local_pkgs,
location=location,
editable=details["editable"],
files=pkg_to_found_files[name]))
if location and not is_subpath(location, venv_path):
unknown_files.add(location)
found_package_count += len(packages)
venvs.append(
VenvEnvironment(path=venv_path,
python_version=self._python_version(venv_path),
packages=packages))
if venvs:
yield (VenvDistribution(name="venv",
venv_version=self._venv_version(),
path=self._venv_exe_path(),
environments=venvs), unknown_files)