def test_get_distribution(self):
# Test for looking up a distribution by name.
# Test the lookup of the towel-stuff distribution
name = 'towel-stuff' # Note: This is different from the directory name
d = DistributionPath()
ed = DistributionPath(include_egg=True)
# Lookup the distribution
dist = d.get_distribution(name)
self.assertIsInstance(dist, InstalledDistribution)
self.assertEqual(dist.name, name)
# Verify that an unknown distribution returns None
self.assertIsNone(d.get_distribution('bogus'))
# Verify partial name matching doesn't work
self.assertIsNone(d.get_distribution('towel'))
# Verify that it does not find egg-info distributions, when not
# instructed to
self.assertIsNone(d.get_distribution('bacon'))
self.assertIsNone(d.get_distribution('cheese'))
self.assertIsNone(d.get_distribution('strawberry'))
self.assertIsNone(d.get_distribution('banana'))
# Now check that it works well in both situations, when egg-info
# is a file and directory respectively.
for name in ('cheese', 'bacon', 'banana', 'strawberry'):
dist = ed.get_distribution(name)
self.assertIsInstance(dist, EggInfoDistribution)
self.assertEqual(dist.name, name)
def test_yield_distribution(self):
# tests the internal function _yield_distributions
checkLists = lambda x, y: self.assertEqual(sorted(x), sorted(y))
eggs = [('bacon', '0.1'), ('banana', '0.4'), ('strawberry', '0.6'),
('truffles', '5.0'), ('cheese', '2.0.2'),
('coconuts-aster', '10.3'), ('nut', 'funkyversion')]
dists = [('choxie', '2.0.0.9'), ('grammar', '1.0a4'),
('towel-stuff', '0.1'), ('babar', '0.1')]
d = DistributionPath(include_egg=False)
d._include_dist = False
checkLists([], d._yield_distributions())
d = DistributionPath(include_egg=True)
d._include_dist = False
found = [(dist.name, dist.version)
for dist in d._yield_distributions()
if dist.path.startswith(self.fake_dists_path)]
checkLists(eggs, found)
d = DistributionPath()
found = [(dist.name, dist.version)
for dist in d._yield_distributions()
if dist.path.startswith(self.fake_dists_path)]
checkLists(dists, found)
d = DistributionPath(include_egg=True)
found = [(dist.name, dist.version)
for dist in d._yield_distributions()
if dist.path.startswith(self.fake_dists_path)]
checkLists(dists + eggs, found)
def test_read_exports(self):
d = DistributionPath().get_distribution('babar')
r = d.exports
self.assertIn('foo', r)
d = r['foo']
self.assertIn('bar', d)
self.check_entry(d['bar'], 'bar', 'baz', 'barbaz', ['a=10', 'b'])
self.assertIn('bar.baz', r)
d = r['bar.baz']
self.assertIn('foofoo', d)
self.check_entry(d['foofoo'], 'foofoo', 'baz.foo', 'bazbar', [])
self.assertIn('real', d)
e = d['real']
self.check_entry(e, 'real', 'cgi', 'print_directory', [])
import cgi
self.assertIs(e.value, cgi.print_directory)
# See issue #78. Test reading an entry_points.txt with leading spaces
TEST_EXPORTS = b"""
[paste.server_runner]
main = waitress:serve_paste
[console_scripts]
waitress-serve = waitress.runner:run
"""
with io.BytesIO(TEST_EXPORTS) as f:
exports = read_exports(f)
self.assertEqual(set(exports.keys()),
set(['paste.server_runner', 'console_scripts']))
def test_path(self):
fakes = os.path.join(HERE, 'fake_dists')
sys.path.insert(0, fakes)
try:
edp = DistributionPath(include_egg=True)
locator = DistPathLocator(edp)
cases = ('babar', 'choxie', 'strawberry', 'towel-stuff',
'coconuts-aster', 'bacon', 'grammar', 'truffles',
'banana', 'cheese')
for name in cases:
d = locator.locate(name, True)
self.assertIsNotNone(d)
r = locator.get_project(name)
expected = {
d.version: d,
'urls': {
d.version: set([d.source_url])
},
'digests': {
d.version: set([None])
}
}
self.assertEqual(r, expected)
d = locator.locate('nonexistent')
self.assertIsNone(d)
r = locator.get_project('nonexistent')
self.assertTrue(len(r) == 2)
finally:
sys.path.pop(0)
def installed_path(): """Return '''DistributionPath''' for installed packages.""" global _inst_path if _inst_path is None: from distlib.database import DistributionPath _inst_path = DistributionPath() return _inst_path
def install_dist(distname, workdir):
pfx = '--install-option='
purelib = pfx + '--install-purelib=%s/purelib' % workdir
platlib = pfx + '--install-platlib=%s/platlib' % workdir
headers = pfx + '--install-headers=%s/headers' % workdir
scripts = pfx + '--install-scripts=%s/scripts' % workdir
data = pfx + '--install-data=%s/data' % workdir
cmd = [
'pip', 'install', '--index-url', 'https://pypi.org/simple/',
'--timeout', '3', '--default-timeout', '3', purelib, platlib, headers,
scripts, data, distname
]
result = {
'scripts': os.path.join(workdir, 'scripts'),
'headers': os.path.join(workdir, 'headers'),
'data': os.path.join(workdir, 'data'),
}
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout, _ = p.communicate()
if p.returncode:
raise ValueError('pip failed to install %s:\n%s' % (distname, stdout))
for dn in ('purelib', 'platlib'):
libdir = os.path.join(workdir, dn)
if os.path.isdir(libdir):
result[dn] = libdir
break
convert_egg_info(libdir, workdir)
dp = DistributionPath([libdir])
dist = next(dp.get_distributions())
md = dist.metadata
result['name'] = md.name
result['version'] = md.version
return result
def dependents(self, logger):
"""Supported API. Return set of bundles that directly depends on this one.
Parameters
----------
logger : :py:class:`~chimerax.core.logger.Logger` instance
Where to log error messages.
Returns
-------
set of :py:class:`~chimerax.core.toolshed.BundleInfo` instances
Dependent bundles.
"""
from . import get_toolshed
from distlib.database import DistributionPath
keep = set()
for d in DistributionPath().get_distributions():
for req in d.run_requires:
if req.split()[0] == self.name:
keep.add(d)
break
ts = get_toolshed()
deps = set()
for d in keep:
bi = ts.find_bundle(d.name, logger)
if bi:
deps.add(bi)
return deps
def register_dist(dist_name, target_prefix):
""" register a distribution with conda. """
# build path to site-packages directory
get_python_version = UnlinkLinkTransaction.get_python_version
python_ver = get_python_version(target_prefix, [], [])
sp_short_path = get_python_site_packages_short_path(python_ver)
sp_full_path = os.path.join(target_prefix, sp_short_path)
# find package details using distlib
dist_path = DistributionPath([
sp_full_path,
], include_egg=True)
dist = dist_path.get_distribution(dist_name)
# create a conda PrefixRecord
files = [
os.path.join(sp_short_path, file_path)
for file_path, file_hash, file_size in dist.list_installed_files()
]
prefix_record = PrefixRecord.from_objects(
name=dist.name,
version=dist.version,
files=files,
build=DEFAULT_BUILD_STR,
build_number=DEFAULT_BUILD_NUMBER,
)
# TODO: conda currently checks that prefix_record.fn ends with .tar.bz2.
# This check should be supressed for prefix record entries which are not
# derived from tarballs so that prefix_record.fn can be set to None.
prefix_record.fn = prefix_record.fn + '.tar.bz2'
print("creating linked package record for %s." % dist_name)
PrefixData(target_prefix).insert(prefix_record)
def get_dists(self, names, include_egg=False):
dists = []
d = DistributionPath(include_egg=include_egg)
for name in names:
dist = d.get_distribution(name)
self.assertNotEqual(dist, None)
dists.append(dist)
return dists
def test_get_distributions(self):
# Lookup all distributions found in the ``sys.path``.
# This test could potentially pick up other installed distributions
non_egg_dists = [('grammar', '1.0a4'), ('choxie', '2.0.0.9'),
('towel-stuff', '0.1'), ('babar', '0.1')]
egg_dists = [('bacon', '0.1'), ('cheese', '2.0.2'),
('coconuts-aster', '10.3'), ('banana', '0.4'),
('strawberry', '0.6'), ('truffles', '5.0'),
('nut', 'funkyversion')]
all_dists = non_egg_dists + egg_dists
d = DistributionPath()
ed = DistributionPath(include_egg=True)
cases = ((d, non_egg_dists, InstalledDistribution),
(ed, all_dists, BaseInstalledDistribution))
fake_dists_path = self.fake_dists_path
for enabled in (True, False):
if not enabled:
d.cache_enabled = False
ed.cache_enabled = False
d.clear_cache()
ed.clear_cache()
for distset, fake_dists, allowed_class in cases:
found_dists = []
# Verify the fake dists have been found.
dists = list(distset.get_distributions())
for dist in dists:
self.assertIsInstance(dist, allowed_class)
if (dist.name in dict(fake_dists)
and dist.path.startswith(fake_dists_path)):
found_dists.append((dist.name, dist.version))
else:
# check that it doesn't find anything more than this
self.assertFalse(dist.path.startswith(fake_dists_path))
# otherwise we don't care what other dists are found
# Finally, test that we found all that we were looking for
self.assertEqual(sorted(found_dists), sorted(fake_dists))
def convert_egg_info(libdir, prefix, options):
files = os.listdir(libdir)
ei = list(filter(lambda d: d.endswith('.egg-info'), files))[0]
olddn = os.path.join(libdir, ei)
di = EGG_INFO_RE.sub('.dist-info', ei)
newdn = os.path.join(libdir, di)
os.rename(olddn, newdn)
if options.compatible:
renames = {}
else:
renames = {
'entry_points.txt': 'EXPORTS',
}
excludes = set([
'SOURCES.txt', # of no interest in/post WHEEL
'installed-files.txt', # replaced by RECORD, so not needed
'requires.txt', # added to METADATA, so not needed
'PKG-INFO', # replaced by METADATA
'not-zip-safe', # not applicable
])
files = os.listdir(newdn)
metadata = mdname = reqts = None
for oldfn in files:
pn = os.path.join(newdn, oldfn)
if oldfn in renames:
os.rename(pn, os.path.join(newdn, renames[oldfn]))
else:
if oldfn == 'requires.txt':
with open(pn, 'r') as f:
reqts = get_requirements(f.read())
elif oldfn == 'PKG-INFO':
metadata = Metadata(path=pn)
pd = get_package_data(metadata.name, metadata.version)
metadata = Metadata(mapping=pd['index-metadata'])
mdname = os.path.join(newdn, 'pydist.json')
if oldfn in excludes or not options.compatible:
os.remove(pn)
if metadata:
# Use Metadata 1.2 or later
metadata.provides += ['%s (%s)' % (metadata.name,
metadata.version)]
# Update if not set up by get_package_data
if reqts and not metadata.run_requires:
metadata.dependencies = reqts
metadata.write(path=mdname)
manifest = Manifest(os.path.dirname(libdir))
manifest.findall()
paths = manifest.allfiles
dp = DistributionPath([libdir])
dist = next(dp.get_distributions())
dist.write_installed_files(paths, prefix)
def find_dependencies(self, package):
# This weird `reload()` is here cause the `get_provider` method that
# feeds `get_distribution` uses a variable (working_set) populated in
# the module body, so it won't get updated just by installing a new
# package.
name = distlib.database.parse_requirement(package).name
dist = DistributionPath().get_distribution(name)
# This is another ugly thing. There's no other way for retrieving the
# dependency list for a package until it's installed. If it is a wheel,
# though, the dependency format will be different.
# e.g: "ejson (==0.1.3)" will become "ejson==0.1.3"
for dependency in dist.requires.union(dist.test_requires):
self.env.request_install(self._spec2installable(dependency),
sender=self.name,
data={'dependency-of': package})
def test_get_file(self):
# Create a fake dist
temp_site_packages = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, temp_site_packages)
dist_name = 'test'
dist_info = os.path.join(temp_site_packages, 'test-0.1.dist-info')
os.mkdir(dist_info)
metadata_path = os.path.join(dist_info, 'pydist.json')
resources_path = os.path.join(dist_info, 'RESOURCES')
md = Metadata()
md.name = 'test'
md.version = '0.1'
md.summary = 'test'
md.write(path=metadata_path)
test_path = 'test.cfg'
fd, test_resource_path = tempfile.mkstemp()
os.close(fd)
self.addCleanup(os.remove, test_resource_path)
fp = open(test_resource_path, 'w')
try:
fp.write('Config')
finally:
fp.close()
fp = open(resources_path, 'w')
try:
fp.write('%s,%s' % (test_path, test_resource_path))
finally:
fp.close()
# Add fake site-packages to sys.path to retrieve fake dist
self.addCleanup(sys.path.remove, temp_site_packages)
sys.path.insert(0, temp_site_packages)
# Try to retrieve resources paths and files
d = DistributionPath()
self.assertEqual(d.get_file_path(dist_name, test_path),
test_resource_path)
self.assertRaises(KeyError, d.get_file_path, dist_name, 'i-dont-exist')
def install_dist(distname, workdir, options):
pfx = '--install-option='
purelib = pfx + '--install-purelib=%s/purelib' % workdir
platlib = pfx + '--install-platlib=%s/platlib' % workdir
headers = pfx + '--install-headers=%s/headers' % workdir
scripts = pfx + '--install-scripts=%s/scripts' % workdir
data = pfx + '--install-data=%s/data' % workdir
# Use the pip adjacent to sys.executable, if any (for virtualenvs)
d = os.path.dirname(sys.executable)
files = filter(lambda o: o in ('pip', 'pip.exe'), os.listdir(d))
if not files:
prog = 'pip'
else:
prog = os.path.join(d, next(files))
cmd = [prog, 'install',
'--no-deps', '--quiet',
'--index-url', 'https://pypi.python.org/simple/',
'--timeout', '3', '--default-timeout', '3',
purelib, platlib, headers, scripts, data, distname]
result = {
'scripts': os.path.join(workdir, 'scripts'),
'headers': os.path.join(workdir, 'headers'),
'data': os.path.join(workdir, 'data'),
}
print('Pipping %s ...' % distname)
p = subprocess.Popen(cmd, shell=False, stdout=sys.stdout,
stderr=subprocess.STDOUT)
stdout, _ = p.communicate()
if p.returncode:
raise ValueError('pip failed to install %s:\n%s' % (distname, stdout))
for dn in ('purelib', 'platlib'):
libdir = os.path.join(workdir, dn)
if os.path.isdir(libdir):
result[dn] = libdir
break
convert_egg_info(libdir, workdir, options)
dp = DistributionPath([libdir])
dist = next(dp.get_distributions())
md = dist.metadata
result['name'] = md.name
result['version'] = md.version
return result
def convert_egg_info(libdir, prefix):
files = os.listdir(libdir)
ei = list(filter(lambda d: d.endswith('.egg-info'), files))[0]
olddn = os.path.join(libdir, ei)
di = EGG_INFO_RE.sub('.dist-info', ei)
newdn = os.path.join(libdir, di)
os.rename(olddn, newdn)
files = os.listdir(newdn)
for oldfn in files:
pn = os.path.join(newdn, oldfn)
if oldfn == 'PKG-INFO':
md = Metadata(path=pn)
mn = os.path.join(newdn, METADATA_FILENAME)
md.write(mn)
os.remove(pn)
manifest = Manifest(os.path.dirname(libdir))
manifest.findall()
dp = DistributionPath([libdir])
dist = next(dp.get_distributions())
dist.write_installed_files(manifest.allfiles, prefix)
def get_package_metadata_paths():
"""
List of package metadata to be loaded on EMR cluster
"""
from distlib.database import DistributionPath
if len(_file_path_to_package_meta_path) > 0:
return _file_path_to_package_meta_path
dist_path = DistributionPath(include_egg=True)
for distribution in dist_path.get_distributions():
metadata_path = distribution.path
for installed_file_path, _hash, _size in distribution.list_installed_files():
absolute_installed_file_path = installed_file_path
if not os.path.isabs(installed_file_path):
absolute_installed_file_path = os.path.join(os.path.dirname(metadata_path), installed_file_path)
normalized_file_path = os.path.realpath(absolute_installed_file_path)
_file_path_to_package_meta_path[normalized_file_path] = metadata_path
return _file_path_to_package_meta_path
def test_exports_iteration(self):
d = DistributionPath()
expected = set((
('bar', 'baz', 'barbaz', ('a=10', 'b')),
('bar', 'crunchie', None, ()),
('bar', 'towel', 'towel', ()),
('baz', 'towel', 'beach_towel', ()),
))
entries = list(d.get_exported_entries('foo'))
for e in entries:
t = e.name, e.prefix, e.suffix, tuple(e.flags)
self.assertIn(t, expected)
expected.remove(t)
self.assertFalse(expected) # nothing left
expected = set((
('bar', 'baz', 'barbaz', ('a=10', 'b')),
('bar', 'crunchie', None, ()),
('bar', 'towel', 'towel', ()),
))
entries = list(d.get_exported_entries('foo', 'bar'))
for e in entries:
t = e.name, e.prefix, e.suffix, tuple(e.flags)
self.assertIn(t, expected)
expected.remove(t)
self.assertFalse(expected) # nothing left
expected = set((
('foofoo', 'baz.foo', 'bazbar', ()),
('real', 'cgi', 'print_directory', ()),
('foofoo', 'ferrero', 'rocher', ()),
('foobar', 'hoopy', 'frood', ('dent', )),
))
entries = list(d.get_exported_entries('bar.baz'))
for e in entries:
t = e.name, e.prefix, e.suffix, tuple(e.flags)
self.assertIn(t, expected)
expected.remove(t)
self.assertFalse(expected) # nothing left
def get_modules(path: str) -> Dict[str, List[str]]:
"""
Returns the actual file name that is imported for a module
"""
modules: Dict[str, str] = {}
dp = DistributionPath([path], include_egg=True)
dists = dp.get_distributions()
for dist in dists:
for filename, _, _ in dist.list_installed_files():
if filename.endswith(('.py')):
parts = os.path.splitext(filename)[0].split(os.sep)
if len(parts
) == 1: # windows sep varies with distribution type
parts = os.path.splitext(filename)[0].split('/')
if parts[-1].startswith(
'_') and not parts[-1].startswith('__'):
continue # ignore internals
elif parts[-1] == '__init__' and parts[-2] != "tests":
module = parts[-2]
if not dist.key in modules:
modules[dist.key] = []
modules[dist.key].append(module)
return modules
def uninstall(self, requirement):
# Currently we assume the distribution path contains only the last
# version installed
package_name = util.parse_requirement(requirement).name
distribution = DistributionPath(
include_egg=True).get_distribution(package_name)
# Oh distlib, if the distribution doesn't exist, we'll get None here
if not distribution:
raise exceptions.PackageNotInstalled(
"There's no package named {0} installed in your environment".
format(package_name))
# Distlib is not that smart about paths for files inside of
# distributions too, so to find the full path to the distribution
# files, we'll have to concatenate them to this base path manually :/
base = os.path.dirname(distribution.path)
# Let's now remove all the installed files
for path, hash_, size in distribution.list_installed_files():
os.unlink(os.path.join(base, path))
# Removing the package directories
os.rmdir(distribution.path)
def test_modules(self):
dp = DistributionPath(include_egg=True)
dist = dp.get_distribution('banana')
self.assertIsInstance(dist, EggInfoDistribution)
self.assertEqual(dist.modules, ['banana', 'cavendish'])
#!/usr/bin/env python from distlib.database import DistributionPath dist_path = DistributionPath() mylist = list(dist_path.get_distributions()) from distlib.database import make_graph print(make_graph(mylist))
def check_installed(cls, requirement):
path = DistributionPath(include_egg=True)
package_name = util.parse_requirement(requirement).name
return path.get_distribution(package_name) is not None
def get_distribution_from_source_file(file_name):
path = DistributionPath(include_egg=True)
distribution = path.get_distribution(
os.path.dirname(file_name) or file_name)
return distribution
def test_caches(self):
# sanity check for internal caches
d = DistributionPath()
for name in ('_cache', '_cache_egg'):
self.assertEqual(getattr(d, name).name, {})
self.assertEqual(getattr(d, name).path, {})
import tempfile
logger = logging.getLogger('wheeler')
from distlib.compat import configparser, filter
from distlib.database import DistributionPath, Distribution, make_graph
from distlib.locators import (JSONLocator, SimpleScrapingLocator,
AggregatingLocator, DependencyFinder)
from distlib.manifest import Manifest
from distlib.metadata import Metadata
from distlib.util import parse_requirement, get_package_data
from distlib.wheel import Wheel
EGG_INFO_RE = re.compile(r'(-py\d\.\d)?\.egg-info', re.I)
INSTALLED_DISTS = DistributionPath(include_egg=True)
def get_requirements(data):
lines = []
for line in data.splitlines():
line = line.strip()
if not line or line[0] == '#':
continue
lines.append(line)
reqts = []
extras = {}
result = {'install': reqts, 'extras': extras}
for line in lines:
if line[0] != '[':
reqts.append(line)
def test_provides(self):
# Test for looking up distributions by what they provide
checkLists = lambda x, y: self.assertEqual(sorted(x), sorted(y))
d = DistributionPath()
ed = DistributionPath(include_egg=True)
l = [dist.name for dist in d.provides_distribution('truffles')]
checkLists(l, ['choxie', 'towel-stuff'])
l = [dist.name for dist in d.provides_distribution('truffles', '1.0')]
checkLists(l, ['choxie', 'towel-stuff'])
l = [dist.name for dist in ed.provides_distribution('truffles', '1.0')]
checkLists(l, ['choxie', 'cheese', 'towel-stuff'])
l = [
dist.name for dist in d.provides_distribution('truffles', '1.1.2')
]
checkLists(l, ['towel-stuff'])
l = [dist.name for dist in d.provides_distribution('truffles', '1.1')]
checkLists(l, ['towel-stuff'])
l = [
dist.name
for dist in d.provides_distribution('truffles', '!=1.1,<=2.0')
]
checkLists(l, ['choxie', 'towel-stuff'])
l = [
dist.name
for dist in ed.provides_distribution('truffles', '!=1.1,<=2.0')
]
checkLists(l, ['choxie', 'bacon', 'cheese', 'towel-stuff'])
l = [dist.name for dist in d.provides_distribution('truffles', '>1.0')]
checkLists(l, ['towel-stuff'])
l = [dist.name for dist in d.provides_distribution('truffles', '>1.5')]
checkLists(l, [])
l = [
dist.name for dist in ed.provides_distribution('truffles', '>1.5')
]
checkLists(l, ['bacon', 'truffles'])
l = [
dist.name for dist in d.provides_distribution('truffles', '>=1.0')
]
checkLists(l, ['choxie', 'towel-stuff'])
l = [
dist.name
for dist in ed.provides_distribution('strawberry', '0.6')
]
checkLists(l, ['coconuts-aster', 'strawberry'])
l = [
dist.name
for dist in ed.provides_distribution('strawberry', '>=0.5')
]
checkLists(l, ['coconuts-aster', 'strawberry'])
l = [
dist.name
for dist in ed.provides_distribution('strawberry', '>0.6')
]
checkLists(l, [])
l = [dist.name for dist in ed.provides_distribution('banana', '0.4')]
checkLists(l, ['banana', 'coconuts-aster'])
l = [dist.name for dist in ed.provides_distribution('banana', '>=0.3')]
checkLists(l, ['banana', 'coconuts-aster'])
l = [dist.name for dist in ed.provides_distribution('banana', '!=0.4')]
checkLists(l, [])
