def installer(req):
# Attempt to obtain the egg from the local cache. If it's an exact match, we can use it.
# If it's not an exact match, then if it's been resolved sufficiently recently, we still
# use it.
dist = egg_obtainer.obtain(req)
if dist and (requirement_is_exact(req) or now - os.path.getmtime(dist.location) < ttl):
return dist
# Failed, so follow through to "remote" resolution
source_translator = SourceTranslator(
interpreter=interpreter,
use_2to3=getattr(req, 'use_2to3', False),
**shared_options)
translator = ChainedTranslator(egg_translator, source_translator)
obtainer = Obtainer(
crawler,
[Fetcher([req.repository])] if getattr(req, 'repository', None) else fetchers,
translator)
dist = obtainer.obtain(req)
if dist:
try:
touch(dist.location)
except OSError:
pass
return dist
def installer(req):
# Attempt to obtain the egg from the local cache. If it's an exact match, we can use it.
# If it's not an exact match, then if it's been resolved sufficiently recently, we still
# use it.
dist = egg_obtainer.obtain(req)
if dist and (requirement_is_exact(req) or now - os.path.getmtime(dist.location) < ttl):
return dist
# Failed, so follow through to "remote" resolution
source_translator = SourceTranslator(
interpreter=interpreter,
use_2to3=getattr(req, 'use_2to3', False),
**shared_options)
translator = ChainedTranslator(egg_translator, source_translator)
obtainer = Obtainer(
crawler,
[Fetcher([req.repository])] if getattr(req, 'repository', None) else fetchers,
translator)
dist = obtainer.obtain(req)
if dist:
try:
touch(dist.location)
except OSError:
pass
return dist
def resolve_and_link(config,
requirement,
target_link,
installer_provider,
logger=print):
if os.path.exists(target_link) and os.path.exists(
os.path.realpath(target_link)):
egg = EggPackage(os.path.realpath(target_link))
if egg.satisfies(requirement):
return egg
fetchers = fetchers_from_config(config)
crawler = crawler_from_config(config)
obtainer = Obtainer(crawler, fetchers, [])
obtainer_iterator = obtainer.iter(requirement)
links = [
link for link in obtainer_iterator if isinstance(link, SourcePackage)
]
for link in links:
logger(' fetching %s' % link.url)
sdist = link.fetch()
logger(' installing %s' % sdist)
installer = installer_provider(sdist)
dist_location = installer.bdist()
target_location = os.path.join(os.path.dirname(target_link),
os.path.basename(dist_location))
shutil.move(dist_location, target_location)
safe_link(target_location, target_link)
logger(' installed %s' % target_location)
return EggPackage(target_location)
def iter(self, requirement):
if hasattr(requirement, 'repository') and requirement.repository:
obtainer = Obtainer(crawler=self._crawler,
fetchers=[Fetcher([requirement.repository])],
translators=self._translator)
for package in obtainer.iter(requirement):
yield package
else:
for package in super(PantsObtainer, self).iter(requirement):
yield package
def iter(self, requirement):
if hasattr(requirement, 'repository') and requirement.repository:
obtainer = Obtainer(
crawler=self._crawler,
fetchers=[Fetcher([requirement.repository])],
translators=self._translator)
for package in obtainer.iter(requirement):
yield package
else:
for package in super(PantsObtainer, self).iter(requirement):
yield package
def setup_distribute(self, interpreter, dest):
obtainer = Obtainer(self._crawler, self._fetchers, [])
obtainer_iterator = obtainer.iter(self._setuptools_requirement)
links = [link for link in obtainer_iterator if isinstance(link, SourceLink)]
for link in links:
self._logger('Fetching %s' % link)
sdist = link.fetch()
self._logger('Installing %s' % sdist)
installer = Installer(sdist, strict=False, interpreter=interpreter)
dist = installer.distribution()
self._logger('Distilling %s' % dist)
egg = Distiller(dist).distill(into=dest)
safe_link(egg, os.path.join(dest, 'distribute'))
break
def test_package_precedence():
source = SourcePackage('psutil-0.6.1.tar.gz')
egg = EggPackage('psutil-0.6.1-py2.6.egg')
# default precedence
assert Obtainer.package_precedence(egg) > Obtainer.package_precedence(source)
# overridden precedence
PRECEDENCE = (EggPackage,)
assert Obtainer.package_precedence(source, PRECEDENCE) == (source.version, -1) # unknown rank
PRECEDENCE = (SourcePackage, EggPackage)
assert Obtainer.package_precedence(source, PRECEDENCE) > Obtainer.package_precedence(
egg, PRECEDENCE)
def setup_distribute(self, interpreter, dest):
obtainer = Obtainer(self._crawler, self._fetchers, [])
obtainer_iterator = obtainer.iter(self._setuptools_requirement)
links = [
link for link in obtainer_iterator if isinstance(link, SourceLink)
]
for link in links:
self._logger('Fetching %s' % link)
sdist = link.fetch()
self._logger('Installing %s' % sdist)
installer = Installer(sdist, strict=False, interpreter=interpreter)
dist = installer.distribution()
self._logger('Distilling %s' % dist)
egg = Distiller(dist).distill(into=dest)
safe_link(egg, os.path.join(dest, 'distribute'))
break
def test_href_translation():
VERSIONS = ['0.4.0', '0.4.1', '0.5.0', '0.6.0']
def fake_link(version):
return 'http://www.example.com/foo/bar/psutil-%s.tar.gz' % version
fc = FakeCrawler([fake_link(v) for v in VERSIONS])
ob = Obtainer(fc, [], [])
for v in VERSIONS:
pkgs = list(ob.iter(Requirement.parse('psutil==%s' % v)))
assert len(pkgs) == 1, 'Version: %s' % v
assert pkgs[0] == SourceLink(fake_link(v))
assert list(ob.iter(Requirement.parse('psutil>=0.5.0'))) == [
SourceLink(fake_link('0.6.0')),
SourceLink(fake_link('0.5.0'))]
assert list(ob.iter(Requirement.parse('psutil'))) == [
SourceLink(fake_link(v)) for v in reversed(VERSIONS)]
def _resolve_and_link(config, requirement, target_link, installer_provider, logger=print):
if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)):
egg = EggPackage(os.path.realpath(target_link))
if egg.satisfies(requirement):
return egg
fetchers = fetchers_from_config(config)
crawler = crawler_from_config(config)
obtainer = Obtainer(crawler, fetchers, [])
obtainer_iterator = obtainer.iter(requirement)
links = [link for link in obtainer_iterator if isinstance(link, SourcePackage)]
for link in links:
logger(' fetching %s' % link.url)
sdist = link.fetch()
logger(' installing %s' % sdist)
installer = installer_provider(sdist)
dist_location = installer.bdist()
target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location))
shutil.move(dist_location, target_location)
_safe_link(target_location, target_link)
logger(' installed %s' % target_location)
return EggPackage(target_location)
def test_href_translation():
VERSIONS = ['0.4.0', '0.4.1', '0.5.0', '0.6.0']
def fake_link(version):
return 'http://www.example.com/foo/bar/psutil-%s.tar.gz' % version
fc = FakeCrawler(map(fake_link, VERSIONS))
ob = Obtainer(fc, [], [])
for v in VERSIONS:
pkgs = list(ob.iter(Requirement.parse('psutil==%s' % v)))
assert len(pkgs) == 1
assert pkgs[0] == SourceLink(fake_link(v))
assert list(ob.iter(Requirement.parse('psutil>=0.5.0'))) == [
SourceLink(fake_link('0.6.0')),
SourceLink(fake_link('0.5.0'))
]
assert list(ob.iter(Requirement.parse('psutil'))) == map(
SourceLink, map(fake_link, reversed(VERSIONS)))
def make_installer(self, reqs, interpreter, platform):
assert len(reqs) == 1 and isinstance(
reqs[0],
PythonRequirement), 'Got requirement list: %s' % (repr(reqs))
req = reqs[0]
fetchers = [Fetcher([req.repository])
] + self._fetchers if req.repository else self._fetchers
translator = ChainedTranslator(
EggTranslator(install_cache=self._install_cache,
platform=platform,
python=interpreter.python),
SourceTranslator(install_cache=self._install_cache,
interpreter=interpreter,
platform=platform,
use_2to3=req.use_2to3))
obtainer = Obtainer(self._crawler, fetchers, translator)
return obtainer.obtain
def resolve_multi(config,
requirements,
interpreter=None,
platforms=None,
conn_timeout=None,
ttl=3600):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param config: Pants :class:`Config` object.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param interpreter: :class:`PythonInterpreter` for which requirements should be resolved.
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param conn_timeout: Optional connection timeout for any remote fetching.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
"""
now = time.time()
distributions = {}
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError('Expected interpreter to be a PythonInterpreter, got %s' % type(interpreter))
install_cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs')
platforms = get_platforms(platforms or config.getlist('python-setup', 'platforms', ['current']))
crawler = crawler_from_config(config, conn_timeout=conn_timeout)
fetchers = fetchers_from_config(config)
for platform in platforms:
env = PantsEnvironment(search_path=[], platform=platform, python=interpreter.python)
working_set = WorkingSet(entries=[])
shared_options = dict(install_cache=install_cache, platform=platform)
egg_translator = EggTranslator(python=interpreter.python, **shared_options)
egg_obtainer = Obtainer(crawler, [Fetcher([install_cache])], egg_translator)
def installer(req):
# Attempt to obtain the egg from the local cache. If it's an exact match, we can use it.
# If it's not an exact match, then if it's been resolved sufficiently recently, we still
# use it.
dist = egg_obtainer.obtain(req)
if dist and (requirement_is_exact(req) or now - os.path.getmtime(dist.location) < ttl):
return dist
# Failed, so follow through to "remote" resolution
source_translator = SourceTranslator(
interpreter=interpreter,
use_2to3=getattr(req, 'use_2to3', False),
**shared_options)
translator = ChainedTranslator(egg_translator, source_translator)
obtainer = Obtainer(
crawler,
[Fetcher([req.repository])] if getattr(req, 'repository', None) else fetchers,
translator)
dist = obtainer.obtain(req)
if dist:
try:
touch(dist.location)
except OSError:
pass
return dist
distributions[platform] = working_set.resolve(requirements, env=env, installer=installer)
return distributions
def test_link_preference():
sl = SourceLink('psutil-0.6.1.tar.gz')
el = EggLink('psutil-0.6.1-py2.6.egg')
assert Obtainer.link_preference(el) > Obtainer.link_preference(sl)
def test_package_precedence():
source = SourcePackage('psutil-0.6.1.tar.gz')
egg = EggPackage('psutil-0.6.1-py2.6.egg')
whl = WheelPackage('psutil-0.6.1-cp26-none-macosx_10_4_x86_64.whl')
# default precedence
assert Obtainer.package_precedence(whl) > Obtainer.package_precedence(egg)
assert Obtainer.package_precedence(egg) > Obtainer.package_precedence(source)
assert Obtainer.package_precedence(whl) > Obtainer.package_precedence(source)
# overridden precedence
PRECEDENCE = (EggPackage, WheelPackage)
assert Obtainer.package_precedence(source, PRECEDENCE) == (source.version, -1) # unknown rank
assert Obtainer.package_precedence(whl, PRECEDENCE) > Obtainer.package_precedence(
source, PRECEDENCE)
assert Obtainer.package_precedence(egg, PRECEDENCE) > Obtainer.package_precedence(
whl, PRECEDENCE)
def test_package_precedence():
source = SourcePackage('psutil-0.6.1.tar.gz')
egg = EggPackage('psutil-0.6.1-py2.6.egg')
whl = WheelPackage('psutil-0.6.1-cp26-none-macosx_10_4_x86_64.whl')
# default precedence
assert Obtainer.package_precedence(whl) > Obtainer.package_precedence(egg)
assert Obtainer.package_precedence(egg) > Obtainer.package_precedence(
source)
assert Obtainer.package_precedence(whl) > Obtainer.package_precedence(
source)
# overridden precedence
PRECEDENCE = (EggPackage, WheelPackage)
assert Obtainer.package_precedence(source,
PRECEDENCE) == (source.version, -1
) # unknown rank
assert Obtainer.package_precedence(
whl, PRECEDENCE) > Obtainer.package_precedence(source, PRECEDENCE)
assert Obtainer.package_precedence(
egg, PRECEDENCE) > Obtainer.package_precedence(whl, PRECEDENCE)
def test_link_preference():
sl = SourceLink('psutil-0.6.1.tar.gz')
el = EggLink('psutil-0.6.1.egg')
assert Obtainer.link_preference(el) > Obtainer.link_preference(sl)