def dumped_chroot(self, targets):
# TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem
# declares it as a dependency. However if we don't then test_antlr() below fails on
# uninitialized options for that subsystem. Hopefully my pending (as of 9/2016) change
# to clean up how we initialize and create instances of subsystems in tests will make
# this problem go away.
self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem,
DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory])
python_repos = PythonRepos.global_instance()
ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance())
thrift_binary_factory = ThriftBinary.Factory.global_instance().create
interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos)
interpreter_cache.setup()
interpreters = list(interpreter_cache.matched_interpreters(
self.python_setup.interpreter_constraints))
self.assertGreater(len(interpreters), 0)
interpreter = interpreters[0]
with temporary_dir() as chroot:
pex_builder = PEXBuilder(path=chroot, interpreter=interpreter)
python_chroot = PythonChroot(python_setup=self.python_setup,
python_repos=python_repos,
ivy_bootstrapper=ivy_bootstrapper,
thrift_binary_factory=thrift_binary_factory,
interpreter=interpreter,
builder=pex_builder,
targets=targets,
platforms=['current'])
try:
python_chroot.dump()
yield pex_builder, python_chroot
finally:
python_chroot.delete()
def _resolve_multi(interpreter, requirements, platforms, find_links):
"""Multi-platform dependency resolution for PEX files.
Returns a list of distributions that must be included in order to satisfy a set of requirements.
That may involve distributions for multiple platforms.
:param interpreter: The :class:`PythonInterpreter` to resolve for.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param platforms: A list of :class:`Platform`s to resolve for.
:param find_links: Additional paths to search for source packages during resolution.
:return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
to satisfy the requirements on that platform.
"""
python_setup = PythonSetup.global_instance()
python_repos = PythonRepos.global_instance()
platforms = platforms or python_setup.platforms
find_links = find_links or []
distributions = {}
fetchers = python_repos.get_fetchers()
fetchers.extend(Fetcher([path]) for path in find_links)
for platform in platforms:
requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir,
str(interpreter.identity))
distributions[platform] = resolve(
requirements=[req.requirement for req in requirements],
interpreter=interpreter,
fetchers=fetchers,
platform=None if platform == 'current' else platform,
context=python_repos.get_network_context(),
cache=requirements_cache_dir,
cache_ttl=python_setup.resolver_cache_ttl,
allow_prereleases=python_setup.resolver_allow_prereleases)
return distributions
def _resolve_multi(interpreter, requirements, platforms, find_links):
"""Multi-platform dependency resolution for PEX files.
Returns a list of distributions that must be included in order to satisfy a set of requirements.
That may involve distributions for multiple platforms.
:param interpreter: The :class:`PythonInterpreter` to resolve for.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param platforms: A list of :class:`Platform`s to resolve for.
:param find_links: Additional paths to search for source packages during resolution.
:return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
to satisfy the requirements on that platform.
"""
python_setup = PythonSetup.global_instance()
python_repos = PythonRepos.global_instance()
platforms = platforms or python_setup.platforms
find_links = find_links or []
distributions = {}
fetchers = python_repos.get_fetchers()
fetchers.extend(Fetcher([path]) for path in find_links)
for platform in platforms:
requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir,
str(interpreter.identity))
distributions[platform] = resolve(
requirements=[req.requirement for req in requirements],
interpreter=interpreter,
fetchers=fetchers,
platform=None if platform == 'current' else platform,
context=python_repos.get_network_context(),
cache=requirements_cache_dir,
cache_ttl=python_setup.resolver_cache_ttl,
allow_prereleases=python_setup.resolver_allow_prereleases)
return distributions
def dumped_chroot(self, targets):
# TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem
# declares it as a dependency. However if we don't then test_antlr() below fails on
# uninitialized options for that subsystem. Hopefully my pending (as of 9/2016) change
# to clean up how we initialize and create instances of subsystems in tests will make
# this problem go away.
self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem,
DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory])
python_repos = PythonRepos.global_instance()
ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance())
thrift_binary_factory = ThriftBinary.Factory.global_instance().create
interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
self.assertIsNotNone(interpreter)
with temporary_dir() as chroot:
pex_builder = PEXBuilder(path=chroot, interpreter=interpreter)
python_chroot = PythonChroot(python_setup=self.python_setup,
python_repos=python_repos,
ivy_bootstrapper=ivy_bootstrapper,
thrift_binary_factory=thrift_binary_factory,
interpreter=interpreter,
builder=pex_builder,
targets=targets,
platforms=['current'])
try:
python_chroot.dump()
yield pex_builder, python_chroot
finally:
python_chroot.delete()
def _interpreter_cache(self):
interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug)
# Cache setup's requirement fetching can hang if run concurrently by another pants proc.
self.context.acquire_lock()
try:
interpreter_cache.setup()
finally:
self.context.release_lock()
return interpreter_cache
def create_chroot(self, interpreter, builder, targets, platforms, extra_requirements):
return PythonChroot(python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
ivy_bootstrapper=self.ivy_bootstrapper,
thrift_binary_factory=self.thrift_binary_factory,
interpreter=interpreter,
builder=builder,
targets=targets,
platforms=platforms,
extra_requirements=extra_requirements,
log=self.context.log)
def _create_interpreter_path_file(self, interpreter_path_file, targets):
interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
safe_mkdir_for(interpreter_path_file)
with open(interpreter_path_file, 'w') as outfile:
outfile.write(b'{}\n'.format(interpreter.binary))
for dist, location in interpreter.extras.items():
dist_name, dist_version = dist
outfile.write(b'{}\t{}\t{}\n'.format(dist_name, dist_version, location))
def test_setup_using_eggs(self):
def link_egg(repo_root, requirement):
existing_dist_location = self._interpreter.get_location(requirement)
if existing_dist_location is not None:
existing_dist = Package.from_href(existing_dist_location)
requirement = '{}=={}'.format(existing_dist.name, existing_dist.raw_version)
distributions = resolve([requirement],
interpreter=self._interpreter,
precedence=(EggPackage, SourcePackage))
self.assertEqual(1, len(distributions))
dist_location = distributions[0].location
self.assertRegexpMatches(dist_location, r'\.egg$')
os.symlink(dist_location, os.path.join(repo_root, os.path.basename(dist_location)))
return Package.from_href(dist_location).raw_version
with temporary_dir() as root:
egg_dir = os.path.join(root, 'eggs')
os.makedirs(egg_dir)
setuptools_version = link_egg(egg_dir, 'setuptools')
wheel_version = link_egg(egg_dir, 'wheel')
interpreter_requirement = self._interpreter.identity.requirement
self.context(for_subsystems=[PythonSetup, PythonRepos], options={
PythonSetup.options_scope: {
'interpreter_cache_dir': None,
'pants_workdir': os.path.join(root, 'workdir'),
'constraints': [interpreter_requirement],
'setuptools_version': setuptools_version,
'wheel_version': wheel_version,
},
PythonRepos.options_scope: {
'indexes': [],
'repos': [egg_dir],
}
})
cache = PythonInterpreterCache(PythonSetup.global_instance(), PythonRepos.global_instance())
interpereters = cache.setup(paths=[os.path.dirname(self._interpreter.binary)],
filters=[str(interpreter_requirement)])
self.assertGreater(len(interpereters), 0)
def assert_egg_extra(interpreter, name, version):
location = interpreter.get_location('{}=={}'.format(name, version))
self.assertIsNotNone(location)
self.assertIsInstance(Package.from_href(location), EggPackage)
for interpreter in interpereters:
assert_egg_extra(interpreter, 'setuptools', setuptools_version)
assert_egg_extra(interpreter, 'wheel', wheel_version)
def create_chroot(self, interpreter, builder, targets, platforms,
extra_requirements):
return PythonChroot(python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
ivy_bootstrapper=self.ivy_bootstrapper,
thrift_binary_factory=self.thrift_binary_factory,
interpreter=interpreter,
builder=builder,
targets=targets,
platforms=platforms,
extra_requirements=extra_requirements,
log=self.context.log)
def _interpreter_cache(self):
interpreter_cache = PythonInterpreterCache(
PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug)
# Cache setup's requirement fetching can hang if run concurrently by another pants proc.
self.context.acquire_lock()
try:
interpreter_cache.setup()
finally:
self.context.release_lock()
return interpreter_cache
def _create_interpreter_path_file(self, interpreter_path_file, targets):
interpreter_cache = PythonInterpreterCache(
PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
safe_mkdir_for(interpreter_path_file)
with open(interpreter_path_file, 'w') as outfile:
outfile.write(b'{}\n'.format(interpreter.binary))
for dist, location in interpreter.extras.items():
dist_name, dist_version = dist
outfile.write(b'{}\t{}\t{}\n'.format(dist_name, dist_version,
location))
def create(cls, builder, log=None):
options = cls.global_instance().get_options()
setuptools_requirement = f'setuptools=={options.setuptools_version}'
log = log or logging.getLogger(__name__)
return PexBuilderWrapper(
builder=builder,
python_repos_subsystem=PythonRepos.global_instance(),
python_setup_subsystem=PythonSetup.global_instance(),
setuptools_requirement=PythonRequirement(
setuptools_requirement),
log=log)
def _gather_sources(self, target_roots):
context = self.context(target_roots=target_roots, for_subsystems=[PythonSetup, PythonRepos])
# We must get an interpreter via the cache, instead of using PythonInterpreter.get() directly,
# to ensure that the interpreter has setuptools and wheel support.
interpreter = PythonInterpreter.get()
interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=context.log.debug)
interpreters = interpreter_cache.setup(paths=[os.path.dirname(interpreter.binary)],
filters=[str(interpreter.identity.requirement)])
context.products.get_data(PythonInterpreter, lambda: interpreters[0])
task = self.create_task(context)
task.execute()
return context.products.get_data(GatherSources.PYTHON_SOURCES)
def _gather_sources(self, target_roots):
context = self.context(target_roots=target_roots, for_subsystems=[PythonSetup, PythonRepos])
# We must get an interpreter via the cache, instead of using PythonInterpreter.get() directly,
# to ensure that the interpreter has setuptools and wheel support.
interpreter = PythonInterpreter.get()
interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=context.log.debug)
interpreters = interpreter_cache.setup(paths=[os.path.dirname(interpreter.binary)],
filters=[str(interpreter.identity.requirement)])
context.products.get_data(PythonInterpreter, lambda: interpreters[0])
task = self.create_task(context)
task.execute()
return context.products.get_data(GatherSources.PythonSources)
def create(cls, builder, log=None, generate_ipex=False):
options = cls.global_instance().get_options()
setuptools_requirement = f"setuptools=={options.setuptools_version}"
pex_requirement = f"pex=={options.pex_version}"
log = log or logging.getLogger(__name__)
return PexBuilderWrapper(
builder=builder,
python_repos_subsystem=PythonRepos.global_instance(),
python_setup_subsystem=PythonSetup.global_instance(),
setuptools_requirement=PythonRequirement(
setuptools_requirement),
pex_requirement=PythonRequirement(pex_requirement),
log=log,
generate_ipex=generate_ipex,
)
def _options(self):
# NB: The PluginResolver runs very early in the pants startup sequence before the standard
# Subsystem facility is wired up. As a result PluginResolver is not itself a Subsystem with
# PythonRepos as a dependency. Instead it does the minimum possible work to hand-roll
# bootstrapping of the Subsystems it needs.
known_scope_infos = PythonRepos.known_scope_infos()
options = self._options_bootstrapper.get_full_options(
known_scope_infos)
# Ignore command line flags since we'd blow up on any we don't understand (most of them).
# If someone wants to bootstrap plugins in a one-off custom way they'll need to use env vars
# or a --pants-config-files pointing to a custom pants.ini snippet.
defaulted_only_options = options.drop_flag_values()
GlobalOptionsRegistrar.register_options_on_scope(
defaulted_only_options)
distinct_optionable_classes = sorted(
{si.optionable_cls
for si in known_scope_infos},
key=lambda o: o.options_scope)
for optionable_cls in distinct_optionable_classes:
optionable_cls.register_options_on_scope(defaulted_only_options)
return defaulted_only_options
def test_namespace_effective(self):
self.create_file(
"src/thrift/com/foo/one.thrift",
contents=dedent(
"""
namespace py foo.bar
struct One {}
"""
),
)
one = self.make_target(
spec="src/thrift/com/foo:one", target_type=PythonThriftLibrary, sources=["one.thrift"]
)
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)
self.create_file(
"src/thrift2/com/foo/two.thrift",
contents=dedent(
"""
namespace py foo.baz
struct Two {}
"""
),
)
two = self.make_target(
spec="src/thrift2/com/foo:two", target_type=PythonThriftLibrary, sources=["two.thrift"]
)
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
self.context(for_subsystems=[PythonInterpreterCache, PythonRepos])
interpreter_cache = PythonInterpreterCache.global_instance()
python_repos = PythonRepos.global_instance()
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
# We need setuptools to import namespace packages under python 2 (via pkg_resources), so we
# prime the PYTHONPATH with a known good version of setuptools.
# TODO(John Sirois): We really should be emitting setuptools in a
# `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`:
# https://github.com/pantsbuild/pants/issues/5975
pythonpath = [os.path.join(get_buildroot(), t.target_base) for t in targets]
for resolved_dist in resolve(
[f"thrift=={self.get_thrift_version(apache_thrift_gen)}", "setuptools==40.6.3"],
interpreter=interpreter,
indexes=python_repos.indexes,
find_links=python_repos.repos,
):
pythonpath.append(resolved_dist.distribution.location)
process = subprocess.Popen(
[
interpreter.binary,
"-c",
"from foo.bar.ttypes import One; from foo.baz.ttypes import Two",
],
env={"PYTHONPATH": os.pathsep.join(pythonpath)},
stderr=subprocess.PIPE,
)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
def __init__(self, *args, **kwargs):
super(PythonTask, self).__init__(*args, **kwargs)
self._interpreter_cache = PythonInterpreterCache(
PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug)
def _interpreter_cache(self):
return PythonInterpreterCache(
PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug
)
def _interpreter_cache(self):
return PythonInterpreterCache(PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug)
def test_setup_using_eggs(self):
def link_egg(repo_root, requirement):
existing_dist_location = self._interpreter.get_location(
requirement)
if existing_dist_location is not None:
existing_dist = Package.from_href(existing_dist_location)
requirement = '{}=={}'.format(existing_dist.name,
existing_dist.raw_version)
distributions = resolve([requirement],
interpreter=self._interpreter,
precedence=(EggPackage, SourcePackage))
self.assertEqual(1, len(distributions))
dist_location = distributions[0].location
self.assertRegexpMatches(dist_location, r'\.egg$')
os.symlink(
dist_location,
os.path.join(repo_root, os.path.basename(dist_location)))
return Package.from_href(dist_location).raw_version
with temporary_dir() as root:
egg_dir = os.path.join(root, 'eggs')
os.makedirs(egg_dir)
setuptools_version = link_egg(egg_dir, 'setuptools')
wheel_version = link_egg(egg_dir, 'wheel')
interpreter_requirement = self._interpreter.identity.requirement
self.context(for_subsystems=[PythonSetup, PythonRepos],
options={
PythonSetup.options_scope: {
'interpreter_cache_dir': None,
'pants_workdir':
os.path.join(root, 'workdir'),
'constraints': [interpreter_requirement],
'setuptools_version': setuptools_version,
'wheel_version': wheel_version,
},
PythonRepos.options_scope: {
'indexes': [],
'repos': [egg_dir],
}
})
cache = PythonInterpreterCache(PythonSetup.global_instance(),
PythonRepos.global_instance())
interpereters = cache.setup(
paths=[os.path.dirname(self._interpreter.binary)],
filters=[str(interpreter_requirement)])
self.assertGreater(len(interpereters), 0)
def assert_egg_extra(interpreter, name, version):
location = interpreter.get_location('{}=={}'.format(
name, version))
self.assertIsNotNone(location)
self.assertIsInstance(Package.from_href(location), EggPackage)
for interpreter in interpereters:
assert_egg_extra(interpreter, 'setuptools', setuptools_version)
assert_egg_extra(interpreter, 'wheel', wheel_version)