def execute(self):
interpreter = None
python_tgts = self.context.targets(
lambda tgt: isinstance(tgt, PythonTarget))
fs = PythonInterpreterFingerprintStrategy(task=self)
with self.invalidated(python_tgts,
fingerprint_strategy=fs) as invalidation_check:
# If there are no relevant targets, we still go through the motions of selecting
# an interpreter, to prevent downstream tasks from having to check for this special case.
if invalidation_check.all_vts:
target_set_id = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts).cache_key.hash
else:
target_set_id = 'no_targets'
interpreter_path_file = os.path.join(self.workdir, target_set_id,
'interpreter.path')
if not os.path.exists(interpreter_path_file):
interpreter_cache = PythonInterpreterCache(
PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug)
# We filter the interpreter cache itself (and not just the interpreters we pull from it)
# because setting up some python versions (e.g., 3<=python<3.3) crashes, and this gives us
# an escape hatch.
filters = self.get_options().constraints or [b'']
# Cache setup's requirement fetching can hang if run concurrently by another pants proc.
self.context.acquire_lock()
try:
interpreter_cache.setup(filters=filters)
finally:
self.context.release_lock()
interpreter = interpreter_cache.select_interpreter_for_targets(
python_tgts)
safe_mkdir_for(interpreter_path_file)
with open(interpreter_path_file, 'w') as outfile:
outfile.write(b'{}\t{}\n'.format(
interpreter.binary, str(interpreter.identity)))
for dist, location in interpreter.extras.items():
dist_name, dist_version = dist
outfile.write(b'{}\t{}\t{}\n'.format(
dist_name, dist_version, location))
if not interpreter:
with open(interpreter_path_file, 'r') as infile:
lines = infile.readlines()
binary, identity = lines[0].strip().split('\t')
extras = {}
for line in lines[1:]:
dist_name, dist_version, location = line.strip().split(
'\t')
extras[(dist_name, dist_version)] = location
interpreter = PythonInterpreter(binary,
PythonIdentity.from_path(identity),
extras)
self.context.products.get_data(PythonInterpreter, lambda: interpreter)
def dumped_chroot(self, targets):
# TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem
# declares it as a dependency. However if we don't then test_antlr() below fails on
# uninitialized options for that subsystem. Hopefully my pending (as of 9/2016) change
# to clean up how we initialize and create instances of subsystems in tests will make
# this problem go away.
self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem,
DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory])
python_repos = PythonRepos.global_instance()
ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance())
thrift_binary_factory = ThriftBinary.Factory.global_instance().create
interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
self.assertIsNotNone(interpreter)
with temporary_dir() as chroot:
pex_builder = PEXBuilder(path=chroot, interpreter=interpreter)
python_chroot = PythonChroot(python_setup=self.python_setup,
python_repos=python_repos,
ivy_bootstrapper=ivy_bootstrapper,
thrift_binary_factory=thrift_binary_factory,
interpreter=interpreter,
builder=pex_builder,
targets=targets,
platforms=['current'])
try:
python_chroot.dump()
yield pex_builder, python_chroot
finally:
python_chroot.delete()
def test_namespace_effective(self):
self.create_file('src/thrift/com/foo/one.thrift',
contents=dedent("""
namespace py foo.bar
struct One {}
"""))
one = self.make_target(spec='src/thrift/com/foo:one',
target_type=PythonThriftLibrary,
sources=['one.thrift'])
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(
one)
self.create_file('src/thrift2/com/foo/two.thrift',
contents=dedent("""
namespace py foo.baz
struct Two {}
"""))
two = self.make_target(spec='src/thrift2/com/foo:two',
target_type=PythonThriftLibrary,
sources=['two.thrift'])
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base,
synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
python_repos = global_subsystem_instance(PythonRepos)
python_setup = global_subsystem_instance(PythonSetup)
interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
pythonpath = [
os.path.join(get_buildroot(), t.target_base) for t in targets
]
for dist in resolve(
['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
interpreter=interpreter,
context=python_repos.get_network_context(),
fetchers=python_repos.get_fetchers()):
pythonpath.append(dist.location)
process = subprocess.Popen([
interpreter.binary, '-c',
'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'
],
env={
'PYTHONPATH':
os.pathsep.join(pythonpath)
},
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
def _create_interpreter_path_file(self, interpreter_path_file, targets):
interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
safe_mkdir_for(interpreter_path_file)
with open(interpreter_path_file, 'w') as outfile:
outfile.write(b'{}\n'.format(interpreter.binary))
for dist, location in interpreter.extras.items():
dist_name, dist_version = dist
outfile.write(b'{}\t{}\t{}\n'.format(dist_name, dist_version, location))
def _create_interpreter_path_file(self, interpreter_path_file, targets):
interpreter_cache = PythonInterpreterCache(
PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
safe_mkdir_for(interpreter_path_file)
with open(interpreter_path_file, 'w') as outfile:
outfile.write(b'{}\n'.format(interpreter.binary))
for dist, location in interpreter.extras.items():
dist_name, dist_version = dist
outfile.write(b'{}\t{}\t{}\n'.format(dist_name, dist_version,
location))
def test_namespace_effective(self):
self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
namespace py foo.bar
struct One {}
"""))
one = self.make_target(spec='src/thrift/com/foo:one',
target_type=PythonThriftLibrary,
sources=['one.thrift'])
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)
self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent("""
namespace py foo.baz
struct Two {}
"""))
two = self.make_target(spec='src/thrift2/com/foo:two',
target_type=PythonThriftLibrary,
sources=['two.thrift'])
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
python_repos = global_subsystem_instance(PythonRepos)
python_setup = global_subsystem_instance(PythonSetup)
interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
# We need setuptools to import namespace packages (via pkg_resources), so we prime the
# PYTHONPATH with interpreter extras, which Pants always populates with setuptools and wheel.
# TODO(John Sirois): We really should be emitting setuptools in a
# `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`:
# https://github.com/pantsbuild/pants/issues/5975
pythonpath = interpreter.extras.values()
pythonpath.extend(os.path.join(get_buildroot(), t.target_base) for t in targets)
for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
interpreter=interpreter,
context=python_repos.get_network_context(),
fetchers=python_repos.get_fetchers()):
pythonpath.append(dist.location)
process = subprocess.Popen([interpreter.binary,
'-c',
'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'],
env={'PYTHONPATH': os.pathsep.join(pythonpath)},
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
def test_namespace_effective(self):
self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
namespace py foo.bar
struct One {}
"""))
one = self.make_target(spec='src/thrift/com/foo:one',
target_type=PythonThriftLibrary,
sources=['one.thrift'])
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)
self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent("""
namespace py foo.baz
struct Two {}
"""))
two = self.make_target(spec='src/thrift2/com/foo:two',
target_type=PythonThriftLibrary,
sources=['two.thrift'])
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
python_repos = global_subsystem_instance(PythonRepos)
python_setup = global_subsystem_instance(PythonSetup)
interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
# We need setuptools to import namespace packages (via pkg_resources), so we prime the
# PYTHONPATH with interpreter extras, which Pants always populates with setuptools and wheel.
# TODO(John Sirois): We really should be emitting setuptools in a
# `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`:
# https://github.com/pantsbuild/pants/issues/5975
pythonpath = interpreter.extras.values()
pythonpath.extend(os.path.join(get_buildroot(), t.target_base) for t in targets)
for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
interpreter=interpreter,
context=python_repos.get_network_context(),
fetchers=python_repos.get_fetchers()):
pythonpath.append(dist.location)
process = subprocess.Popen([interpreter.binary,
'-c',
'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'],
env={'PYTHONPATH': os.pathsep.join(pythonpath)},
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
def execute(self):
interpreter = None
python_tgts = self.context.targets(lambda tgt: isinstance(tgt, PythonTarget))
fs = PythonInterpreterFingerprintStrategy(task=self)
with self.invalidated(python_tgts, fingerprint_strategy=fs) as invalidation_check:
# If there are no relevant targets, we still go through the motions of selecting
# an interpreter, to prevent downstream tasks from having to check for this special case.
if invalidation_check.all_vts:
target_set_id = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts).cache_key.hash
else:
target_set_id = 'no_targets'
interpreter_path_file = os.path.join(self.workdir, target_set_id, 'interpreter.path')
if not os.path.exists(interpreter_path_file):
interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug)
# We filter the interpreter cache itself (and not just the interpreters we pull from it)
# because setting up some python versions (e.g., 3<=python<3.3) crashes, and this gives us
# an escape hatch.
filters = self.get_options().constraints or [b'']
# Cache setup's requirement fetching can hang if run concurrently by another pants proc.
self.context.acquire_lock()
try:
interpreter_cache.setup(filters=filters)
finally:
self.context.release_lock()
interpreter = interpreter_cache.select_interpreter_for_targets(python_tgts)
safe_mkdir_for(interpreter_path_file)
with open(interpreter_path_file, 'w') as outfile:
outfile.write(b'{}\t{}\n'.format(interpreter.binary, str(interpreter.identity)))
for dist, location in interpreter.extras.items():
dist_name, dist_version = dist
outfile.write(b'{}\t{}\t{}\n'.format(dist_name, dist_version, location))
if not interpreter:
with open(interpreter_path_file, 'r') as infile:
lines = infile.readlines()
binary, identity = lines[0].strip().split('\t')
extras = {}
for line in lines[1:]:
dist_name, dist_version, location = line.strip().split('\t')
extras[(dist_name, dist_version)] = location
interpreter = PythonInterpreter(binary, PythonIdentity.from_path(identity), extras)
self.context.products.get_data(PythonInterpreter, lambda: interpreter)
def test_namespace_effective(self):
self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
namespace py foo.bar
struct One {}
"""))
one = self.make_target(spec='src/thrift/com/foo:one',
target_type=PythonThriftLibrary,
sources=['one.thrift'])
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)
self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent("""
namespace py foo.baz
struct Two {}
"""))
two = self.make_target(spec='src/thrift2/com/foo:two',
target_type=PythonThriftLibrary,
sources=['two.thrift'])
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
python_repos = global_subsystem_instance(PythonRepos)
python_setup = global_subsystem_instance(PythonSetup)
interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
pythonpath = [os.path.join(get_buildroot(), t.target_base) for t in targets]
for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
interpreter=interpreter,
context=python_repos.get_network_context(),
fetchers=python_repos.get_fetchers()):
pythonpath.append(dist.location)
process = subprocess.Popen([interpreter.binary,
'-c',
'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'],
env={'PYTHONPATH': os.pathsep.join(pythonpath)},
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
class PythonTask(Task):
# If needed, we set this as the executable entry point of any chroots we create.
CHROOT_EXECUTABLE_NAME = '__pants_executable__'
@classmethod
def implementation_version(cls):
return super(PythonTask,
cls).implementation_version() + [('PythonTask', 1)]
@classmethod
def subsystem_dependencies(cls):
return (super(PythonTask, cls).subsystem_dependencies() +
(IvySubsystem, PythonSetup, PythonRepos,
ThriftBinary.Factory.scoped(cls)))
def __init__(self, *args, **kwargs):
super(PythonTask, self).__init__(*args, **kwargs)
self._interpreter_cache = PythonInterpreterCache(
PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=self.context.log.debug)
def select_interpreter_for_targets(self, targets):
"""Pick an interpreter compatible with all the specified targets."""
return self._interpreter_cache.select_interpreter_for_targets(targets)
@property
def chroot_cache_dir(self):
return PythonSetup.global_instance().chroot_cache_dir
@property
def ivy_bootstrapper(self):
return Bootstrapper(ivy_subsystem=IvySubsystem.global_instance())
@property
def thrift_binary_factory(self):
return ThriftBinary.Factory.scoped_instance(self).create
def create_chroot(self, interpreter, builder, targets, platforms,
extra_requirements):
return PythonChroot(python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
ivy_bootstrapper=self.ivy_bootstrapper,
thrift_binary_factory=self.thrift_binary_factory,
interpreter=interpreter,
builder=builder,
targets=targets,
platforms=platforms,
extra_requirements=extra_requirements,
log=self.context.log)
def cached_chroot(self,
interpreter,
pex_info,
targets,
platforms=None,
extra_requirements=None,
executable_file_content=None):
"""Returns a cached PythonChroot created with the specified args.
The returned chroot will be cached for future use.
:rtype: pants.backend.python.python_chroot.PythonChroot
TODO: Garbage-collect old chroots, so they don't pile up?
TODO: Ideally chroots would just be products produced by some other task. But that's
a bit too complicated to implement right now, as we'd need a way to request
chroots for a variety of sets of targets.
"""
# This PexInfo contains any customizations specified by the caller.
# The process of building a pex modifies it further.
pex_info = pex_info or PexInfo.default()
path = self._chroot_path(interpreter, pex_info, targets, platforms,
extra_requirements, executable_file_content)
if not os.path.exists(path):
path_tmp = path + '.tmp'
self._build_chroot(path_tmp, interpreter, pex_info, targets,
platforms, extra_requirements,
executable_file_content)
shutil.move(path_tmp, path)
# We must read the PexInfo that was frozen into the pex, so we get the modifications
# created when that pex was built.
pex_info = PexInfo.from_pex(path)
# Now create a PythonChroot wrapper without dumping it.
builder = PEXBuilder(path=path,
interpreter=interpreter,
pex_info=pex_info,
copy=True)
return self.create_chroot(interpreter=interpreter,
builder=builder,
targets=targets,
platforms=platforms,
extra_requirements=extra_requirements)
@contextmanager
def temporary_chroot(self,
interpreter,
pex_info,
targets,
platforms,
extra_requirements=None,
executable_file_content=None):
path = tempfile.mkdtemp(
) # Not a contextmanager: chroot.delete() will clean this up anyway.
pex_info = pex_info or PexInfo.default()
chroot = self._build_chroot(path, interpreter, pex_info, targets,
platforms, extra_requirements,
executable_file_content)
yield chroot
chroot.delete()
def _build_chroot(self,
path,
interpreter,
pex_info,
targets,
platforms,
extra_requirements=None,
executable_file_content=None):
"""Create a PythonChroot with the specified args."""
builder = PEXBuilder(path=path,
interpreter=interpreter,
pex_info=pex_info,
copy=True)
with self.context.new_workunit('chroot'):
chroot = self.create_chroot(interpreter=interpreter,
builder=builder,
targets=targets,
platforms=platforms,
extra_requirements=extra_requirements)
chroot.dump()
if executable_file_content is not None:
with open(
os.path.join(
path, '{}.py'.format(self.CHROOT_EXECUTABLE_NAME)),
'w') as outfile:
outfile.write(executable_file_content)
# Override any user-specified entry point, under the assumption that the
# executable_file_content does what the user intends (including, probably, calling that
# underlying entry point).
pex_info.entry_point = self.CHROOT_EXECUTABLE_NAME
builder.freeze()
return chroot
def _chroot_path(self, interpreter, pex_info, targets, platforms,
extra_requirements, executable_file_content):
"""Pick a unique, well-known directory name for the chroot with the specified parameters.
TODO: How many of these do we expect to have? Currently they are all under a single
directory, and some filesystems (E.g., HFS+) don't handle directories with thousands of
entries well. GC'ing old chroots may be enough of a solution, assuming this is even a problem.
"""
fingerprint_components = [str(interpreter.identity)]
if pex_info:
# TODO(John Sirois): When https://rbcommons.com/s/twitter/r/2517/ lands, leverage the dump
# **kwargs to sort keys or else find some other better way to get a stable fingerprint of
# PexInfo.
fingerprint_components.append(
json.dumps(json.loads(pex_info.dump()), sort_keys=True))
fingerprint_components.extend(
sorted(t.transitive_invalidation_hash() for t in set(targets)))
if platforms:
fingerprint_components.extend(sorted(set(platforms)))
if extra_requirements:
# TODO(John Sirois): The extras should be uniqified before fingerprinting, but
# PythonRequirement arguably does not have a proper __eq__. For now we lean on the cache_key
# of unique PythonRequirement being unique - which is probably good enough (the cache key is
# narrower than the full scope of PythonRequirement attributes at present, thus the hedge).
fingerprint_components.extend(
sorted(set(r.cache_key() for r in extra_requirements)))
if executable_file_content is not None:
fingerprint_components.append(executable_file_content)
fingerprint = hash_utils.hash_all(fingerprint_components)
return os.path.join(self.chroot_cache_dir, fingerprint)
