def __init__(self,
target,
run_tracker,
interpreter=None,
conn_timeout=None):
self.target = target
self.interpreter = interpreter or PythonInterpreter.get()
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
run_info = run_tracker.run_info
build_properties = {}
build_properties.update(
run_info.add_basic_info(run_id=None, timestamp=time.time()))
build_properties.update(run_info.add_scm_info())
pexinfo = target.pexinfo.copy()
pexinfo.build_properties = build_properties
builder = PEXBuilder(distpath,
pex_info=pexinfo,
interpreter=self.interpreter)
self.chroot = PythonChroot(targets=[target],
builder=builder,
platforms=target.platforms,
interpreter=self.interpreter,
conn_timeout=conn_timeout)
def temporary_chroot(self, interpreter=None, pex_info=None, targets=None,
extra_requirements=None, platforms=None, pre_freeze=None):
"""Yields a temporary PythonChroot created with the specified args.
pre_freeze is an optional function run on the chroot just before freezing its builder,
to allow for any extra modification.
"""
path = tempfile.mkdtemp()
builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info)
with self.context.new_workunit('chroot'):
chroot = PythonChroot(
context=self.context,
python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
targets=targets,
extra_requirements=extra_requirements,
builder=builder,
platforms=platforms,
interpreter=interpreter)
chroot.dump()
if pre_freeze:
pre_freeze(chroot)
builder.freeze()
yield chroot
chroot.delete()
def execute(self):
binary = self.require_single_root_target()
if isinstance(binary, PythonBinary):
# We can't throw if binary isn't a PythonBinary, because perhaps we were called on a
# jvm_binary, in which case we have to no-op and let jvm_run do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(self.context.targets())
with self.temporary_pex_builder(interpreter=interpreter, pex_info=binary.pexinfo) as builder:
chroot = PythonChroot(
targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
self.context.release_lock()
with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]):
args = self.get_options().args + self.get_passthru_args()
po = pex.run(blocking=False, args=args)
try:
result = po.wait()
if result != 0:
raise TaskError('python {args} ... exited non-zero ({code})' %
dict(args=args, code=result),
exit_code=result)
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def _test_runner(self, targets, workunit):
interpreter = self.select_interpreter_for_targets(targets)
builder = PEXBuilder(interpreter=interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
context=self.context,
python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=interpreter)
try:
builder = chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
with self._maybe_shard() as shard_args:
with self._maybe_emit_junit_xml(targets) as junit_args:
with self._maybe_emit_coverage_data(targets,
builder.path(),
pex,
workunit) as coverage_args:
yield pex, shard_args + junit_args + coverage_args
finally:
chroot.delete()
def execute(self):
binary = self.require_single_root_target()
if isinstance(binary, PythonBinary):
# We can't throw if binary isn't a PythonBinary, because perhaps we were called on a
# jvm_binary, in which case we have to no-op and let jvm_run do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(
self.context.targets())
with self.temporary_pex_builder(
interpreter=interpreter,
pex_info=binary.pexinfo) as builder:
chroot = PythonChroot(targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
self.context.lock.release()
with self.context.new_workunit(name='run',
labels=[WorkUnit.RUN]):
po = pex.run(blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def execute(self):
binary = self.require_single_root_target()
if isinstance(binary, PythonBinary):
# We can't throw if binary isn't a PythonBinary, because perhaps we were called on a
# jvm_binary, in which case we have to no-op and let jvm_run do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(
self.context.targets())
with self.temporary_pex_builder(
interpreter=interpreter,
pex_info=binary.pexinfo) as builder:
chroot = PythonChroot(targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter)
chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
self.context.release_lock()
with self.context.new_workunit(name='run',
labels=[WorkUnit.RUN]):
args = self.get_options().args + self.get_passthru_args()
po = pex.run(blocking=False, args=args)
try:
result = po.wait()
if result != 0:
raise TaskError(
'python {args} ... exited non-zero ({code})' %
dict(args=args, code=result),
exit_code=result)
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def _build_chroot(self,
path,
interpreter,
pex_info,
targets,
platforms,
extra_requirements=None,
executable_file_content=None):
"""Create a PythonChroot with the specified args."""
builder = PEXBuilder(path=path,
interpreter=interpreter,
pex_info=pex_info)
with self.context.new_workunit('chroot'):
chroot = PythonChroot(context=self.context,
python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
interpreter=interpreter,
builder=builder,
targets=targets,
platforms=platforms,
extra_requirements=extra_requirements)
chroot.dump()
if executable_file_content is not None:
with open(
os.path.join(
path, '{}.py'.format(self.CHROOT_EXECUTABLE_NAME)),
'w') as outfile:
outfile.write(executable_file_content)
# Override any user-specified entry point, under the assumption that the
# executable_file_content does what the user intends (including, probably, calling that
# underlying entry point).
pex_info.entry_point = self.CHROOT_EXECUTABLE_NAME
builder.freeze()
return chroot
def dumped_chroot(self, targets):
python_repos = create_subsystem(PythonRepos)
with subsystem_instance(IvySubsystem) as ivy_subsystem:
ivy_bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem)
with subsystem_instance(ThriftBinary.Factory) as thrift_binary_factory:
interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos)
interpreter_cache.setup()
interpreters = list(interpreter_cache.matched_interpreters([
self.python_setup.interpreter_requirement]))
self.assertGreater(len(interpreters), 0)
interpreter = interpreters[0]
with temporary_dir() as chroot:
pex_builder = PEXBuilder(path=chroot, interpreter=interpreter)
python_chroot = PythonChroot(python_setup=self.python_setup,
python_repos=python_repos,
ivy_bootstrapper=ivy_bootstrapper,
thrift_binary_factory=thrift_binary_factory.create,
interpreter=interpreter,
builder=pex_builder,
targets=targets,
platforms=['current'])
try:
python_chroot.dump()
yield pex_builder, python_chroot
finally:
python_chroot.delete()
def create_binary(self, binary):
if binary.compatibility:
interpreter = self.select_interpreter(binary.compatibility)
else:
interpreter = self.interpreter
distpath = tempfile.mktemp(dir=self._distdir, prefix=binary.name)
run_info = self.context.run_tracker.run_info
build_properties = {}
build_properties.update(
run_info.add_basic_info(run_id=None, timestamp=time.time()))
build_properties.update(run_info.add_scm_info())
pexinfo = binary.pexinfo.copy()
pexinfo.build_properties = build_properties
builder = PEXBuilder(distpath,
pex_info=pexinfo,
interpreter=interpreter)
chroot = PythonChroot(targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
pex_path = os.path.join(self._distdir, '%s.pex' % binary.name)
builder = chroot.dump()
try:
builder.build(pex_path)
finally:
builder.chroot().delete()
def dumped_chroot(self, targets):
# TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem
# declares it as a dependency. However if we don't then test_antlr() below fails on
# uninitialized options for that subsystem. Hopefully my pending (as of 9/2016) change
# to clean up how we initialize and create instances of subsystems in tests will make
# this problem go away.
self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem,
DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory])
python_repos = PythonRepos.global_instance()
ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance())
thrift_binary_factory = ThriftBinary.Factory.global_instance().create
interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos)
interpreter_cache.setup()
interpreters = list(interpreter_cache.matched_interpreters(
self.python_setup.interpreter_constraints))
self.assertGreater(len(interpreters), 0)
interpreter = interpreters[0]
with temporary_dir() as chroot:
pex_builder = PEXBuilder(path=chroot, interpreter=interpreter)
python_chroot = PythonChroot(python_setup=self.python_setup,
python_repos=python_repos,
ivy_bootstrapper=ivy_bootstrapper,
thrift_binary_factory=thrift_binary_factory,
interpreter=interpreter,
builder=pex_builder,
targets=targets,
platforms=['current'])
try:
python_chroot.dump()
yield pex_builder, python_chroot
finally:
python_chroot.delete()
def execute(self):
binary = self.require_single_root_target()
if isinstance(binary, PythonBinary):
# We can't throw if binary isn't a PythonBinary, because perhaps we were called on a
# jvm_binary, in which case we have to no-op and let jvm_run do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(self.context.targets())
with self.temporary_pex_builder(interpreter=interpreter, pex_info=binary.pexinfo) as builder:
chroot = PythonChroot(
targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
self.context.lock.release()
with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]):
po = pex.run(blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def execute(self):
if self.old_options.pex and self.old_options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.old_options.entry_point and self.old_options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.old_options.verbose:
print('Build operating on targets: %s' % ' '.join(str(target) for target in self.targets))
builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
pex_info=self.binary.pexinfo if self.binary else None)
if self.old_options.entry_point:
builder.set_entry_point(self.old_options.entry_point)
if self.old_options.ipython:
if not self.config.has_section('python-ipython'):
self.error('No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
if builder.info.entry_point is None:
self.error('Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython', 'requirements', default=[])
for requirement in requirements:
self.extra_requirements.append(PythonRequirement(requirement))
executor = PythonChroot(
targets=self.targets,
extra_requirements=self.extra_requirements,
builder=builder,
platforms=self.binary.platforms if self.binary else None,
interpreter=self.interpreter,
conn_timeout=self.old_options.conn_timeout)
executor.dump()
if self.old_options.pex:
pex_name = self.binary.name if self.binary else Target.maybe_readable_identify(self.targets)
pex_path = os.path.join(self.root_dir, 'dist', '%s.pex' % pex_name)
builder.build(pex_path)
print('Wrote %s' % pex_path)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def __init__(self, target, run_tracker, interpreter=None):
self.target = target
self.interpreter = interpreter or PythonInterpreter.get()
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.from_cache()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
run_info = run_tracker.run_info
build_properties = {}
build_properties.update(run_info.add_basic_info(run_id=None, timestamp=time.time()))
build_properties.update(run_info.add_scm_info())
pexinfo = target.pexinfo.copy()
pexinfo.build_properties = build_properties
builder = PEXBuilder(distpath, pex_info=pexinfo, interpreter=self.interpreter)
self.chroot = PythonChroot(
targets=[target],
builder=builder,
platforms=target.platforms,
interpreter=self.interpreter)
def execute(self):
(accept_predicate,
reject_predicate) = Target.lang_discriminator('python')
targets = self.require_homogeneous_targets(accept_predicate,
reject_predicate)
if targets:
# We can't throw if the target isn't a python target, because perhaps we were called on a
# JVM target, in which case we have to no-op and let scala repl do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(targets)
extra_requirements = []
if self.context.options.python_repl_ipython:
entry_point = self.context.config.get(
'python-ipython',
'entry_point',
default='IPython:start_ipython')
ipython_requirements = self.context.config.getlist(
'python-ipython',
'requirements',
default=['ipython==1.0.0'])
for req in ipython_requirements:
extra_requirements.append(PythonRequirement(req))
else:
entry_point = 'code:interact'
with self.temporary_pex_builder(
interpreter=interpreter) as builder:
builder.set_entry_point(entry_point)
chroot = PythonChroot(targets=targets,
extra_requirements=extra_requirements,
builder=builder,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
self.context.lock.release()
with stty_utils.preserve_stty_settings():
with self.context.new_workunit(name='run',
labels=[WorkUnit.RUN]):
po = pex.run(blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
pass
def _run_python_tests(self, targets, stdout, stderr):
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current', ),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = []
test_args.extend(PythonTestBuilder.generate_junit_args(targets))
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(targets)
test_args.extend(args)
sources = list(
itertools.chain(
*[t.sources_relative_to_buildroot() for t in targets]))
pex = PEX(builder.path(), interpreter=self.interpreter)
rc = pex.run(args=test_args + sources,
blocking=True,
setsid=True,
stdout=stdout,
stderr=stderr)
# TODO(wickman): If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestResult.rc(rc)
except Exception:
import traceback
print('Failed to run test!', file=stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
return rv
def _test_runner(self, targets, stdout, stderr):
builder = PEXBuilder(interpreter=self._interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current', ),
interpreter=self._interpreter)
try:
builder = chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=self._interpreter)
with self._maybe_emit_junit_xml(targets) as junit_args:
with self._maybe_emit_coverage_data(targets, builder.path(),
pex, stdout,
stderr) as coverage_args:
yield pex, junit_args + coverage_args
finally:
chroot.delete()
def create_chroot(self, interpreter, builder, targets, platforms,
extra_requirements):
return PythonChroot(python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
ivy_bootstrapper=self.ivy_bootstrapper,
thrift_binary_factory=self.thrift_binary_factory,
interpreter=interpreter,
builder=builder,
targets=targets,
platforms=platforms,
extra_requirements=extra_requirements)
def create_binary(self, binary):
interpreter = self.select_interpreter_for_targets(binary.closure())
run_info = self.context.run_tracker.run_info
build_properties = {}
build_properties.update(run_info.add_basic_info(run_id=None, timestamp=time.time()))
build_properties.update(run_info.add_scm_info())
pexinfo = binary.pexinfo.copy()
pexinfo.build_properties = build_properties
with self.temporary_pex_builder(pex_info=pexinfo, interpreter=interpreter) as builder:
chroot = PythonChroot(
targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter)
pex_path = os.path.join(self._distdir, '%s.pex' % binary.name)
chroot.dump()
builder.build(pex_path)
def execute(self):
(accept_predicate, reject_predicate) = Target.lang_discriminator('python')
targets = self.require_homogeneous_targets(accept_predicate, reject_predicate)
if targets:
# We can't throw if the target isn't a python target, because perhaps we were called on a
# JVM target, in which case we have to no-op and let scala repl do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(targets)
extra_requirements = []
if self.get_options().ipython:
entry_point = self.context.config.get('python-ipython', 'entry_point',
default='IPython:start_ipython')
ipython_requirements = self.context.config.getlist('python-ipython', 'requirements',
default=['ipython==1.0.0'])
for req in ipython_requirements:
extra_requirements.append(PythonRequirement(req))
else:
entry_point = 'code:interact'
with self.temporary_pex_builder(interpreter=interpreter) as builder:
builder.set_entry_point(entry_point)
chroot = PythonChroot(
targets=targets,
extra_requirements=extra_requirements,
builder=builder,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
self.context.lock.release()
with stty_utils.preserve_stty_settings():
with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]):
po = pex.run(blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
pass
def dumped_chroot(self, targets):
python_repos = create_subsystem(PythonRepos)
with subsystem_instance(IvySubsystem) as ivy_subsystem:
ivy_bootstrapper = Bootstrapper(ivy_subsystem=ivy_subsystem)
with subsystem_instance(
ThriftBinary.Factory) as thrift_binary_factory:
interpreter_cache = PythonInterpreterCache(
self.python_setup, python_repos)
interpreter_cache.setup()
interpreters = list(
interpreter_cache.matches(
[self.python_setup.interpreter_requirement]))
self.assertGreater(len(interpreters), 0)
interpreter = interpreters[0]
with temporary_dir() as chroot:
pex_builder = PEXBuilder(path=chroot,
interpreter=interpreter)
python_chroot = PythonChroot(
python_setup=self.python_setup,
python_repos=python_repos,
ivy_bootstrapper=ivy_bootstrapper,
thrift_binary_factory=thrift_binary_factory.create,
interpreter=interpreter,
builder=pex_builder,
targets=targets,
platforms=['current'])
try:
python_chroot.dump()
yield pex_builder, python_chroot
finally:
python_chroot.delete()
def dumped_chroot(self, targets):
# TODO(benjy): We shouldn't need to mention DistributionLocator here, as IvySubsystem
# declares it as a dependency. However if we don't then test_antlr() below fails on
# uninitialized options for that subsystem. Hopefully my pending (as of 9/2016) change
# to clean up how we initialize and create instances of subsystems in tests will make
# this problem go away.
self.context(for_subsystems=[PythonRepos, PythonSetup, IvySubsystem,
DistributionLocator, ThriftBinary.Factory, BinaryUtil.Factory])
python_repos = PythonRepos.global_instance()
ivy_bootstrapper = Bootstrapper(ivy_subsystem=IvySubsystem.global_instance())
thrift_binary_factory = ThriftBinary.Factory.global_instance().create
interpreter_cache = PythonInterpreterCache(self.python_setup, python_repos)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
self.assertIsNotNone(interpreter)
with temporary_dir() as chroot:
pex_builder = PEXBuilder(path=chroot, interpreter=interpreter)
python_chroot = PythonChroot(python_setup=self.python_setup,
python_repos=python_repos,
ivy_bootstrapper=ivy_bootstrapper,
thrift_binary_factory=thrift_binary_factory,
interpreter=interpreter,
builder=pex_builder,
targets=targets,
platforms=['current'])
try:
python_chroot.dump()
yield pex_builder, python_chroot
finally:
python_chroot.delete()
def _run_python_tests(self, targets, stdout, stderr):
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = []
test_args.extend(PythonTestBuilder.generate_junit_args(targets))
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(targets)
test_args.extend(args)
sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets]))
pex = PEX(builder.path(), interpreter=self.interpreter)
rc = pex.run(args=test_args + sources, blocking=True, setsid=True,
stdout=stdout, stderr=stderr)
# TODO(wickman): If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestResult.rc(rc)
except Exception:
import traceback
print('Failed to run test!', file=stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
return rv
def _test_runner(self, targets, stdout, stderr):
builder = PEXBuilder(interpreter=self._interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=self._interpreter)
try:
builder = chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=self._interpreter)
with self._maybe_emit_junit_xml(targets) as junit_args:
with self._maybe_emit_coverage_data(targets,
builder.path(),
pex,
stdout,
stderr) as coverage_args:
yield pex, junit_args + coverage_args
finally:
chroot.delete()
def create_binary(self, binary):
interpreter = self.select_interpreter_for_targets(binary.closure())
run_info = self.context.run_tracker.run_info
build_properties = {}
build_properties.update(run_info.add_basic_info(run_id=None, timestamp=time.time()))
build_properties.update(run_info.add_scm_info())
pexinfo = binary.pexinfo.copy()
pexinfo.build_properties = build_properties
with self.temporary_pex_builder(pex_info=pexinfo, interpreter=interpreter) as builder:
chroot = PythonChroot(
context=self.context,
targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter)
pex_path = os.path.join(self._distdir, '%s.pex' % binary.name)
chroot.dump()
builder.build(pex_path)
def cached_chroot(self,
interpreter,
pex_info,
targets,
platforms,
extra_requirements=None,
executable_file_content=None):
"""Returns a cached PythonChroot created with the specified args.
The returned chroot will be cached for future use.
TODO: Garbage-collect old chroots, so they don't pile up?
TODO: Ideally chroots would just be products produced by some other task. But that's
a bit too complicated to implement right now, as we'd need a way to request
chroots for a variety of sets of targets.
"""
# This PexInfo contains any customizations specified by the caller.
# The process of building a pex modifies it further.
pex_info = pex_info or PexInfo.default()
path = self._chroot_path(PythonSetup.global_instance(), interpreter,
pex_info, targets, platforms,
extra_requirements, executable_file_content)
if not os.path.exists(path):
path_tmp = path + '.tmp'
self._build_chroot(path_tmp, interpreter, pex_info, targets,
platforms, extra_requirements,
executable_file_content)
shutil.move(path_tmp, path)
# We must read the PexInfo that was frozen into the pex, so we get the modifications
# created when that pex was built.
pex_info = PexInfo.from_pex(path)
# Now create a PythonChroot wrapper without dumping it.
builder = PEXBuilder(path=path,
interpreter=interpreter,
pex_info=pex_info)
chroot = PythonChroot(context=self.context,
python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
interpreter=interpreter,
builder=builder,
targets=targets,
platforms=platforms,
extra_requirements=extra_requirements)
# TODO: Doesn't really need to be a contextmanager, but it's convenient to make it so
# while transitioning calls to temporary_chroot to calls to cached_chroot.
# We can revisit after that transition is complete.
yield chroot
def _build_chroot(self, path, interpreter, pex_info, targets, platforms,
extra_requirements=None, executable_file_content=None):
"""Create a PythonChroot with the specified args."""
builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info)
with self.context.new_workunit('chroot'):
chroot = PythonChroot(
context=self.context,
python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
interpreter=interpreter,
builder=builder,
targets=targets,
platforms=platforms,
extra_requirements=extra_requirements)
chroot.dump()
if executable_file_content is not None:
with open(os.path.join(path, '{}.py'.format(self.CHROOT_EXECUTABLE_NAME)), 'w') as outfile:
outfile.write(executable_file_content)
# Override any user-specified entry point, under the assumption that the
# executable_file_content does what the user intends (including, probably, calling that
# underlying entry point).
pex_info.entry_point = self.CHROOT_EXECUTABLE_NAME
builder.freeze()
return chroot
class PythonBinaryBuilder(object):
class NotABinaryTargetException(Exception):
pass
def __init__(self,
target,
run_tracker,
interpreter=None,
conn_timeout=None):
self.target = target
self.interpreter = interpreter or PythonInterpreter.get()
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
run_info = run_tracker.run_info
build_properties = {}
build_properties.update(
run_info.add_basic_info(run_id=None, timestamp=time.time()))
build_properties.update(run_info.add_scm_info())
pexinfo = target.pexinfo.copy()
pexinfo.build_properties = build_properties
builder = PEXBuilder(distpath,
pex_info=pexinfo,
interpreter=self.interpreter)
self.chroot = PythonChroot(targets=[target],
builder=builder,
platforms=target.platforms,
interpreter=self.interpreter,
conn_timeout=conn_timeout)
def run(self):
print('Building PythonBinary %s:' % self.target)
env = self.chroot.dump()
filename = os.path.join(self.distdir, '%s.pex' % self.target.name)
env.build(filename)
print('Wrote %s' % filename)
return 0
class PythonBinaryBuilder(object):
class NotABinaryTargetException(Exception):
pass
def __init__(self, context, target, run_tracker, interpreter=None):
self.context = context
self.target = target
self.interpreter = interpreter or PythonInterpreter.get()
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.from_cache()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
run_info = run_tracker.run_info
build_properties = {}
build_properties.update(run_info.add_basic_info(run_id=None, timestamp=time.time()))
build_properties.update(run_info.add_scm_info())
pexinfo = target.pexinfo.copy()
pexinfo.build_properties = build_properties
builder = PEXBuilder(distpath, pex_info=pexinfo, interpreter=self.interpreter)
self.chroot = PythonChroot(
context=self.context,
targets=[target],
builder=builder,
platforms=target.platforms,
interpreter=self.interpreter)
def run(self):
print('Building PythonBinary %s:' % self.target)
env = self.chroot.dump()
filename = os.path.join(self.distdir, '%s.pex' % self.target.name)
env.build(filename)
print('Wrote %s' % filename)
return 0
def test_get_current_platform():
expected_platforms = [Platform.current(), "linux-x86_64"]
assert set(expected_platforms) == set(PythonChroot.get_platforms(["current", "linux-x86_64"]))
def execute(self):
if self.options.pex and self.options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.options.entry_point and self.options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.options.verbose:
print('Build operating on targets: %s' %
' '.join(str(target) for target in self.targets))
builder = PEXBuilder(
tempfile.mkdtemp(),
interpreter=self.interpreter,
pex_info=self.binary.pexinfo if self.binary else None)
if self.options.entry_point:
builder.set_entry_point(self.options.entry_point)
if self.options.ipython:
if not self.config.has_section('python-ipython'):
self.error(
'No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython',
'entry_point')
if builder.info.entry_point is None:
self.error(
'Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython',
'requirements',
default=[])
for requirement in requirements:
self.extra_requirements.append(PythonRequirement(requirement))
executor = PythonChroot(
targets=self.targets,
extra_requirements=self.extra_requirements,
builder=builder,
platforms=self.binary.platforms if self.binary else None,
interpreter=self.interpreter,
conn_timeout=self.options.conn_timeout)
executor.dump()
if self.options.pex:
pex_name = self.binary.name if self.binary else Target.maybe_readable_identify(
self.targets)
pex_path = os.path.join(self.root_dir, 'dist', '%s.pex' % pex_name)
builder.build(pex_path)
print('Wrote %s' % pex_path)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def _compile_target(self, target):
# "Compiles" a target by forming an isolated chroot of its sources and transitive deps and then
# attempting to import each of the target's sources in the case of a python library or else the
# entry point in the case of a python binary.
#
# For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like:
#
# if __name__ == '__main__':
# import lib.core
# import lib.util
#
# For a binary with entry point lib.bin:main the "compiler" main file would look like:
#
# if __name__ == '__main__':
# from lib.bin import main
#
# In either case the main file is executed within the target chroot to reveal missing BUILD
# dependencies.
with self.context.new_workunit(name=target.address.spec):
modules = []
if isinstance(target, PythonBinary):
source = 'entry_point {}'.format(target.entry_point)
components = target.entry_point.rsplit(':', 1)
module = components[0]
if len(components) == 2:
function = components[1]
data = TemplateData(source=source,
import_statement='from {} import {}'.format(module, function))
else:
data = TemplateData(source=source, import_statement='import {}'.format(module))
modules.append(data)
else:
for path in target.sources_relative_to_source_root():
if path.endswith('.py'):
if os.path.basename(path) == '__init__.py':
module_path = os.path.dirname(path)
else:
module_path, _ = os.path.splitext(path)
source = 'file {}'.format(os.path.join(target.target_base, path))
module = module_path.replace(os.path.sep, '.')
data = TemplateData(source=source, import_statement='import {}'.format(module))
modules.append(data)
if not modules:
# Nothing to eval, so a trivial compile success.
return 0
interpreter = self.select_interpreter_for_targets([target])
if isinstance(target, PythonBinary):
pexinfo, platforms = target.pexinfo, target.platforms
else:
pexinfo, platforms = None, None
with self.temporary_pex_builder(interpreter=interpreter, pex_info=pexinfo) as builder:
with self.context.new_workunit(name='resolve'):
chroot = PythonChroot(
context=self.context,
targets=[target],
builder=builder,
platforms=platforms,
interpreter=interpreter)
chroot.dump()
with temporary_file() as imports_file:
generator = Generator(pkgutil.get_data(__name__, self._EVAL_TEMPLATE_PATH),
chroot=chroot.path(),
modules=modules)
generator.write(imports_file)
imports_file.close()
builder.set_executable(imports_file.name, '__pants_python_eval__.py')
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
with self.context.new_workunit(name='eval',
labels=[WorkUnit.COMPILER, WorkUnit.RUN, WorkUnit.TOOL],
cmd=' '.join(pex.cmdline())) as workunit:
returncode = pex.run(stdout=workunit.output('stdout'), stderr=workunit.output('stderr'))
workunit.set_outcome(WorkUnit.SUCCESS if returncode == 0 else WorkUnit.FAILURE)
if returncode != 0:
self.context.log.error('Failed to eval {}'.format(target.address.spec))
return returncode
def test_get_current_platform():
expected_platforms = [Platform.current(), 'linux-x86_64']
assert set(expected_platforms) == set(
PythonChroot.get_platforms(['current', 'linux-x86_64']))
def _compile_target(self, target):
# "Compiles" a target by forming an isolated chroot of its sources and transitive deps and then
# attempting to import each of the target's sources in the case of a python library or else the
# entry point in the case of a python binary.
#
# For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like:
#
# if __name__ == '__main__':
# import lib.core
# import lib.util
#
# For a binary with entry point lib.bin:main the "compiler" main file would look like:
#
# if __name__ == '__main__':
# from lib.bin import main
#
# In either case the main file is executed within the target chroot to reveal missing BUILD
# dependencies.
with self.context.new_workunit(name=target.address.spec):
modules = []
if isinstance(target, PythonBinary):
source = 'entry_point {}'.format(target.entry_point)
components = target.entry_point.rsplit(':', 1)
module = components[0]
if len(components) == 2:
function = components[1]
data = TemplateData(
source=source,
import_statement='from {} import {}'.format(
module, function))
else:
data = TemplateData(
source=source,
import_statement='import {}'.format(module))
modules.append(data)
else:
for path in target.sources_relative_to_source_root():
if path.endswith('.py'):
if os.path.basename(path) == '__init__.py':
module_path = os.path.dirname(path)
else:
module_path, _ = os.path.splitext(path)
source = 'file {}'.format(
os.path.join(target.target_base, path))
module = module_path.replace(os.path.sep, '.')
data = TemplateData(
source=source,
import_statement='import {}'.format(module))
modules.append(data)
if not modules:
# Nothing to eval, so a trivial compile success.
return 0
interpreter = self.select_interpreter_for_targets([target])
if isinstance(target, PythonBinary):
pexinfo, platforms = target.pexinfo, target.platforms
else:
pexinfo, platforms = None, None
with self.temporary_pex_builder(interpreter=interpreter,
pex_info=pexinfo) as builder:
with self.context.new_workunit(name='resolve'):
chroot = PythonChroot(context=self.context,
targets=[target],
builder=builder,
platforms=platforms,
interpreter=interpreter)
chroot.dump()
with temporary_file() as imports_file:
generator = Generator(pkgutil.get_data(
__name__, self._EVAL_TEMPLATE_PATH),
chroot=chroot.path(),
modules=modules)
generator.write(imports_file)
imports_file.close()
builder.set_executable(imports_file.name,
'__pants_python_eval__.py')
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
with self.context.new_workunit(
name='eval',
labels=[
WorkUnit.COMPILER, WorkUnit.RUN, WorkUnit.TOOL
],
cmd=' '.join(pex.cmdline())) as workunit:
returncode = pex.run(stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'))
workunit.set_outcome(WorkUnit.SUCCESS if returncode ==
0 else WorkUnit.FAILURE)
if returncode != 0:
self.context.log.error('Failed to eval {}'.format(
target.address.spec))
return returncode
def test_get_current_platform(): expected_platforms = [Platform.current(), 'linux-x86_64'] assert set(expected_platforms) == set(PythonChroot.get_platforms(['current', 'linux-x86_64']))
def test_get_current_platform(self):
expected_platforms = [Platform.current(), 'linux-x86_64']
self.assertEqual(set(expected_platforms),
set(PythonChroot.get_platforms(['current', 'linux-x86_64'])))
def test_get_current_platform(self):
expected_platforms = [Platform.current(), 'linux-x86_64']
self.assertEqual(
set(expected_platforms),
set(PythonChroot.get_platforms(['current', 'linux-x86_64'])))
