class PythonToolInstance:
logger = logging.getLogger(__name__)
def __init__(self, pex_path, interpreter):
self._pex = PEX(pex_path, interpreter=interpreter)
self._interpreter = interpreter
@property
def pex(self):
return self._pex
@property
def interpreter(self):
return self._interpreter
def _pretty_cmdline(self, args):
return safe_shlex_join(self._pex.cmdline(args))
def output(self, args, stdin_payload=None, binary_mode=False, **kwargs):
process = self._pex.run(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
with_chroot=False,
blocking=False,
**kwargs)
if stdin_payload is not None:
stdin_payload = ensure_binary(stdin_payload)
(stdout, stderr) = process.communicate(input=stdin_payload)
if not binary_mode:
stdout = stdout.decode()
stderr = stderr.decode()
return (stdout, stderr, process.returncode, self._pretty_cmdline(args))
@contextmanager
def run_with(self, workunit_factory, args, **kwargs):
# TODO(John Sirois): remove when we ingest a pex with a fix for:
# https://github.com/pantsbuild/pex/issues/707
# Ensure we don't leak source files or undeclared 3rdparty requirements into the PEX
# environment.
supplied_env = kwargs.pop('env', None)
env = (supplied_env or os.environ).copy()
pythonpath = env.pop('PYTHONPATH', None)
if pythonpath:
self.logger.warning(
'scrubbed PYTHONPATH={} from environment'.format(pythonpath))
cmdline = self._pretty_cmdline(args)
with workunit_factory(cmd=cmdline) as workunit:
exit_code = self._pex.run(args,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'),
with_chroot=False,
blocking=True,
env=env,
**kwargs)
yield cmdline, exit_code, workunit
def run(self, *args, **kwargs):
with self.run_with(*args, **kwargs) as (cmdline, exit_code, _):
return cmdline, exit_code
class PythonToolInstance:
logger = logging.getLogger(__name__)
def __init__(self, pex_path, interpreter):
self._pex = PEX(pex_path, interpreter=interpreter)
self._interpreter = interpreter
@property
def pex(self):
return self._pex
@property
def interpreter(self):
return self._interpreter
def _pretty_cmdline(self, args):
return safe_shlex_join(self._pex.cmdline(args))
def output(self, args, stdin_payload=None, binary_mode=False, **kwargs):
process = self._pex.run(
args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
with_chroot=False,
blocking=False,
**kwargs,
)
if stdin_payload is not None:
stdin_payload = ensure_binary(stdin_payload)
(stdout, stderr) = process.communicate(input=stdin_payload)
if not binary_mode:
stdout = stdout.decode()
stderr = stderr.decode()
return (stdout, stderr, process.returncode, self._pretty_cmdline(args))
@contextmanager
def run_with(self, workunit_factory, args, **kwargs):
cmdline = self._pretty_cmdline(args)
with workunit_factory(cmd=cmdline) as workunit:
exit_code = self._pex.run(
args,
stdout=workunit.output("stdout"),
stderr=workunit.output("stderr"),
with_chroot=False,
blocking=True,
**kwargs,
)
yield cmdline, exit_code, workunit
def run(self, *args, **kwargs):
with self.run_with(*args, **kwargs) as (cmdline, exit_code, _):
return cmdline, exit_code
def execute_codegen(self, target, results_dir):
self.context.log.info("Processing target {}".format(target))
requirements_pex = self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)
interpreter = self.context.products.get_data(PythonInterpreter)
pex_info = PexInfo.default(interpreter)
pex_info.pex_path = requirements_pex.path()
with temporary_dir() as source_pex_chroot:
sources_pex_builder = PEXBuilder(
path=source_pex_chroot,
interpreter=interpreter,
copy=True,
pex_info=pex_info
)
pex_build_util.dump_sources(sources_pex_builder, target, self.context.log)
sources_pex_builder.freeze()
codegen_pex = PEX(sources_pex_builder.path(), interpreter)
setup_py_paths = []
for source in target.sources_relative_to_source_root():
if os.path.basename(source) == 'setup.py':
setup_py_paths.append(source)
if len(setup_py_paths) != 1:
raise TaskError(
'Expected target {} to own exactly one setup.py, found {}'.format(
setup_py_paths,
len(setup_py_paths)
)
)
setup_py_path = setup_py_paths[0]
result_code = codegen_pex.run(
with_chroot=True,
blocking=True,
args=(setup_py_path, 'build_ext', '--inplace', '--verbose'),
# Passing PATH helps cython find the correct c++ compiler
env={'libraries': results_dir, 'PATH': os.getenv('PATH')}
)
if result_code != 0:
raise TaskError(
'creating cython library failed',
exit_code=result_code,
failed_targets=[target]
)
library_source_path = os.path.join(
sources_pex_builder.path(),
os.path.dirname(setup_py_path),
target.output
)
library_output = os.path.join(results_dir, target.output)
safe_mkdir_for(library_output)
shutil.move(library_source_path, library_output)
self.context.log.info(
'created library {}'.format(os.path.relpath(library_output, get_buildroot()))
)
def execute(self):
binary = self.require_single_root_target()
if isinstance(binary, PythonBinary):
# We can't throw if binary isn't a PythonBinary, because perhaps we were called on a
# jvm_binary, in which case we have to no-op and let jvm_run do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(
self.context.targets())
with self.temporary_pex_builder(
interpreter=interpreter,
pex_info=binary.pexinfo) as builder:
chroot = PythonChroot(targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
self.context.lock.release()
with self.context.new_workunit(name='run',
labels=[WorkUnit.RUN]):
po = pex.run(blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def main():
parser = configure_clp()
options, args = parser.parse_args()
with TraceLogger.env_override(PEX_VERBOSE=options.verbosity):
pex_builder = build_pex(args, options)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), args),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
return pex.run(args=list(args))
def main():
parser, resolver_options_builder = configure_clp()
# split arguments early because optparse is dumb
args = sys.argv[1:]
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def main():
parser = configure_clp()
options, args = parser.parse_args()
verbosity = 5 if options.verbosity else -1
with Tracer.env_override(PEX_VERBOSE=verbosity, PEX_HTTP=verbosity):
pex_builder = build_pex(args, options)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), args), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
return pex.run(args=list(args))
def execute(self, **pex_run_kwargs):
(accept_predicate, reject_predicate) = Target.lang_discriminator('python')
targets = self.require_homogeneous_targets(accept_predicate, reject_predicate)
if targets:
# We can't throw if the target isn't a python target, because perhaps we were called on a
# JVM target, in which case we have to no-op and let scala repl do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(targets)
extra_requirements = []
if self.get_options().ipython:
entry_point = self.get_options().ipython_entry_point
for req in self.get_options().ipython_requirements:
extra_requirements.append(PythonRequirement(req))
else:
entry_point = 'code:interact'
with self.temporary_chroot(interpreter=interpreter, targets=targets,
extra_requirements=extra_requirements,
pre_freeze=lambda ch: ch.builder.set_entry_point(entry_point)) as chroot:
pex = PEX(chroot.builder.path(), interpreter=interpreter)
self.context.release_lock()
with stty_utils.preserve_stty_settings():
with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]):
po = pex.run(blocking=False, **pex_run_kwargs)
try:
return po.wait()
except KeyboardInterrupt:
pass
def execute(self):
binary = self.require_single_root_target()
if isinstance(binary, PythonBinary):
# We can't throw if binary isn't a PythonBinary, because perhaps we were called on a
# jvm_binary, in which case we have to no-op and let jvm_run do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(self.context.targets())
with self.temporary_pex_builder(interpreter=interpreter, pex_info=binary.pexinfo) as builder:
chroot = PythonChroot(
targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
self.context.lock.release()
with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]):
po = pex.run(blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def execute(self):
binary = self.require_single_root_target()
if isinstance(binary, PythonBinary):
# We can't throw if binary isn't a PythonBinary, because perhaps we were called on a
# jvm_binary, in which case we have to no-op and let jvm_run do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(self.context.targets())
with self.temporary_chroot(interpreter=interpreter, pex_info=binary.pexinfo, targets=[binary], platforms=binary.platforms) as chroot:
pex = PEX(chroot.builder.path(), interpreter=interpreter)
self.context.release_lock()
with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]):
args = []
for arg in self.get_options().args:
args.extend(safe_shlex_split(arg))
args += self.get_passthru_args()
po = pex.run(blocking=False, args=args)
try:
result = po.wait()
if result != 0:
msg = '{interpreter} {entry_point} {args} ... exited non-zero ({code})'.format(
interpreter=interpreter.binary,
entry_point=binary.entry_point,
args=' '.join(args),
code=result)
raise TaskError(msg, exit_code=result)
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def execute(self):
binary = self.require_single_root_target()
if isinstance(binary, PythonBinary):
# We can't throw if binary isn't a PythonBinary, because perhaps we were called on a
# jvm_binary, in which case we have to no-op and let jvm_run do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(self.context.targets())
with self.temporary_pex_builder(interpreter=interpreter, pex_info=binary.pexinfo) as builder:
chroot = PythonChroot(
targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
self.context.release_lock()
with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]):
args = self.get_options().args + self.get_passthru_args()
po = pex.run(blocking=False, args=args)
try:
result = po.wait()
if result != 0:
raise TaskError('python {args} ... exited non-zero ({code})' %
dict(args=args, code=result),
exit_code=result)
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def execute(self):
binary = self.require_single_root_target()
if isinstance(binary, PythonBinary):
# We can't throw if binary isn't a PythonBinary, because perhaps we were called on a
# jvm_binary, in which case we have to no-op and let jvm_run do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(
self.context.targets())
with self.temporary_pex_builder(
interpreter=interpreter,
pex_info=binary.pexinfo) as builder:
chroot = PythonChroot(targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter)
chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
self.context.release_lock()
with self.context.new_workunit(name='run',
labels=[WorkUnit.RUN]):
args = self.get_options().args + self.get_passthru_args()
po = pex.run(blocking=False, args=args)
try:
result = po.wait()
if result != 0:
raise TaskError(
'python {args} ... exited non-zero ({code})' %
dict(args=args, code=result),
exit_code=result)
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def _spawn_pip_isolated(self, args, cache=None, interpreter=None):
pip_args = [
'--disable-pip-version-check', '--isolated', '--exists-action', 'i'
]
# The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor of 3.
pex_verbosity = ENV.PEX_VERBOSE
pip_verbosity = pex_verbosity // 3
if pip_verbosity > 0:
pip_args.append('-{}'.format('v' * pip_verbosity))
else:
pip_args.append('-q')
if cache:
pip_args.extend(['--cache-dir', cache])
else:
pip_args.append('--no-cache-dir')
command = pip_args + args
with ENV.strip().patch(PEX_ROOT=ENV.PEX_ROOT,
PEX_VERBOSE=str(pex_verbosity)) as env:
from pex.pex import PEX
pip = PEX(pex=self._pip_pex_path, interpreter=interpreter)
return Job(command=pip.cmdline(command),
process=pip.run(args=command, env=env, blocking=False))
class PythonToolInstance(object):
def __init__(self, pex_path, interpreter):
self._pex = PEX(pex_path, interpreter=interpreter)
self._interpreter = interpreter
@property
def pex(self):
return self._pex
@property
def interpreter(self):
return self._interpreter
def _pretty_cmdline(self, args):
return safe_shlex_join(self._pex.cmdline(args))
def output(self, args, stdin_payload=None, binary_mode=False, **kwargs):
process = self._pex.run(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
with_chroot=False,
blocking=False,
**kwargs)
if stdin_payload is not None:
stdin_payload = ensure_binary(stdin_payload)
(stdout, stderr) = process.communicate(input=stdin_payload)
if not binary_mode:
stdout = stdout.decode('utf-8')
stderr = stderr.decode('utf-8')
return (stdout, stderr, process.returncode, self._pretty_cmdline(args))
@contextmanager
def run_with(self, workunit_factory, args, **kwargs):
cmdline = self._pretty_cmdline(args)
with workunit_factory(cmd=cmdline) as workunit:
exit_code = self._pex.run(args,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'),
with_chroot=False,
blocking=True,
**kwargs)
yield cmdline, exit_code, workunit
def run(self, *args, **kwargs):
with self.run_with(*args, **kwargs) as (cmdline, exit_code, _):
return cmdline, exit_code
def test_osx_platform_intel_issue_523():
def bad_interpreter(include_site_extras=True):
return PythonInterpreter.from_binary(
_KNOWN_BAD_APPLE_INTERPRETER,
include_site_extras=include_site_extras)
interpreter = bad_interpreter(include_site_extras=False)
with temporary_dir() as cache:
# We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to
# successfully install psutil.
for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT):
for resolved_dist in resolver.resolve(
[requirement],
cache=cache,
# We can't use wheels since we're bootstrapping them.
precedence=(SourcePackage, EggPackage),
interpreter=interpreter):
dist = resolved_dist.distribution
interpreter = interpreter.with_extra(dist.key, dist.version,
dist.location)
with nested(
yield_pex_builder(installer_impl=WheelInstaller,
interpreter=interpreter),
temporary_filename()) as (pb, pex_file):
for resolved_dist in resolver.resolve(['psutil==5.4.3'],
cache=cache,
precedence=(SourcePackage,
WheelPackage),
interpreter=interpreter):
pb.add_dist_location(resolved_dist.distribution.location)
pb.build(pex_file)
# NB: We want PEX to find the bare bad interpreter at runtime.
pex = PEX(pex_file, interpreter=bad_interpreter())
args = [
'-c',
'import pkg_resources; print(pkg_resources.get_supported_platform())'
]
env = os.environ.copy()
env['PEX_VERBOSE'] = '1'
process = pex.run(args=args,
env=env,
blocking=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
assert 0 == process.returncode, (
'Process failed with exit code {} and stderr:\n{}'.format(
process.returncode, stderr))
# Verify this all worked under the previously problematic pkg_resources-reported platform.
release, _, _ = platform.mac_ver()
major_minor = '.'.join(release.split('.')[:2])
assert to_bytes(
'macosx-{}-intel'.format(major_minor)) == stdout.strip()
def execute(self):
if self.old_options.pex and self.old_options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.old_options.entry_point and self.old_options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.old_options.verbose:
print('Build operating on targets: %s' % ' '.join(str(target) for target in self.targets))
builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
pex_info=self.binary.pexinfo if self.binary else None)
if self.old_options.entry_point:
builder.set_entry_point(self.old_options.entry_point)
if self.old_options.ipython:
if not self.config.has_section('python-ipython'):
self.error('No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
if builder.info.entry_point is None:
self.error('Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython', 'requirements', default=[])
for requirement in requirements:
self.extra_requirements.append(PythonRequirement(requirement))
executor = PythonChroot(
targets=self.targets,
extra_requirements=self.extra_requirements,
builder=builder,
platforms=self.binary.platforms if self.binary else None,
interpreter=self.interpreter,
conn_timeout=self.old_options.conn_timeout)
executor.dump()
if self.old_options.pex:
pex_name = self.binary.name if self.binary else Target.maybe_readable_identify(self.targets)
pex_path = os.path.join(self.root_dir, 'dist', '%s.pex' % pex_name)
builder.build(pex_path)
print('Wrote %s' % pex_path)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def run_simple_pex(pex, args=(), interpreter=None, stdin=None, **kwargs):
p = PEX(pex, interpreter=interpreter)
process = p.run(args=args,
blocking=False,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**kwargs)
stdout, _ = process.communicate(input=stdin)
return stdout.replace(b'\r', b''), process.returncode
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.'
)
with ENV.patch(PEX_VERBOSE=str(options.verbosity),
PEX_ROOT=options.pex_root) as patched_env:
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options)
pex_builder.freeze(bytecode_compile=options.compile)
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
V=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(
tmp_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!',
V=1)
log('Running on platform {} but built for {}'.format(
Platform.current(), ', '.join(map(str,
options.platforms))),
V=1)
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
V=options.verbosity)
sys.exit(pex.run(args=list(cmdline), env=patched_env))
def run_simple_pex(pex, args=(), interpreter=None, stdin=None, **kwargs):
p = PEX(pex, interpreter=interpreter)
process = p.run(args=args,
blocking=False,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**kwargs)
stdout, _ = process.communicate(input=stdin)
print(stdout.decode('utf-8') if PY3 else stdout)
return stdout.replace(b'\r', b''), process.returncode
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.'
)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(
options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
pex_builder.freeze()
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!'
)
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
sys.exit(pex.run(args=list(cmdline)))
def _run_mypy(self, py3_interpreter, mypy_args, **kwargs):
pex_info = PexInfo.default()
pex_info.entry_point = 'mypy'
mypy_version = self.get_options().mypy_version
mypy_requirement_pex = self.resolve_requirement_strings(
py3_interpreter, ['mypy=={}'.format(mypy_version)])
path = os.path.realpath(os.path.join(self.workdir, str(py3_interpreter.identity), mypy_version))
if not os.path.isdir(path):
self.merge_pexes(path, pex_info, py3_interpreter, [mypy_requirement_pex])
pex = PEX(path, py3_interpreter)
return pex.run(mypy_args, **kwargs)
def _run_mypy(self, py3_interpreter, mypy_args, **kwargs):
pex_info = PexInfo.default()
pex_info.entry_point = 'mypy'
mypy_version = self.get_options().mypy_version
mypy_requirement_pex = self.resolve_requirement_strings(
py3_interpreter, ['mypy=={}'.format(mypy_version)])
path = os.path.realpath(os.path.join(self.workdir, str(py3_interpreter.identity), mypy_version))
if not os.path.isdir(path):
self.merge_pexes(path, pex_info, py3_interpreter, [mypy_requirement_pex])
pex = PEX(path, py3_interpreter)
return pex.run(mypy_args, **kwargs)
class PythonToolInstance(object):
def __init__(self, pex_path, interpreter):
self._pex = PEX(pex_path, interpreter=interpreter)
def run(self, workunit_factory, args, **kwargs):
cmdline = ' '.join(self._pex.cmdline(args))
with workunit_factory(cmd=cmdline) as workunit:
exit_code = self._pex.run(args,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'),
with_chroot=False,
blocking=True,
**kwargs)
return cmdline, exit_code
def execute(self):
binary = self.require_single_root_target()
if isinstance(binary, PythonBinary):
# We can't throw if binary isn't a PythonBinary, because perhaps we were called on a
# jvm_binary, in which case we have to no-op and let jvm_run do its thing.
# TODO(benjy): Use MutexTask to coordinate this.
interpreter = self.context.products.get_data(PythonInterpreter)
with temporary_dir() as tmpdir:
# Create a wrapper pex to "merge" the other pexes into via PEX_PATH.
builder = PEXBuilder(tmpdir,
interpreter,
pex_info=binary.pexinfo)
builder.freeze()
pexes = [
self.context.products.get_data(
ResolveRequirements.REQUIREMENTS_PEX),
self.context.products.get_data(
GatherSources.PYTHON_SOURCES)
]
# TODO: Expose the path as a property in pex, instead of relying on
# fishing it out of the cmdline.
pex_path = os.pathsep.join([pex.cmdline()[1] for pex in pexes])
pex = PEX(tmpdir, interpreter)
self.context.release_lock()
with self.context.new_workunit(name='run',
labels=[WorkUnitLabel.RUN]):
args = []
for arg in self.get_options().args:
args.extend(safe_shlex_split(arg))
args += self.get_passthru_args()
po = pex.run(blocking=False,
args=args,
env={'PEX_PATH': pex_path})
try:
result = po.wait()
if result != 0:
msg = '{interpreter} {entry_point} {args} ... exited non-zero ({code})'.format(
interpreter=interpreter.binary,
entry_point=binary.entry_point,
args=' '.join(args),
code=result)
raise TaskError(msg, exit_code=result)
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.')
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
pex_builder.freeze(bytecode_compile=options.compile)
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, V=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(
tmp_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time
)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!')
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline),
V=options.verbosity)
sys.exit(pex.run(args=list(cmdline)))
def execute(self):
(accept_predicate,
reject_predicate) = Target.lang_discriminator('python')
targets = self.require_homogeneous_targets(accept_predicate,
reject_predicate)
if targets:
# We can't throw if the target isn't a python target, because perhaps we were called on a
# JVM target, in which case we have to no-op and let scala repl do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(targets)
extra_requirements = []
if self.context.options.python_repl_ipython:
entry_point = self.context.config.get(
'python-ipython',
'entry_point',
default='IPython:start_ipython')
ipython_requirements = self.context.config.getlist(
'python-ipython',
'requirements',
default=['ipython==1.0.0'])
for req in ipython_requirements:
extra_requirements.append(PythonRequirement(req))
else:
entry_point = 'code:interact'
with self.temporary_pex_builder(
interpreter=interpreter) as builder:
builder.set_entry_point(entry_point)
chroot = PythonChroot(targets=targets,
extra_requirements=extra_requirements,
builder=builder,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
self.context.lock.release()
with stty_utils.preserve_stty_settings():
with self.context.new_workunit(name='run',
labels=[WorkUnit.RUN]):
po = pex.run(blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
pass
def _spawn_pip_isolated(self, args, cache=None, interpreter=None):
pip_args = [
# We vendor the version of pip we want so pip should never check for updates.
"--disable-pip-version-check",
# Don't read pip configuration files like `~/.config/pip/pip.conf`.
"--isolated",
# If we want to warn about a version of python we support, we should do it, not pip.
"--no-python-version-warning",
# If pip encounters a duplicate file path during its operations we don't want it to prompt
# and we'd also like to know about this since it should never occur. We leverage the pip
# global option:
# --exists-action <action>
# Default action when a path already exists: (s)witch, (i)gnore, (w)ipe, (b)ackup, (a)bort.
"--exists-action",
"a",
]
# The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor of 3.
pex_verbosity = ENV.PEX_VERBOSE
pip_verbosity = pex_verbosity // 3
if pip_verbosity > 0:
pip_args.append("-{}".format("v" * pip_verbosity))
else:
pip_args.append("-q")
if cache:
pip_args.extend(["--cache-dir", cache])
else:
pip_args.append("--no-cache-dir")
command = pip_args + args
with ENV.strip().patch(PEX_ROOT=cache or ENV.PEX_ROOT,
PEX_VERBOSE=str(pex_verbosity)) as env:
# Guard against API calls from environment with ambient PYTHONPATH preventing pip PEX
# bootstrapping. See: https://github.com/pantsbuild/pex/issues/892
pythonpath = env.pop("PYTHONPATH", None)
if pythonpath:
TRACER.log(
"Scrubbed PYTHONPATH={} from the pip PEX environment.".
format(pythonpath),
V=3)
from pex.pex import PEX
pip = PEX(pex=self._pip_pex_path, interpreter=interpreter)
return Job(command=pip.cmdline(command),
process=pip.run(args=command, env=env, blocking=False))
def run_simple_pex(
pex, # type: str
args=(), # type: Iterable[str]
interpreter=None, # type: Optional[PythonInterpreter]
stdin=None, # type: Optional[bytes]
**kwargs # type: Any
):
# type: (...) -> Tuple[bytes, int]
p = PEX(pex, interpreter=interpreter)
process = p.run(args=args,
blocking=False,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**kwargs)
stdout, _ = process.communicate(input=stdin)
return stdout.replace(b"\r", b""), process.returncode
def main(args=None):
args = args or sys.argv[1:]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(
options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform and Platform.current() not in options.platform:
log('WARNING: attempting to run PEX with incompatible platforms!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def execute(self):
binary = self.require_single_root_target()
if isinstance(binary, PythonBinary):
# We can't throw if binary isn't a PythonBinary, because perhaps we were called on a
# jvm_binary, in which case we have to no-op and let jvm_run do its thing.
# TODO(benjy): Use MutexTask to coordinate this.
interpreter = self.context.products.get_data(PythonInterpreter)
with temporary_dir() as tmpdir:
# Create a wrapper pex to "merge" the other pexes into via PEX_PATH.
builder = PEXBuilder(tmpdir, interpreter, pex_info=binary.pexinfo)
builder.freeze()
pexes = [
self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX),
self.context.products.get_data(GatherSources.PYTHON_SOURCES),
]
# TODO: Expose the path as a property in pex, instead of relying on
# fishing it out of the cmdline.
pex_path = os.pathsep.join([pex.cmdline()[1] for pex in pexes])
pex = PEX(tmpdir, interpreter)
self.context.release_lock()
with self.context.new_workunit(name="run", labels=[WorkUnitLabel.RUN]):
args = []
for arg in self.get_options().args:
args.extend(safe_shlex_split(arg))
args += self.get_passthru_args()
po = pex.run(blocking=False, args=args, env={"PEX_PATH": pex_path})
try:
result = po.wait()
if result != 0:
msg = "{interpreter} {entry_point} {args} ... exited non-zero ({code})".format(
interpreter=interpreter.binary,
entry_point=binary.entry_point,
args=" ".join(args),
code=result,
)
raise TaskError(msg, exit_code=result)
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def _run_python_tests(self, targets, stdout, stderr):
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current', ),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = []
test_args.extend(PythonTestBuilder.generate_junit_args(targets))
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(targets)
test_args.extend(args)
sources = list(
itertools.chain(
*[t.sources_relative_to_buildroot() for t in targets]))
pex = PEX(builder.path(), interpreter=self.interpreter)
rc = pex.run(args=test_args + sources,
blocking=True,
setsid=True,
stdout=stdout,
stderr=stderr)
# TODO(wickman): If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestResult.rc(rc)
except Exception:
import traceback
print('Failed to run test!', file=stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
return rv
def _spawn_pip_isolated(self, args, cache=None, interpreter=None):
pip_args = [
# We vendor the version of pip we want so pip should never check for updates.
'--disable-pip-version-check',
# Don't read pip configuration files like `~/.config/pip/pip.conf`.
'--isolated',
# If we want to warn about a version of python we support, we should do it, not pip.
'--no-python-version-warning'
]
# The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor of 3.
pex_verbosity = ENV.PEX_VERBOSE
pip_verbosity = pex_verbosity // 3
if pip_verbosity > 0:
pip_args.append('-{}'.format('v' * pip_verbosity))
else:
pip_args.append('-q')
if cache:
pip_args.extend(['--cache-dir', cache])
else:
pip_args.append('--no-cache-dir')
command = pip_args + args
with ENV.strip().patch(PEX_ROOT=ENV.PEX_ROOT, PEX_VERBOSE=str(pex_verbosity)) as env:
# Guard against API calls from environment with ambient PYTHONPATH preventing pip PEX
# bootstrapping. See: https://github.com/pantsbuild/pex/issues/892
pythonpath = env.pop('PYTHONPATH', None)
if pythonpath:
TRACER.log('Scrubbed PYTHONPATH={} from the pip PEX environment.'.format(pythonpath), V=3)
from pex.pex import PEX
pip = PEX(pex=self._pip_pex_path, interpreter=interpreter)
return Job(
command=pip.cmdline(command),
process=pip.run(
args=command,
env=env,
blocking=False
)
)
def test_execute_interpreter_dashm_module():
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot)
pex_builder.add_source(None, 'foo/__init__.py')
with tempfile.NamedTemporaryFile() as fp:
fp.write(b'import sys; print(" ".join(sys.argv))')
fp.flush()
pex_builder.add_source(fp.name, 'foo/bar.py')
pex_builder.freeze()
pex = PEX(pex_chroot)
process = pex.run(args=['-m', 'foo.bar', 'one', 'two'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
blocking=False)
stdout, stderr = process.communicate()
assert 0 == process.returncode
assert b'foo.bar one two\n' == stdout
assert b'' == stderr
def main(args=None):
args = args or sys.argv[1:]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def test_pex_builder_preamble():
# type: () -> None
with temporary_dir() as td:
target = os.path.join(td, "foo.pex")
should_create = os.path.join(td, "foo.1")
tempfile_preamble = "\n".join(
["import sys", "open('{0}', 'w').close()".format(should_create), "sys.exit(3)"]
)
pb = PEXBuilder(preamble=tempfile_preamble)
pb.build(target)
assert not os.path.exists(should_create)
pex = PEX(target, interpreter=pb.interpreter)
process = pex.run(blocking=False)
process.wait()
assert process.returncode == 3
assert os.path.exists(should_create)
def execute(self):
(accept_predicate, reject_predicate) = Target.lang_discriminator('python')
targets = self.require_homogeneous_targets(accept_predicate, reject_predicate)
if targets:
# We can't throw if the target isn't a python target, because perhaps we were called on a
# JVM target, in which case we have to no-op and let scala repl do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
interpreter = self.select_interpreter_for_targets(targets)
extra_requirements = []
if self.get_options().ipython:
entry_point = self.context.config.get('python-ipython', 'entry_point',
default='IPython:start_ipython')
ipython_requirements = self.context.config.getlist('python-ipython', 'requirements',
default=['ipython==1.0.0'])
for req in ipython_requirements:
extra_requirements.append(PythonRequirement(req))
else:
entry_point = 'code:interact'
with self.temporary_pex_builder(interpreter=interpreter) as builder:
builder.set_entry_point(entry_point)
chroot = PythonChroot(
targets=targets,
extra_requirements=extra_requirements,
builder=builder,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
self.context.lock.release()
with stty_utils.preserve_stty_settings():
with self.context.new_workunit(name='run', labels=[WorkUnit.RUN]):
po = pex.run(blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
pass
def test_execute_interpreter_dashm_module():
# type: () -> None
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot)
pex_builder.add_source(None, "foo/__init__.py")
with tempfile.NamedTemporaryFile() as fp:
fp.write(b'import sys; print(" ".join(sys.argv))')
fp.flush()
pex_builder.add_source(fp.name, "foo/bar.py")
pex_builder.freeze()
pex = PEX(pex_chroot)
process = pex.run(
args=["-m", "foo.bar", "one", "two"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
blocking=False,
)
stdout, stderr = process.communicate()
assert 0 == process.returncode
assert b"foo.bar one two\n" == stdout
assert b"" == stderr
def _run_python_tests(self, targets, stdout, stderr):
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = []
test_args.extend(PythonTestBuilder.generate_junit_args(targets))
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(targets)
test_args.extend(args)
sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets]))
pex = PEX(builder.path(), interpreter=self.interpreter)
rc = pex.run(args=test_args + sources, blocking=True, setsid=True,
stdout=stdout, stderr=stderr)
# TODO(wickman): If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestResult.rc(rc)
except Exception:
import traceback
print('Failed to run test!', file=stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
return rv
def test_pex_builder_preamble():
with temporary_dir() as td:
target = os.path.join(td, 'foo.pex')
should_create = os.path.join(td, 'foo.1')
tempfile_preamble = "\n".join([
"import sys",
"open('{0}', 'w').close()".format(should_create),
"sys.exit(3)"
])
pb = PEXBuilder(preamble=tempfile_preamble)
pb.build(target)
assert not os.path.exists(should_create)
pex = PEX(target, interpreter=pb.interpreter)
process = pex.run(blocking=False)
process.wait()
assert process.returncode == 3
assert os.path.exists(should_create)
def main():
parser, resolver_options_builder = configure_clp()
# split arguments early because optparse is dumb
args = sys.argv[1:]
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser = configure_clp()
try:
separator = args.index("--")
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options = parser.parse_args(args=args)
# Ensure the TMPDIR is an absolute path (So subprocesses that change CWD can find it) and
# that it exists.
tmpdir = os.path.realpath(options.tmpdir)
if not os.path.exists(tmpdir):
die("The specified --tmpdir does not exist: {}".format(tmpdir))
if not os.path.isdir(tmpdir):
die("The specified --tmpdir is not a directory: {}".format(tmpdir))
tempfile.tempdir = os.environ["TMPDIR"] = tmpdir
if options.cache_dir:
pex_warnings.warn(
"The --cache-dir option is deprecated, use --pex-root instead.")
if options.pex_root and options.cache_dir != options.pex_root:
die("Both --cache-dir and --pex-root were passed with conflicting values. "
"Just set --pex-root.")
if options.disable_cache:
def warn_ignore_pex_root(set_via):
pex_warnings.warn(
"The pex root has been set via {via} but --disable-cache is also set. "
"Ignoring {via} and disabling caches.".format(via=set_via))
if options.cache_dir:
warn_ignore_pex_root("--cache-dir")
elif options.pex_root:
warn_ignore_pex_root("--pex-root")
elif os.environ.get("PEX_ROOT"):
warn_ignore_pex_root("PEX_ROOT")
pex_root = safe_mkdtemp()
else:
pex_root = options.cache_dir or options.pex_root or ENV.PEX_ROOT
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.'
)
if options.pex_repository and (options.indexes or options.find_links):
die('The "--pex-repository" option cannot be used together with the "--index" or '
'"--find-links" options.')
with ENV.patch(PEX_VERBOSE=str(options.verbosity),
PEX_ROOT=pex_root,
TMPDIR=tmpdir) as patched_env:
with TRACER.timed("Building pex"):
pex_builder = build_pex(options.requirements,
options,
cache=ENV.PEX_ROOT)
pex_builder.freeze(bytecode_compile=options.compile)
interpreter = pex_builder.interpreter
pex = PEX(pex_builder.path(),
interpreter=interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log("Saving PEX file to %s" % options.pex_name,
V=options.verbosity)
pex_builder.build(
options.pex_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time,
)
if options.seed != Seed.NONE:
seed_info = seed_cache(options,
pex,
verbose=options.seed == Seed.VERBOSE)
print(seed_info)
else:
if not _compatible_with_current_platform(interpreter,
options.platforms):
log("WARNING: attempting to run PEX with incompatible platforms!",
V=1)
log(
"Running on platform {} but built for {}".format(
interpreter.platform,
", ".join(map(str, options.platforms))),
V=1,
)
log(
"Running PEX file at %s with args %s" %
(pex_builder.path(), cmdline),
V=options.verbosity,
)
sys.exit(pex.run(args=list(cmdline), env=patched_env))
def _compile_target(self, vt):
"""'Compiles' a python target.
'Compiling' means forming an isolated chroot of its sources and transitive deps and then
attempting to import each of the target's sources in the case of a python library or else the
entry point in the case of a python binary.
For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like:
if __name__ == '__main__':
import lib.core
import lib.util
For a binary with entry point lib.bin:main the "compiler" main file would look like:
if __name__ == '__main__':
from lib.bin import main
In either case the main file is executed within the target chroot to reveal missing BUILD
dependencies.
"""
target = vt.target
with self.context.new_workunit(name=target.address.spec):
modules = self._get_modules(target)
if not modules:
# Nothing to eval, so a trivial compile success.
return 0
interpreter = self._get_interpreter_for_target_closure(target)
reqs_pex = self._resolve_requirements_for_versioned_target_closure(interpreter, vt)
srcs_pex = self._source_pex_for_versioned_target_closure(interpreter, vt)
# Create the executable pex.
exec_pex_parent = os.path.join(self.workdir, 'executable_pex')
executable_file_content = self._get_executable_file_content(exec_pex_parent, modules)
hasher = hashlib.sha1()
hasher.update(reqs_pex.path().encode('utf-8'))
hasher.update(srcs_pex.path().encode('utf-8'))
hasher.update(executable_file_content.encode('utf-8'))
exec_file_hash = hasher.hexdigest()
exec_pex_path = os.path.realpath(os.path.join(exec_pex_parent, exec_file_hash))
if not os.path.isdir(exec_pex_path):
with safe_concurrent_creation(exec_pex_path) as safe_path:
# Write the entry point.
safe_mkdir(safe_path)
with open(os.path.join(safe_path, '{}.py'.format(self._EXEC_NAME)), 'w') as outfile:
outfile.write(executable_file_content)
pex_info = (target.pexinfo if isinstance(target, PythonBinary) else None) or PexInfo()
# Override any user-specified entry point, under the assumption that the
# executable_file_content does what the user intends (including, probably, calling that
# underlying entry point).
pex_info.entry_point = self._EXEC_NAME
pex_info.pex_path = ':'.join(pex.path() for pex in (reqs_pex, srcs_pex) if pex)
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
builder.freeze()
pex = PEX(exec_pex_path, interpreter)
with self.context.new_workunit(name='eval',
labels=[WorkUnitLabel.COMPILER, WorkUnitLabel.RUN,
WorkUnitLabel.TOOL],
cmd=' '.join(pex.cmdline())) as workunit:
returncode = pex.run(stdout=workunit.output('stdout'), stderr=workunit.output('stderr'))
workunit.set_outcome(WorkUnit.SUCCESS if returncode == 0 else WorkUnit.FAILURE)
if returncode != 0:
self.context.log.error('Failed to eval {}'.format(target.address.spec))
return returncode
def _compile_target(self, target):
# "Compiles" a target by forming an isolated chroot of its sources and transitive deps and then
# attempting to import each of the target's sources in the case of a python library or else the
# entry point in the case of a python binary.
#
# For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like:
#
# if __name__ == '__main__':
# import lib.core
# import lib.util
#
# For a binary with entry point lib.bin:main the "compiler" main file would look like:
#
# if __name__ == '__main__':
# from lib.bin import main
#
# In either case the main file is executed within the target chroot to reveal missing BUILD
# dependencies.
with self.context.new_workunit(name=target.address.spec):
modules = []
if isinstance(target, PythonBinary):
source = 'entry_point {}'.format(target.entry_point)
components = target.entry_point.rsplit(':', 1)
module = components[0]
if len(components) == 2:
function = components[1]
data = TemplateData(
source=source,
import_statement='from {} import {}'.format(
module, function))
else:
data = TemplateData(
source=source,
import_statement='import {}'.format(module))
modules.append(data)
else:
for path in target.sources_relative_to_source_root():
if path.endswith('.py'):
if os.path.basename(path) == '__init__.py':
module_path = os.path.dirname(path)
else:
module_path, _ = os.path.splitext(path)
source = 'file {}'.format(
os.path.join(target.target_base, path))
module = module_path.replace(os.path.sep, '.')
data = TemplateData(
source=source,
import_statement='import {}'.format(module))
modules.append(data)
if not modules:
# Nothing to eval, so a trivial compile success.
return 0
interpreter = self.select_interpreter_for_targets([target])
if isinstance(target, PythonBinary):
pexinfo, platforms = target.pexinfo, target.platforms
else:
pexinfo, platforms = None, None
with self.temporary_pex_builder(interpreter=interpreter,
pex_info=pexinfo) as builder:
with self.context.new_workunit(name='resolve'):
chroot = PythonChroot(context=self.context,
targets=[target],
builder=builder,
platforms=platforms,
interpreter=interpreter)
chroot.dump()
with temporary_file() as imports_file:
generator = Generator(pkgutil.get_data(
__name__, self._EVAL_TEMPLATE_PATH),
chroot=chroot.path(),
modules=modules)
generator.write(imports_file)
imports_file.close()
builder.set_executable(imports_file.name,
'__pants_python_eval__.py')
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
with self.context.new_workunit(
name='eval',
labels=[
WorkUnit.COMPILER, WorkUnit.RUN, WorkUnit.TOOL
],
cmd=' '.join(pex.cmdline())) as workunit:
returncode = pex.run(stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'))
workunit.set_outcome(WorkUnit.SUCCESS if returncode ==
0 else WorkUnit.FAILURE)
if returncode != 0:
self.context.log.error('Failed to eval {}'.format(
target.address.spec))
return returncode
def _spawn_pip_isolated(
self,
args, # type: Iterable[str]
package_index_configuration=None, # type: Optional[PackageIndexConfiguration]
cache=None, # type: Optional[str]
interpreter=None, # type: Optional[PythonInterpreter]
):
# type: (...) -> Job
pip_args = [
# We vendor the version of pip we want so pip should never check for updates.
"--disable-pip-version-check",
# If we want to warn about a version of python we support, we should do it, not pip.
"--no-python-version-warning",
# If pip encounters a duplicate file path during its operations we don't want it to
# prompt and we'd also like to know about this since it should never occur. We leverage
# the pip global option:
# --exists-action <action>
# Default action when a path already exists: (s)witch, (i)gnore, (w)ipe, (b)ackup,
# (a)bort.
"--exists-action",
"a",
]
if not package_index_configuration or package_index_configuration.isolated:
# Don't read PIP_ environment variables or pip configuration files like
# `~/.config/pip/pip.conf`.
pip_args.append("--isolated")
# The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor
# of 3.
pip_verbosity = ENV.PEX_VERBOSE // 3
if pip_verbosity > 0:
pip_args.append("-{}".format("v" * pip_verbosity))
else:
pip_args.append("-q")
if cache:
pip_args.extend(["--cache-dir", cache])
else:
pip_args.append("--no-cache-dir")
command = pip_args + list(args)
# N.B.: Package index options in Pep always have the same option names, but they are
# registered as subcommand-specific, so we must append them here _after_ the pip subcommand
# specified in `args`.
if package_index_configuration:
command.extend(package_index_configuration.args)
env = package_index_configuration.env if package_index_configuration else {}
with ENV.strip().patch(
PEX_ROOT=cache or ENV.PEX_ROOT, PEX_VERBOSE=str(ENV.PEX_VERBOSE), **env
) as env:
# Guard against API calls from environment with ambient PYTHONPATH preventing pip PEX
# bootstrapping. See: https://github.com/pantsbuild/pex/issues/892
pythonpath = env.pop("PYTHONPATH", None)
if pythonpath:
TRACER.log(
"Scrubbed PYTHONPATH={} from the pip PEX environment.".format(pythonpath), V=3
)
from pex.pex import PEX
pip = PEX(pex=self._pip_pex_path, interpreter=interpreter)
return Job(
command=pip.cmdline(command), process=pip.run(args=command, env=env, blocking=False)
)
def _compile_target(self, vt):
"""'Compiles' a python target.
'Compiling' means forming an isolated chroot of its sources and transitive deps and then
attempting to import each of the target's sources in the case of a python library or else the
entry point in the case of a python binary.
For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like:
if __name__ == '__main__':
import lib.core
import lib.util
For a binary with entry point lib.bin:main the "compiler" main file would look like:
if __name__ == '__main__':
from lib.bin import main
In either case the main file is executed within the target chroot to reveal missing BUILD
dependencies.
"""
target = vt.target
with self.context.new_workunit(name=target.address.spec):
modules = self._get_modules(target)
if not modules:
# Nothing to eval, so a trivial compile success.
return 0
interpreter = self._get_interpreter_for_target_closure(target)
reqs_pex = self._resolve_requirements_for_versioned_target_closure(
interpreter, vt)
srcs_pex = self._source_pex_for_versioned_target_closure(
interpreter, vt)
# Create the executable pex.
exec_pex_parent = os.path.join(self.workdir, 'executable_pex')
executable_file_content = self._get_executable_file_content(
exec_pex_parent, modules)
hasher = hashlib.sha1()
hasher.update(reqs_pex.path().encode('utf-8'))
hasher.update(srcs_pex.path().encode('utf-8'))
hasher.update(executable_file_content.encode('utf-8'))
exec_file_hash = hasher.hexdigest()
exec_pex_path = os.path.realpath(
os.path.join(exec_pex_parent, exec_file_hash))
if not os.path.isdir(exec_pex_path):
with safe_concurrent_creation(exec_pex_path) as safe_path:
# Write the entry point.
safe_mkdir(safe_path)
with open(
os.path.join(safe_path,
'{}.py'.format(self._EXEC_NAME)),
'w') as outfile:
outfile.write(executable_file_content)
pex_info = (target.pexinfo if isinstance(
target, PythonBinary) else None) or PexInfo()
# Override any user-specified entry point, under the assumption that the
# executable_file_content does what the user intends (including, probably, calling that
# underlying entry point).
pex_info.entry_point = self._EXEC_NAME
pex_info.pex_path = ':'.join(pex.path()
for pex in (reqs_pex,
srcs_pex) if pex)
builder = PEXBuilder(safe_path,
interpreter,
pex_info=pex_info)
builder.freeze()
pex = PEX(exec_pex_path, interpreter)
with self.context.new_workunit(
name='eval',
labels=[
WorkUnitLabel.COMPILER, WorkUnitLabel.RUN,
WorkUnitLabel.TOOL
],
cmd=' '.join(pex.cmdline())) as workunit:
returncode = pex.run(stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'))
workunit.set_outcome(WorkUnit.SUCCESS if returncode ==
0 else WorkUnit.FAILURE)
if returncode != 0:
self.context.log.error('Failed to eval {}'.format(
target.address.spec))
return returncode
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.cache_dir:
pex_warnings.warn('The --cache-dir option is deprecated, use --pex-root instead.')
if options.pex_root and options.cache_dir != options.pex_root:
die('Both --cache-dir and --pex-root were passed with conflicting values. '
'Just set --pex-root.')
if options.disable_cache:
def warn_ignore_pex_root(set_via):
pex_warnings.warn('The pex root has been set via {via} but --disable-cache is also set. '
'Ignoring {via} and disabling caches.'.format(via=set_via))
if options.cache_dir:
warn_ignore_pex_root('--cache-dir')
elif options.pex_root:
warn_ignore_pex_root('--pex-root')
elif os.environ.get('PEX_ROOT'):
warn_ignore_pex_root('PEX_ROOT')
pex_root = safe_mkdtemp()
else:
pex_root = options.cache_dir or options.pex_root or ENV.PEX_ROOT
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.')
with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=pex_root) as patched_env:
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, cache=ENV.PEX_ROOT)
pex_builder.freeze(bytecode_compile=options.compile)
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, V=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(
tmp_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time
)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!', V=1)
log('Running on platform {} but built for {}'
.format(Platform.current(), ', '.join(map(str, options.platforms))), V=1)
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline),
V=options.verbosity)
sys.exit(pex.run(args=list(cmdline), env=patched_env))
def _compile_target(self, target):
# "Compiles" a target by forming an isolated chroot of its sources and transitive deps and then
# attempting to import each of the target's sources in the case of a python library or else the
# entry point in the case of a python binary.
#
# For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like:
#
# if __name__ == '__main__':
# import lib.core
# import lib.util
#
# For a binary with entry point lib.bin:main the "compiler" main file would look like:
#
# if __name__ == '__main__':
# from lib.bin import main
#
# In either case the main file is executed within the target chroot to reveal missing BUILD
# dependencies.
with self.context.new_workunit(name=target.address.spec):
modules = []
if isinstance(target, PythonBinary):
source = 'entry_point {}'.format(target.entry_point)
components = target.entry_point.rsplit(':', 1)
module = components[0]
if len(components) == 2:
function = components[1]
data = TemplateData(source=source,
import_statement='from {} import {}'.format(module, function))
else:
data = TemplateData(source=source, import_statement='import {}'.format(module))
modules.append(data)
else:
for path in target.sources_relative_to_source_root():
if path.endswith('.py'):
if os.path.basename(path) == '__init__.py':
module_path = os.path.dirname(path)
else:
module_path, _ = os.path.splitext(path)
source = 'file {}'.format(os.path.join(target.target_base, path))
module = module_path.replace(os.path.sep, '.')
data = TemplateData(source=source, import_statement='import {}'.format(module))
modules.append(data)
if not modules:
# Nothing to eval, so a trivial compile success.
return 0
interpreter = self.select_interpreter_for_targets([target])
if isinstance(target, PythonBinary):
pexinfo, platforms = target.pexinfo, target.platforms
else:
pexinfo, platforms = None, None
with temporary_file() as imports_file:
def pre_freeze(chroot):
generator = Generator(pkgutil.get_data(__name__, self._EVAL_TEMPLATE_PATH),
chroot=chroot.path(),
modules=modules)
generator.write(imports_file)
imports_file.close()
chroot.builder.set_executable(imports_file.name, '__pants_python_eval__.py')
with self.temporary_chroot(interpreter=interpreter, pex_info=pexinfo,
targets=[target], platforms=platforms,
pre_freeze=pre_freeze) as chroot:
pex = PEX(chroot.builder.path(), interpreter=interpreter)
with self.context.new_workunit(name='eval',
labels=[WorkUnit.COMPILER, WorkUnit.RUN, WorkUnit.TOOL],
cmd=' '.join(pex.cmdline())) as workunit:
returncode = pex.run(stdout=workunit.output('stdout'), stderr=workunit.output('stderr'))
workunit.set_outcome(WorkUnit.SUCCESS if returncode == 0 else WorkUnit.FAILURE)
if returncode != 0:
self.context.log.error('Failed to eval {}'.format(target.address.spec))
return returncode
class Isort(object):
class Factory(Subsystem):
options_scope = 'isort'
@classmethod
def register_options(cls, register):
super(IsortPrep.Isort.Factory, cls).register_options(register)
register('--version',
default='4.3.4',
advanced=True,
fingerprint=True,
help='The version of isort to use.')
register(
'--additional-requirements',
default=['setuptools'],
type=list_option,
advanced=True,
fingerprint=True,
help=
'Additional undeclared dependencies of the requested isort version.'
)
@classmethod
def create_requirements(cls, context, workdir):
options = cls.global_instance().get_options()
address = Address(spec_path=fast_relpath(
workdir, get_buildroot()),
target_name='isort')
requirements = ['isort=={}'.format(options.version)
] + options.additional_requirements
context.build_graph.inject_synthetic_target(
address=address,
target_type=PythonRequirementLibrary,
requirements=[PythonRequirement(r) for r in requirements])
return context.build_graph.get_target(address=address)
@classmethod
def build_isort_pex(cls, context, interpreter, pex_path,
requirements_lib):
with safe_concurrent_creation(pex_path) as chroot:
pex_builder = PexBuilderWrapper(
PEXBuilder(path=chroot, interpreter=interpreter),
PythonRepos.global_instance(),
PythonSetup.global_instance(), context.log)
pex_builder.add_requirement_libs_from(
req_libs=[requirements_lib])
pex_builder.set_script('isort')
pex_builder.freeze()
def __init__(self, pex_path, interpreter=None):
self._pex = PEX(pex_path, interpreter=interpreter)
def run(self, workunit_factory, args, **kwargs):
cmdline = ' '.join(self._pex.cmdline(args))
with workunit_factory(cmd=cmdline) as workunit:
exit_code = self._pex.run(args,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'),
with_chroot=False,
blocking=True,
**kwargs)
return cmdline, exit_code
