class PythonToolInstance(object):
def __init__(self, pex_path, interpreter):
self._pex = PEX(pex_path, interpreter=interpreter)
self._interpreter = interpreter
@property
def pex(self):
return self._pex
@property
def interpreter(self):
return self._interpreter
def _pretty_cmdline(self, args):
return safe_shlex_join(self._pex.cmdline(args))
def output(self, args, stdin_payload=None, binary_mode=False, **kwargs):
process = self._pex.run(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
with_chroot=False,
blocking=False,
**kwargs)
if stdin_payload is not None:
stdin_payload = ensure_binary(stdin_payload)
(stdout, stderr) = process.communicate(input=stdin_payload)
if not binary_mode:
stdout = stdout.decode('utf-8')
stderr = stderr.decode('utf-8')
return (stdout, stderr, process.returncode, self._pretty_cmdline(args))
@contextmanager
def run_with(self, workunit_factory, args, **kwargs):
cmdline = self._pretty_cmdline(args)
with workunit_factory(cmd=cmdline) as workunit:
exit_code = self._pex.run(args,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'),
with_chroot=False,
blocking=True,
**kwargs)
yield cmdline, exit_code, workunit
def run(self, *args, **kwargs):
with self.run_with(*args, **kwargs) as (cmdline, exit_code, _):
return cmdline, exit_code
def _compile_target(self, vt):
"""'Compiles' a python target.
'Compiling' means forming an isolated chroot of its sources and transitive deps and then
attempting to import each of the target's sources in the case of a python library or else the
entry point in the case of a python binary.
For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like:
if __name__ == '__main__':
import lib.core
import lib.util
For a binary with entry point lib.bin:main the "compiler" main file would look like:
if __name__ == '__main__':
from lib.bin import main
In either case the main file is executed within the target chroot to reveal missing BUILD
dependencies.
"""
target = vt.target
with self.context.new_workunit(name=target.address.spec):
modules = self._get_modules(target)
if not modules:
# Nothing to eval, so a trivial compile success.
return 0
interpreter = self._get_interpreter_for_target_closure(target)
reqs_pex = self._resolve_requirements_for_versioned_target_closure(interpreter, vt)
srcs_pex = self._source_pex_for_versioned_target_closure(interpreter, vt)
# Create the executable pex.
exec_pex_parent = os.path.join(self.workdir, 'executable_pex')
executable_file_content = self._get_executable_file_content(exec_pex_parent, modules)
hasher = hashlib.sha1()
hasher.update(reqs_pex.path().encode('utf-8'))
hasher.update(srcs_pex.path().encode('utf-8'))
hasher.update(executable_file_content.encode('utf-8'))
exec_file_hash = hasher.hexdigest()
exec_pex_path = os.path.realpath(os.path.join(exec_pex_parent, exec_file_hash))
if not os.path.isdir(exec_pex_path):
with safe_concurrent_creation(exec_pex_path) as safe_path:
# Write the entry point.
safe_mkdir(safe_path)
with open(os.path.join(safe_path, '{}.py'.format(self._EXEC_NAME)), 'w') as outfile:
outfile.write(executable_file_content)
pex_info = (target.pexinfo if isinstance(target, PythonBinary) else None) or PexInfo()
# Override any user-specified entry point, under the assumption that the
# executable_file_content does what the user intends (including, probably, calling that
# underlying entry point).
pex_info.entry_point = self._EXEC_NAME
pex_info.pex_path = ':'.join(pex.path() for pex in (reqs_pex, srcs_pex) if pex)
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
builder.freeze()
pex = PEX(exec_pex_path, interpreter)
with self.context.new_workunit(name='eval',
labels=[WorkUnitLabel.COMPILER, WorkUnitLabel.RUN,
WorkUnitLabel.TOOL],
cmd=' '.join(pex.cmdline())) as workunit:
returncode = pex.run(stdout=workunit.output('stdout'), stderr=workunit.output('stderr'))
workunit.set_outcome(WorkUnit.SUCCESS if returncode == 0 else WorkUnit.FAILURE)
if returncode != 0:
self.context.log.error('Failed to eval {}'.format(target.address.spec))
return returncode
def _compile_target(self, target):
# "Compiles" a target by forming an isolated chroot of its sources and transitive deps and then
# attempting to import each of the target's sources in the case of a python library or else the
# entry point in the case of a python binary.
#
# For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like:
#
# if __name__ == '__main__':
# import lib.core
# import lib.util
#
# For a binary with entry point lib.bin:main the "compiler" main file would look like:
#
# if __name__ == '__main__':
# from lib.bin import main
#
# In either case the main file is executed within the target chroot to reveal missing BUILD
# dependencies.
with self.context.new_workunit(name=target.address.spec):
modules = []
if isinstance(target, PythonBinary):
source = 'entry_point {}'.format(target.entry_point)
components = target.entry_point.rsplit(':', 1)
module = components[0]
if len(components) == 2:
function = components[1]
data = TemplateData(source=source,
import_statement='from {} import {}'.format(module, function))
else:
data = TemplateData(source=source, import_statement='import {}'.format(module))
modules.append(data)
else:
for path in target.sources_relative_to_source_root():
if path.endswith('.py'):
if os.path.basename(path) == '__init__.py':
module_path = os.path.dirname(path)
else:
module_path, _ = os.path.splitext(path)
source = 'file {}'.format(os.path.join(target.target_base, path))
module = module_path.replace(os.path.sep, '.')
data = TemplateData(source=source, import_statement='import {}'.format(module))
modules.append(data)
if not modules:
# Nothing to eval, so a trivial compile success.
return 0
interpreter = self.select_interpreter_for_targets([target])
if isinstance(target, PythonBinary):
pexinfo, platforms = target.pexinfo, target.platforms
else:
pexinfo, platforms = None, None
with temporary_file() as imports_file:
def pre_freeze(chroot):
generator = Generator(pkgutil.get_data(__name__, self._EVAL_TEMPLATE_PATH),
chroot=chroot.path(),
modules=modules)
generator.write(imports_file)
imports_file.close()
chroot.builder.set_executable(imports_file.name, '__pants_python_eval__.py')
with self.temporary_chroot(interpreter=interpreter, pex_info=pexinfo,
targets=[target], platforms=platforms,
pre_freeze=pre_freeze) as chroot:
pex = PEX(chroot.builder.path(), interpreter=interpreter)
with self.context.new_workunit(name='eval',
labels=[WorkUnit.COMPILER, WorkUnit.RUN, WorkUnit.TOOL],
cmd=' '.join(pex.cmdline())) as workunit:
returncode = pex.run(stdout=workunit.output('stdout'), stderr=workunit.output('stderr'))
workunit.set_outcome(WorkUnit.SUCCESS if returncode == 0 else WorkUnit.FAILURE)
if returncode != 0:
self.context.log.error('Failed to eval {}'.format(target.address.spec))
return returncode
def _compile_target(self, target):
# "Compiles" a target by forming an isolated chroot of its sources and transitive deps and then
# attempting to import each of the target's sources in the case of a python library or else the
# entry point in the case of a python binary.
#
# For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like:
#
# if __name__ == '__main__':
# import lib.core
# import lib.util
#
# For a binary with entry point lib.bin:main the "compiler" main file would look like:
#
# if __name__ == '__main__':
# from lib.bin import main
#
# In either case the main file is executed within the target chroot to reveal missing BUILD
# dependencies.
with self.context.new_workunit(name=target.address.spec):
modules = []
if isinstance(target, PythonBinary):
source = 'entry_point {}'.format(target.entry_point)
components = target.entry_point.rsplit(':', 1)
module = components[0]
if len(components) == 2:
function = components[1]
data = TemplateData(
source=source,
import_statement='from {} import {}'.format(
module, function))
else:
data = TemplateData(
source=source,
import_statement='import {}'.format(module))
modules.append(data)
else:
for path in target.sources_relative_to_source_root():
if path.endswith('.py'):
if os.path.basename(path) == '__init__.py':
module_path = os.path.dirname(path)
else:
module_path, _ = os.path.splitext(path)
source = 'file {}'.format(
os.path.join(target.target_base, path))
module = module_path.replace(os.path.sep, '.')
data = TemplateData(
source=source,
import_statement='import {}'.format(module))
modules.append(data)
if not modules:
# Nothing to eval, so a trivial compile success.
return 0
interpreter = self.select_interpreter_for_targets([target])
if isinstance(target, PythonBinary):
pexinfo, platforms = target.pexinfo, target.platforms
else:
pexinfo, platforms = None, None
with self.temporary_pex_builder(interpreter=interpreter,
pex_info=pexinfo) as builder:
with self.context.new_workunit(name='resolve'):
chroot = PythonChroot(context=self.context,
targets=[target],
builder=builder,
platforms=platforms,
interpreter=interpreter)
chroot.dump()
with temporary_file() as imports_file:
generator = Generator(pkgutil.get_data(
__name__, self._EVAL_TEMPLATE_PATH),
chroot=chroot.path(),
modules=modules)
generator.write(imports_file)
imports_file.close()
builder.set_executable(imports_file.name,
'__pants_python_eval__.py')
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
with self.context.new_workunit(
name='eval',
labels=[
WorkUnit.COMPILER, WorkUnit.RUN, WorkUnit.TOOL
],
cmd=' '.join(pex.cmdline())) as workunit:
returncode = pex.run(stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'))
workunit.set_outcome(WorkUnit.SUCCESS if returncode ==
0 else WorkUnit.FAILURE)
if returncode != 0:
self.context.log.error('Failed to eval {}'.format(
target.address.spec))
return returncode
def _compile_target(self, vt):
"""'Compiles' a python target.
'Compiling' means forming an isolated chroot of its sources and transitive deps and then
attempting to import each of the target's sources in the case of a python library or else the
entry point in the case of a python binary.
For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like:
if __name__ == '__main__':
import lib.core
import lib.util
For a binary with entry point lib.bin:main the "compiler" main file would look like:
if __name__ == '__main__':
from lib.bin import main
In either case the main file is executed within the target chroot to reveal missing BUILD
dependencies.
"""
target = vt.target
with self.context.new_workunit(name=target.address.spec):
modules = self._get_modules(target)
if not modules:
# Nothing to eval, so a trivial compile success.
return 0
interpreter = self._get_interpreter_for_target_closure(target)
reqs_pex = self._resolve_requirements_for_versioned_target_closure(
interpreter, vt)
srcs_pex = self._source_pex_for_versioned_target_closure(
interpreter, vt)
# Create the executable pex.
exec_pex_parent = os.path.join(self.workdir, 'executable_pex')
executable_file_content = self._get_executable_file_content(
exec_pex_parent, modules)
hasher = hashlib.sha1()
hasher.update(executable_file_content)
exec_file_hash = hasher.hexdigest()
exec_pex_path = os.path.realpath(
os.path.join(exec_pex_parent, exec_file_hash))
if not os.path.isdir(exec_pex_path):
with safe_concurrent_creation(exec_pex_path) as safe_path:
# Write the entry point.
safe_mkdir(safe_path)
with open(
os.path.join(safe_path,
'{}.py'.format(self._EXEC_NAME)),
'w') as outfile:
outfile.write(executable_file_content)
pex_info = (target.pexinfo if isinstance(
target, PythonBinary) else None) or PexInfo()
# Override any user-specified entry point, under the assumption that the
# executable_file_content does what the user intends (including, probably, calling that
# underlying entry point).
pex_info.entry_point = self._EXEC_NAME
builder = PEXBuilder(safe_path,
interpreter,
pex_info=pex_info)
builder.freeze()
exec_pex = PEX(exec_pex_path, interpreter)
extra_pex_paths = [
pex.path() for pex in filter(None, [reqs_pex, srcs_pex])
]
pex = WrappedPEX(exec_pex, interpreter, extra_pex_paths)
with self.context.new_workunit(
name='eval',
labels=[
WorkUnitLabel.COMPILER, WorkUnitLabel.RUN,
WorkUnitLabel.TOOL
],
cmd=' '.join(exec_pex.cmdline())) as workunit:
returncode = pex.run(stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'))
workunit.set_outcome(WorkUnit.SUCCESS if returncode ==
0 else WorkUnit.FAILURE)
if returncode != 0:
self.context.log.error('Failed to eval {}'.format(
target.address.spec))
return returncode
def _spawn_pip_isolated(
self,
args, # type: Iterable[str]
package_index_configuration=None, # type: Optional[PackageIndexConfiguration]
cache=None, # type: Optional[str]
interpreter=None, # type: Optional[PythonInterpreter]
):
# type: (...) -> Job
pip_args = [
# We vendor the version of pip we want so pip should never check for updates.
"--disable-pip-version-check",
# If we want to warn about a version of python we support, we should do it, not pip.
"--no-python-version-warning",
# If pip encounters a duplicate file path during its operations we don't want it to
# prompt and we'd also like to know about this since it should never occur. We leverage
# the pip global option:
# --exists-action <action>
# Default action when a path already exists: (s)witch, (i)gnore, (w)ipe, (b)ackup,
# (a)bort.
"--exists-action",
"a",
]
resolver_version = (package_index_configuration.resolver_version
if package_index_configuration else
ResolverVersion.PIP_LEGACY)
pip_args.extend(resolver_version.pip_args)
if not package_index_configuration or package_index_configuration.isolated:
# Don't read PIP_ environment variables or pip configuration files like
# `~/.config/pip/pip.conf`.
pip_args.append("--isolated")
# The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor
# of 3.
pip_verbosity = ENV.PEX_VERBOSE // 3
if pip_verbosity > 0:
pip_args.append("-{}".format("v" * pip_verbosity))
else:
pip_args.append("-q")
if cache:
pip_args.extend(["--cache-dir", cache])
else:
pip_args.append("--no-cache-dir")
command = pip_args + list(args)
# N.B.: Package index options in Pep always have the same option names, but they are
# registered as subcommand-specific, so we must append them here _after_ the pip subcommand
# specified in `args`.
if package_index_configuration:
command.extend(package_index_configuration.args)
env = package_index_configuration.env if package_index_configuration else {}
with ENV.strip().patch(PEX_ROOT=cache or ENV.PEX_ROOT,
PEX_VERBOSE=str(ENV.PEX_VERBOSE),
**env) as env:
# Guard against API calls from environment with ambient PYTHONPATH preventing pip PEX
# bootstrapping. See: https://github.com/pantsbuild/pex/issues/892
pythonpath = env.pop("PYTHONPATH", None)
if pythonpath:
TRACER.log(
"Scrubbed PYTHONPATH={} from the pip PEX environment.".
format(pythonpath),
V=3)
from pex.pex import PEX
pip = PEX(pex=self._pip_pex_path, interpreter=interpreter)
return Job(command=pip.cmdline(command),
process=pip.run(args=command, env=env, blocking=False))
class Isort(object):
class Factory(Subsystem):
options_scope = 'isort'
@classmethod
def register_options(cls, register):
super(IsortPrep.Isort.Factory, cls).register_options(register)
register('--version',
default='4.3.4',
advanced=True,
fingerprint=True,
help='The version of isort to use.')
register(
'--additional-requirements',
default=['setuptools'],
type=list_option,
advanced=True,
fingerprint=True,
help=
'Additional undeclared dependencies of the requested isort version.'
)
@classmethod
def create_requirements(cls, context, workdir):
options = cls.global_instance().get_options()
address = Address(spec_path=fast_relpath(
workdir, get_buildroot()),
target_name='isort')
requirements = ['isort=={}'.format(options.version)
] + options.additional_requirements
context.build_graph.inject_synthetic_target(
address=address,
target_type=PythonRequirementLibrary,
requirements=[PythonRequirement(r) for r in requirements])
return context.build_graph.get_target(address=address)
@classmethod
def build_isort_pex(cls, context, interpreter, pex_path,
requirements_lib):
with safe_concurrent_creation(pex_path) as chroot:
builder = PEXBuilder(path=chroot, interpreter=interpreter)
dump_requirement_libs(builder=builder,
interpreter=interpreter,
req_libs=[requirements_lib],
log=context.log)
builder.set_script('isort')
builder.freeze()
def __init__(self, pex_path, interpreter=None):
self._pex = PEX(pex_path, interpreter=interpreter)
def run(self, workunit_factory, args, **kwargs):
cmdline = ' '.join(self._pex.cmdline(args))
with workunit_factory(cmd=cmdline) as workunit:
exit_code = self._pex.run(args,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'),
with_chroot=False,
blocking=True,
**kwargs)
return cmdline, exit_code
def _spawn_pip_isolated(
self,
args, # type: Iterable[str]
package_index_configuration=None, # type: Optional[PackageIndexConfiguration]
cache=None, # type: Optional[str]
interpreter=None, # type: Optional[PythonInterpreter]
pip_verbosity=0, # type: int
**popen_kwargs # type: Any
):
# type: (...) -> Tuple[List[str], subprocess.Popen]
pip_args = [
# We vendor the version of pip we want so pip should never check for updates.
"--disable-pip-version-check",
# If we want to warn about a version of python we support, we should do it, not pip.
"--no-python-version-warning",
# If pip encounters a duplicate file path during its operations we don't want it to
# prompt and we'd also like to know about this since it should never occur. We leverage
# the pip global option:
# --exists-action <action>
# Default action when a path already exists: (s)witch, (i)gnore, (w)ipe, (b)ackup,
# (a)bort.
"--exists-action",
"a",
]
python_interpreter = interpreter or PythonInterpreter.get()
pip_args.extend(
self._calculate_resolver_version_args(
python_interpreter, package_index_configuration=package_index_configuration
)
)
if not package_index_configuration or package_index_configuration.isolated:
# Don't read PIP_ environment variables or pip configuration files like
# `~/.config/pip/pip.conf`.
pip_args.append("--isolated")
# The max pip verbosity is -vvv and for pex it's -vvvvvvvvv; so we scale down by a factor
# of 3.
pip_verbosity = pip_verbosity or (ENV.PEX_VERBOSE // 3)
if pip_verbosity > 0:
pip_args.append("-{}".format("v" * pip_verbosity))
else:
pip_args.append("-q")
if cache:
pip_args.extend(["--cache-dir", cache])
else:
pip_args.append("--no-cache-dir")
command = pip_args + list(args)
# N.B.: Package index options in Pep always have the same option names, but they are
# registered as subcommand-specific, so we must append them here _after_ the pip subcommand
# specified in `args`.
if package_index_configuration:
command.extend(package_index_configuration.args)
env = package_index_configuration.env if package_index_configuration else {}
with ENV.strip().patch(
PEX_ROOT=cache or ENV.PEX_ROOT, PEX_VERBOSE=str(ENV.PEX_VERBOSE), **env
) as env:
# Guard against API calls from environment with ambient PYTHONPATH preventing pip PEX
# bootstrapping. See: https://github.com/pantsbuild/pex/issues/892
pythonpath = env.pop("PYTHONPATH", None)
if pythonpath:
TRACER.log(
"Scrubbed PYTHONPATH={} from the pip PEX environment.".format(pythonpath), V=3
)
from pex.pex import PEX
pip = PEX(pex=self._pip_pex_path, interpreter=python_interpreter)
# Pip has no discernable stdout / stderr discipline with its logging. Pex guarantees
# stdout will only contain useable (parseable) data and all logging will go to stderr.
# To uphold the Pex standard, force Pip to comply by re-directing stdout to stderr.
#
# See:
# + https://github.com/pantsbuild/pex/issues/1267
# + https://github.com/pypa/pip/issues/9420
stdout = popen_kwargs.pop("stdout", sys.stderr.fileno())
return pip.cmdline(command), pip.run(
args=command, env=env, blocking=False, stdout=stdout, **popen_kwargs
)
