def _parse_libraries_from_compiler_search_dirs(self, compiler_exe, env):
# This argument is supported by at least gcc and clang.
cmd = [compiler_exe, '-print-search-dirs']
try:
# Get stderr interspersed in the error message too -- this should not affect output parsing.
compiler_output = subprocess.check_output(cmd,
env=env,
stderr=subprocess.STDOUT)
except OSError as e:
# We use `safe_shlex_join` here to pretty-print the command.
raise self.ParseSearchDirsError(
"Invocation of '{}' with argv '{}' failed.".format(
compiler_exe, safe_shlex_join(cmd)), e)
except subprocess.CalledProcessError as e:
raise self.ParseSearchDirsError(
"Invocation of '{}' with argv '{}' exited with non-zero code {}. output:\n{}"
.format(compiler_exe, safe_shlex_join(cmd), e.returncode,
e.output), e)
libs_line = self._search_dirs_libraries_regex.search(compiler_output)
if not libs_line:
raise self.ParseSearchDirsError(
"Could not parse libraries from output of {!r}:\n{}".format(
safe_shlex_join(cmd), compiler_output))
return libs_line.group(1).split(':')
def as_environment(self):
ret = {}
if self.setup_requires_site_dir:
ret['PYTHONPATH'] = self.setup_requires_site_dir.site_dir
# FIXME(#5951): the below is a lot of error-prone repeated logic -- we need a way to compose
# executables more hygienically. We should probably be composing each datatype's members, and
# only creating an environment at the very end.
native_tools = self.setup_py_native_tools
if native_tools:
# TODO: an as_tuple() method for datatypes would make this destructuring cleaner!
plat = native_tools.platform
cc = native_tools.c_compiler
cxx = native_tools.cpp_compiler
linker = native_tools.linker
all_path_entries = cc.path_entries + cxx.path_entries + linker.path_entries
ret['PATH'] = create_path_env_var(all_path_entries)
all_library_dirs = cc.library_dirs + cxx.library_dirs + linker.library_dirs
if all_library_dirs:
joined_library_dirs = create_path_env_var(all_library_dirs)
ret['LIBRARY_PATH'] = joined_library_dirs
dynamic_lib_env_var = plat.resolve_platform_specific({
'darwin':
lambda: 'DYLD_LIBRARY_PATH',
'linux':
lambda: 'LD_LIBRARY_PATH',
})
ret[dynamic_lib_env_var] = joined_library_dirs
all_include_dirs = cc.include_dirs + cxx.include_dirs
if all_include_dirs:
ret['CPATH'] = create_path_env_var(all_include_dirs)
all_cflags_for_platform = plat.resolve_platform_specific({
'darwin':
lambda: ['-mmacosx-version-min=10.11'],
'linux':
lambda: [],
})
if all_cflags_for_platform:
ret['CFLAGS'] = safe_shlex_join(all_cflags_for_platform)
ret['CC'] = cc.exe_filename
ret['CXX'] = cxx.exe_filename
ret['LDSHARED'] = linker.exe_filename
all_new_ldflags = plat.resolve_platform_specific(
self._SHARED_CMDLINE_ARGS)
ret['LDFLAGS'] = safe_shlex_join(all_new_ldflags)
return ret
def _create_dist(self, dist_tgt, dist_target_dir, setup_requires_pex,
snapshot_fingerprint, is_platform_specific):
"""Create a .whl file for the specified python_distribution target."""
self._copy_sources(dist_tgt, dist_target_dir)
setup_py_snapshot_version_argv = self._generate_snapshot_bdist_wheel_argv(
snapshot_fingerprint, is_platform_specific)
cmd = safe_shlex_join(
setup_requires_pex.cmdline(setup_py_snapshot_version_argv))
with self.context.new_workunit('setup.py',
cmd=cmd,
labels=[WorkUnitLabel.TOOL
]) as workunit:
with pushd(dist_target_dir):
result = setup_requires_pex.run(
args=setup_py_snapshot_version_argv,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'))
if result != 0:
raise self.BuildLocalPythonDistributionsError(
"Installation of python distribution from target {target} into directory {into_dir} "
"failed (return value of run() was: {rc!r}).\n"
"The pex with any requirements is located at: {interpreter}.\n"
"The host system's compiler and linker were used.\n"
"The setup command was: {command}.".format(
target=dist_tgt,
into_dir=dist_target_dir,
rc=result,
interpreter=setup_requires_pex.path(),
command=setup_py_snapshot_version_argv))
def _process_session(self):
"""Process the outputs of the nailgun session.
:raises: :class:`NailgunProtocol.ProcessStreamTimeout` if a timeout set from a signal handler
with .set_exit_timeout() completes.
:raises: :class:`Exception` if the session completes before the timeout, the `reason` argument
to .set_exit_timeout() will be raised."""
try:
for chunk_type, payload in self.iter_chunks(
MaybeShutdownSocket(self._sock),
return_bytes=True,
timeout_object=self,
):
# TODO(#6579): assert that we have at this point received all the chunk types in
# ChunkType.REQUEST_TYPES, then require PID and PGRP (exactly once?), and then allow any of
# ChunkType.EXECUTION_TYPES.
if chunk_type == ChunkType.STDOUT:
self._write_flush(self._stdout, payload)
elif chunk_type == ChunkType.STDERR:
self._write_flush(self._stderr, payload)
elif chunk_type == ChunkType.EXIT:
self._write_flush(self._stdout)
self._write_flush(self._stderr)
return int(payload)
elif chunk_type == ChunkType.PID:
self.remote_pid = int(payload)
self.remote_process_cmdline = psutil.Process(self.remote_pid).cmdline()
if self._remote_pid_callback:
self._remote_pid_callback(self.remote_pid)
elif chunk_type == ChunkType.PGRP:
self.remote_pgrp = int(payload)
if self._remote_pgrp_callback:
self._remote_pgrp_callback(self.remote_pgrp)
elif chunk_type == ChunkType.START_READING_INPUT:
self._maybe_start_input_writer()
else:
raise self.ProtocolError('received unexpected chunk {} -> {}'.format(chunk_type, payload))
except NailgunProtocol.ProcessStreamTimeout as e:
assert(self.remote_pid is not None)
# NB: We overwrite the process title in the pantsd process, which causes it to have an
# argv with lots of empty spaces for some reason. We filter those out and pretty-print the
# rest here.
filtered_remote_cmdline = safe_shlex_join(
arg for arg in self.remote_process_cmdline if arg != '')
logger.warning(
"timed out when attempting to gracefully shut down the remote client executing \"{}\". "
"sending SIGKILL to the remote client at pid: {}. message: {}"
.format(filtered_remote_cmdline, self.remote_pid, e))
finally:
# Bad chunk types received from the server can throw NailgunProtocol.ProtocolError in
# NailgunProtocol.iter_chunks(). This ensures the NailgunStreamWriter is always stopped.
self._maybe_stop_input_writer()
# If an asynchronous error was set at any point (such as in a signal handler), we want to make
# sure we clean up the remote process before exiting with error.
if self._exit_reason:
if self.remote_pgrp:
safe_kill(self.remote_pgrp, signal.SIGKILL)
if self.remote_pid:
safe_kill(self.remote_pid, signal.SIGKILL)
raise self._exit_reason
def _process_session(self):
"""Process the outputs of the nailgun session.
:raises: :class:`NailgunProtocol.ProcessStreamTimeout` if a timeout set from a signal handler
with .set_exit_timeout() completes.
:raises: :class:`Exception` if the session completes before the timeout, the `reason` argument
to .set_exit_timeout() will be raised."""
try:
for chunk_type, payload in self.iter_chunks(
MaybeShutdownSocket(self._sock),
return_bytes=True,
timeout_object=self,
):
# TODO(#6579): assert that we have at this point received all the chunk types in
# ChunkType.REQUEST_TYPES, then require PID and PGRP (exactly once?), and then allow any of
# ChunkType.EXECUTION_TYPES.
if chunk_type == ChunkType.STDOUT:
self._write_flush(self._stdout, payload)
elif chunk_type == ChunkType.STDERR:
self._write_flush(self._stderr, payload)
elif chunk_type == ChunkType.EXIT:
self._write_flush(self._stdout)
self._write_flush(self._stderr)
return int(payload)
elif chunk_type == ChunkType.PID:
self.remote_pid = int(payload)
self.remote_process_cmdline = psutil.Process(self.remote_pid).cmdline()
if self._remote_pid_callback:
self._remote_pid_callback(self.remote_pid)
elif chunk_type == ChunkType.PGRP:
self.remote_pgrp = int(payload)
if self._remote_pgrp_callback:
self._remote_pgrp_callback(self.remote_pgrp)
elif chunk_type == ChunkType.START_READING_INPUT:
self._maybe_start_input_writer()
else:
raise self.ProtocolError('received unexpected chunk {} -> {}'.format(chunk_type, payload))
except NailgunProtocol.ProcessStreamTimeout as e:
assert(self.remote_pid is not None)
# NB: We overwrite the process title in the pantsd process, which causes it to have an
# argv with lots of empty spaces for some reason. We filter those out and pretty-print the
# rest here.
filtered_remote_cmdline = safe_shlex_join(
arg for arg in self.remote_process_cmdline if arg != '')
logger.warning(
"timed out when attempting to gracefully shut down the remote client executing \"{}\". "
"sending SIGKILL to the remote client at pid: {}. message: {}"
.format(filtered_remote_cmdline, self.remote_pid, e))
finally:
# Bad chunk types received from the server can throw NailgunProtocol.ProtocolError in
# NailgunProtocol.iter_chunks(). This ensures the NailgunStreamWriter is always stopped.
self._maybe_stop_input_writer()
# If an asynchronous error was set at any point (such as in a signal handler), we want to make
# sure we clean up the remote process before exiting with error.
if self._exit_reason:
if self.remote_pgrp:
safe_kill(self.remote_pgrp, signal.SIGKILL)
if self.remote_pid:
safe_kill(self.remote_pid, signal.SIGKILL)
raise self._exit_reason
def _create_dist(self, dist_tgt, dist_target_dir, setup_py_runner,
snapshot_fingerprint, is_platform_specific):
"""Create a .whl file for the specified python_distribution target."""
self._copy_sources(dist_tgt, dist_target_dir)
setup_py_snapshot_version_argv = self._generate_snapshot_bdist_wheel_argv(
snapshot_fingerprint, is_platform_specific)
cmd = safe_shlex_join(
setup_py_runner.cmdline(setup_py_snapshot_version_argv))
with self.context.new_workunit("setup.py",
cmd=cmd,
labels=[WorkUnitLabel.TOOL
]) as workunit:
try:
setup_py_runner.run_setup_command(
source_dir=Path(dist_target_dir),
setup_command=setup_py_snapshot_version_argv,
stdout=workunit.output("stdout"),
stderr=workunit.output("stderr"),
)
except SetupPyRunner.CommandFailure as e:
raise self.BuildLocalPythonDistributionsError(
f"Installation of python distribution from target {dist_tgt} into directory "
f"{dist_target_dir} failed using the host system's compiler and linker: {e}"
)
def _run_zef_command(self, workunit_factory, argv):
subproc_env = os.environ.copy()
subproc_env['PATH'] = create_path_env_var(self.path_entries,
subproc_env,
prepend=True)
all_argv = ['zef'] + argv
pretty_printed_argv = safe_shlex_join(all_argv)
try:
if workunit_factory:
with workunit_factory(cmd=pretty_printed_argv) as workunit:
return subprocess.check_call(
all_argv,
env=subproc_env,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'))
else:
output = subprocess.check_output(all_argv, env=subproc_env)
logger.debug(
"output from running zef command {!r} with env {!r}:\n{}".
format(all_argv, subproc_env, output))
except (OSError, subprocess.CalledProcessError) as e:
raise self.ZefException("Error with zef command '{}': {}".format(
pretty_printed_argv, e),
e,
exit_code=e.returncode)
def run_tests(self, fail_fast, test_targets, args_by_target):
self.context.log.debug('test_targets: {}'.format(test_targets))
with self.chroot(test_targets, self._maybe_workdir) as chroot:
cmdline_args = self._build_and_test_flags + [
args_by_target[t].import_path for t in test_targets
] + self.get_passthru_args()
gopath = create_path_env_var(args_by_target[t].gopath
for t in test_targets)
go_cmd = self.go_dist.create_go_cmd('test',
gopath=gopath,
args=cmdline_args)
self.context.log.debug('go_cmd: {}'.format(go_cmd))
workunit_labels = [WorkUnitLabel.TOOL, WorkUnitLabel.TEST]
with self.context.new_workunit(name='go test',
cmd=safe_shlex_join(go_cmd.cmdline),
labels=workunit_labels) as workunit:
exit_code = self.spawn_and_wait(test_targets,
workunit=workunit,
go_cmd=go_cmd,
cwd=chroot)
return TestResult.rc(exit_code)
def _create_dist(self,
dist_tgt,
dist_target_dir,
setup_requires_pex,
snapshot_fingerprint,
is_platform_specific):
"""Create a .whl file for the specified python_distribution target."""
self._copy_sources(dist_tgt, dist_target_dir)
setup_py_snapshot_version_argv = self._generate_snapshot_bdist_wheel_argv(
snapshot_fingerprint, is_platform_specific)
cmd = safe_shlex_join(setup_requires_pex.cmdline(setup_py_snapshot_version_argv))
with self.context.new_workunit('setup.py', cmd=cmd, labels=[WorkUnitLabel.TOOL]) as workunit:
with pushd(dist_target_dir):
result = setup_requires_pex.run(args=setup_py_snapshot_version_argv,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'))
if result != 0:
raise self.BuildLocalPythonDistributionsError(
"Installation of python distribution from target {target} into directory {into_dir} "
"failed (return value of run() was: {rc!r}).\n"
"The pex with any requirements is located at: {interpreter}.\n"
"The host system's compiler and linker were used.\n"
"The setup command was: {command}."
.format(target=dist_tgt,
into_dir=dist_target_dir,
rc=result,
interpreter=setup_requires_pex.path(),
command=setup_py_snapshot_version_argv))
def _do_run_tests_with_args(self, pex, args):
try:
env = dict(os.environ)
# Ensure we don't leak source files or undeclared 3rdparty requirements into the pytest PEX
# environment.
pythonpath = env.pop('PYTHONPATH', None)
if pythonpath:
self.context.log.warn(
'scrubbed PYTHONPATH={} from pytest environment'.format(
pythonpath))
# But allow this back door for users who do want to force something onto the test pythonpath,
# e.g., modules required during a debugging session.
extra_pythonpath = self.get_options().extra_pythonpath
if extra_pythonpath:
env['PYTHONPATH'] = os.pathsep.join(extra_pythonpath)
# The pytest runner we use accepts a --pdb argument that will launch an interactive pdb
# session on any test failure. In order to support use of this pass-through flag we must
# turn off stdin buffering that otherwise occurs. Setting the PYTHONUNBUFFERED env var to
# any value achieves this in python2.7. We'll need a different solution when we support
# running pants under CPython 3 which does not unbuffer stdin using this trick.
env['PYTHONUNBUFFERED'] = '1'
# pytest uses py.io.terminalwriter for output. That class detects the terminal
# width and attempts to use all of it. However we capture and indent the console
# output, leading to weird-looking line wraps. So we trick the detection code
# into thinking the terminal window is narrower than it is.
env['COLUMNS'] = str(int(os.environ.get('COLUMNS', 80)) - 30)
profile = self.get_options().profile
if profile:
env['PEX_PROFILE_FILENAME'] = '{0}.subprocess.{1:.6f}'.format(
profile, time.time())
with self.context.new_workunit(
name='run',
cmd=safe_shlex_join(pex.cmdline(args)),
labels=[WorkUnitLabel.TOOL,
WorkUnitLabel.TEST]) as workunit:
# NB: Constrain the pex environment to ensure the use of the selected interpreter!
env.update(self._ensure_pytest_interpreter_search_path())
rc = self.spawn_and_wait(pex,
workunit=workunit,
args=args,
setsid=True,
env=env)
return PytestResult.rc(rc)
except ErrorWhileTesting:
# spawn_and_wait wraps the test runner in a timeout, so it could
# fail with a ErrorWhileTesting. We can't just set PythonTestResult
# to a failure because the resultslog doesn't have all the failures
# when tests are killed with a timeout. Therefore we need to re-raise
# here.
raise
except Exception:
self.context.log.error('Failed to run test!')
self.context.log.info(traceback.format_exc())
return PytestResult.exception()
def _invoke_compiler_exe(self, cmd, env):
try:
# Get stderr interspersed in the error message too -- this should not affect output parsing.
compiler_output = subprocess.check_output(
cmd, env=env, stderr=subprocess.STDOUT).decode()
except OSError as e:
# We use `safe_shlex_join` here to pretty-print the command.
raise self.ParseSearchDirsError(
"Process invocation with argv '{}' and environment {!r} failed."
.format(safe_shlex_join(cmd), env), e)
except subprocess.CalledProcessError as e:
raise self.ParseSearchDirsError(
"Process invocation with argv '{}' and environment {!r} exited with non-zero code {}. "
"output:\n{}".format(safe_shlex_join(cmd), env, e.returncode,
e.output), e)
return compiler_output
def _invoke_compiler_exe(self, cmd, env):
try:
# Get stderr interspersed in the error message too -- this should not affect output parsing.
compiler_output = subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT).decode('utf-8')
except OSError as e:
# We use `safe_shlex_join` here to pretty-print the command.
raise self.ParseSearchDirsError(
"Process invocation with argv '{}' and environment {!r} failed."
.format(safe_shlex_join(cmd), env),
e)
except subprocess.CalledProcessError as e:
raise self.ParseSearchDirsError(
"Process invocation with argv '{}' and environment {!r} exited with non-zero code {}. "
"output:\n{}"
.format(safe_shlex_join(cmd), env, e.returncode, e.output),
e)
return compiler_output
def _parse_libraries_from_compiler_search_dirs(self, compiler_exe, env):
# This argument is supported by at least gcc and clang.
cmd = [compiler_exe, '-print-search-dirs']
compiler_output = self._invoke_compiler_exe(cmd, env)
libs_line = self._search_dirs_libraries_regex.search(compiler_output)
if not libs_line:
raise self.ParseSearchDirsError(
"Could not parse libraries from output of {!r}:\n{}".format(
safe_shlex_join(cmd), compiler_output))
return libs_line.group(1).split(':')
def _parse_libraries_from_compiler_search_dirs(self, compiler_exe, env):
# This argument is supported by at least gcc and clang.
cmd = [compiler_exe, '-print-search-dirs']
compiler_output = self._invoke_compiler_exe(cmd, env)
libs_line = self._search_dirs_libraries_regex.search(compiler_output)
if not libs_line:
raise self.ParseSearchDirsError(
"Could not parse libraries from output of {!r}:\n{}"
.format(safe_shlex_join(cmd), compiler_output))
return libs_line.group(1).split(':')
def get_invocation_environment_dict(self, platform):
ret = super(CompilerMixin, self).get_invocation_environment_dict(platform).copy()
if self.include_dirs:
ret['CPATH'] = create_path_env_var(self.include_dirs)
all_cflags_for_platform = platform.resolve_platform_specific({
'darwin': lambda: ['-mmacosx-version-min=10.11'],
'linux': lambda: [],
})
ret['CFLAGS'] = safe_shlex_join(all_cflags_for_platform)
return ret
def run_setup_command(self, *, source_dir: Path,
setup_command: Iterable[str], **kwargs) -> None:
"""Runs the given setup.py command against the setup.py project in `source_dir`.
:raises: :class:`SetupPyRunner.CommandFailure` if there was a problem executing the command.
"""
with pushd(str(source_dir)):
result = self._requirements_pex.run(
args=self._create_python_args(setup_command), **kwargs)
if result != 0:
pex_command = safe_shlex_join(self.cmdline(setup_command))
raise self.CommandFailure(
f"Failed to execute {pex_command} using {self}")
def _run_rakudobrew_command(self, argv):
subproc_env = os.environ.copy()
subproc_env['PATH'] = create_path_env_var(self.path_entries,
subproc_env,
prepend=True)
all_argv = ['rakudobrew'] + argv
pretty_printed_argv = safe_shlex_join(all_argv)
try:
return subprocess.check_output(all_argv, env=subproc_env)
except (OSError, subprocess.CalledProcessError) as e:
raise self.RakudoBrewBootstrapError(
"Error with rakudobrew command '{}': {}".format(
pretty_printed_argv, e), e)
def _do_run_tests_with_args(self, pex, args):
try:
env = dict(os.environ)
# Ensure we don't leak source files or undeclared 3rdparty requirements into the py.test PEX
# environment.
pythonpath = env.pop('PYTHONPATH', None)
if pythonpath:
self.context.log.warn('scrubbed PYTHONPATH={} from py.test environment'.format(pythonpath))
# But allow this back door for users who do want to force something onto the test pythonpath,
# e.g., modules required during a debugging session.
extra_pythonpath = self.get_options().extra_pythonpath
if extra_pythonpath:
env['PYTHONPATH'] = os.pathsep.join(extra_pythonpath)
# The pytest runner we use accepts a --pdb argument that will launch an interactive pdb
# session on any test failure. In order to support use of this pass-through flag we must
# turn off stdin buffering that otherwise occurs. Setting the PYTHONUNBUFFERED env var to
# any value achieves this in python2.7. We'll need a different solution when we support
# running pants under CPython 3 which does not unbuffer stdin using this trick.
env['PYTHONUNBUFFERED'] = '1'
# pytest uses py.io.terminalwriter for output. That class detects the terminal
# width and attempts to use all of it. However we capture and indent the console
# output, leading to weird-looking line wraps. So we trick the detection code
# into thinking the terminal window is narrower than it is.
env['COLUMNS'] = str(int(os.environ.get('COLUMNS', 80)) - 30)
profile = self.get_options().profile
if profile:
env['PEX_PROFILE_FILENAME'] = '{0}.subprocess.{1:.6f}'.format(profile, time.time())
with self.context.new_workunit(name='run',
cmd=safe_shlex_join(pex.cmdline(args)),
labels=[WorkUnitLabel.TOOL, WorkUnitLabel.TEST]) as workunit:
# NB: Constrain the pex environment to ensure the use of the selected interpreter!
env.update(self._constrain_pytest_interpreter_search_path())
rc = self.spawn_and_wait(pex, workunit=workunit, args=args, setsid=True, env=env)
return PytestResult.rc(rc)
except ErrorWhileTesting:
# spawn_and_wait wraps the test runner in a timeout, so it could
# fail with a ErrorWhileTesting. We can't just set PythonTestResult
# to a failure because the resultslog doesn't have all the failures
# when tests are killed with a timeout. Therefore we need to re-raise
# here.
raise
except Exception:
self.context.log.error('Failed to run test!')
self.context.log.info(traceback.format_exc())
return PytestResult.exception()
def open(self, files: Iterable[PurePath]) -> None:
for request in self._iter_openers(files):
open_command = safe_shlex_join(request.argv)
try:
result = self.runner.run_process(request)
if result.exit_code != 0:
self.console.print_stderr(
f"Failed to open files for viewing using `{open_command}` - received exit "
f"code {result.exit_code}.")
except Exception as e:
self.console.print_stderr(
f"Failed to open files for viewing using "
f"`{open_command}`: {e}")
self.console.print_stderr(
f"Ensure {self.program} is installed on your `PATH` and "
f"re-run this goal.")
def _invoke_capturing_output(self, cmd, env=None):
env = env or {}
try:
with environment_as(**env):
return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('utf-8')
except subprocess.CalledProcessError as e:
raise Exception(
"Command failed while invoking the native toolchain "
"with code '{code}', cwd='{cwd}', cmd='{cmd}', env='{env}'. "
"Combined stdout and stderr:\n{out}"
.format(code=e.returncode,
cwd=os.getcwd(),
# safe_shlex_join() is just for pretty-printing.
cmd=safe_shlex_join(cmd),
env=env,
out=e.output),
e)
def _invoke_capturing_output(self, cmd, env=None):
if env is None:
env = os.environ.copy()
try:
with environment_as(**env):
return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
raise Exception(
"Command failed while invoking the native toolchain "
"with code '{code}', cwd='{cwd}', cmd='{cmd}', env='{env}'. "
"Combined stdout and stderr:\n{out}".format(
code=e.returncode,
cwd=os.getcwd(),
# safe_shlex_join() is just for pretty-printing.
cmd=safe_shlex_join(cmd),
env=env,
out=e.output),
e)
def get_invocation_environment_dict(self, platform):
ret = super(Linker, self).get_invocation_environment_dict(platform).copy()
# TODO: set all LDFLAGS in here or in further specializations of Linker instead of in individual
# tasks.
all_ldflags_for_platform = platform.resolve_platform_specific({
'darwin': lambda: ['-mmacosx-version-min=10.11'],
'linux': lambda: [],
})
ret.update({
'LDSHARED': self.exe_filename,
# FIXME: this overloads the meaning of 'library_dirs' to also mean "directories containing
# static libraries required for creating an executable" (currently, libc). These concepts
# should be distinct.
'LIBRARY_PATH': create_path_env_var(self.library_dirs),
'LDFLAGS': safe_shlex_join(all_ldflags_for_platform),
})
return ret
def invoke_perl6(self, argv, perl6_env, workunit_factory=None):
full_argv = [self._perl6_exe_filename] + list(argv)
subproc_env = self._get_perl6_subproc_os_env(perl6_env)
pretty_printed_argv = safe_shlex_join(full_argv)
try:
logger.debug('running perl6 comand {!r} with env {!r}'.format(full_argv, subproc_env))
if workunit_factory:
with workunit_factory(cmd=pretty_printed_argv) as workunit:
# TODO: should we be catching KeyboardInterrupt or something?
return subprocess.check_call(
full_argv,
env=subproc_env,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'))
else:
return subprocess.check_call(full_argv, env=subproc_env)
except (OSError, subprocess.CalledProcessError) as e:
raise self.Perl6InvocationError(
"Error with perl6 command '{}': {}".format(pretty_printed_argv, e),
e,
exit_code=e.returncode)
def _pretty_cmdline(self, args): return safe_shlex_join(self._pex.cmdline(args))
def _runtool_hermetic(self, main, tool_name, distribution, input_digest,
ctx):
tool_classpath_abs = self._rsc_classpath
tool_classpath = fast_relpath_collection(tool_classpath_abs)
jvm_options = self._jvm_options
if self._rsc.use_native_image:
#jvm_options = []
if jvm_options:
raise ValueError(
"`{}` got non-empty jvm_options when running with a graal native-image, but this is "
"unsupported. jvm_options received: {}".format(
self.options_scope, safe_shlex_join(jvm_options)))
native_image_path, native_image_snapshot = self._rsc.native_image(
self.context)
additional_snapshots = [native_image_snapshot]
initial_args = [native_image_path]
else:
additional_snapshots = []
initial_args = [
distribution.java,
] + self.get_options().jvm_options + [
'-cp',
os.pathsep.join(tool_classpath),
main,
]
argfile_snapshot, = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([fast_relpath(ctx.args_file, get_buildroot())]),
get_buildroot(),
),
])
cmd = initial_args + ['@{}'.format(argfile_snapshot.files[0])]
pathglobs = list(tool_classpath)
if pathglobs:
root = PathGlobsAndRoot(PathGlobs(tuple(pathglobs)),
get_buildroot())
# dont capture snapshot, if pathglobs is empty
path_globs_input_digest = self.context._scheduler.capture_snapshots(
(root, ))[0].directory_digest
epr_input_files = self.context._scheduler.merge_directories(
((path_globs_input_digest, ) if path_globs_input_digest else ()) +
((input_digest, ) if input_digest else ()) +
tuple(s.directory_digest for s in additional_snapshots) +
(argfile_snapshot.directory_digest, ))
epr = ExecuteProcessRequest(
argv=tuple(cmd),
input_files=epr_input_files,
output_files=(fast_relpath(ctx.rsc_jar_file.path,
get_buildroot()), ),
output_directories=tuple(),
timeout_seconds=15 * 60,
description='run {} for {}'.format(tool_name, ctx.target),
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying.home` because
# ExecuteProcessRequest requires an existing, local jdk location.
jdk_home=distribution.underlying_home,
)
res = self.context.execute_process_synchronously_without_raising(
epr, self.name(), [WorkUnitLabel.COMPILER])
if res.exit_code != 0:
raise TaskError(res.stderr, exit_code=res.exit_code)
# TODO: parse the output of -Xprint:timings for rsc and write it to self._record_target_stats()!
res.output_directory_digest.dump(ctx.rsc_jar_file.path)
ctx.rsc_jar_file = ClasspathEntry(ctx.rsc_jar_file.path,
res.output_directory_digest)
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(
# NB the first element here is the root to materialize into, not the dir to snapshot
get_buildroot(),
res.output_directory_digest), ))
return res
def _runtool_hermetic(self, main, tool_name, distribution, input_digest,
ctx):
use_youtline = tool_name == "scalac-outliner"
tool_classpath_abs = self._scalac_classpath if use_youtline else self._rsc_classpath
tool_classpath = fast_relpath_collection(tool_classpath_abs)
rsc_jvm_options = Rsc.global_instance().get_options().jvm_options
if not use_youtline and self._rsc.use_native_image:
if rsc_jvm_options:
raise ValueError(
"`{}` got non-empty jvm_options when running with a graal native-image, but this is "
"unsupported. jvm_options received: {}".format(
self.options_scope, safe_shlex_join(rsc_jvm_options)))
native_image_path, native_image_snapshot = self._rsc.native_image(
self.context)
additional_snapshots = [native_image_snapshot]
initial_args = [native_image_path]
else:
additional_snapshots = []
initial_args = (
[distribution.java] + rsc_jvm_options +
["-cp", os.pathsep.join(tool_classpath), main])
(argfile_snapshot, ) = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([fast_relpath(ctx.args_file, get_buildroot())]),
get_buildroot(),
),
])
cmd = initial_args + [f"@{argfile_snapshot.files[0]}"]
pathglobs = list(tool_classpath)
if pathglobs:
root = PathGlobsAndRoot(PathGlobs(tuple(pathglobs)),
get_buildroot())
# dont capture snapshot, if pathglobs is empty
path_globs_input_digest = self.context._scheduler.capture_snapshots(
(root, ))[0].digest
epr_input_files = self.context._scheduler.merge_directories(
((path_globs_input_digest, ) if path_globs_input_digest else ()) +
((input_digest, ) if input_digest else ()) +
tuple(s.digest
for s in additional_snapshots) + (argfile_snapshot.digest, ))
epr = Process(
argv=tuple(cmd),
input_digest=epr_input_files,
output_files=(fast_relpath(ctx.rsc_jar_file.path,
get_buildroot()), ),
output_directories=tuple(),
timeout_seconds=15 * 60,
description=f"run {tool_name} for {ctx.target}",
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying.home` because
# Process requires an existing, local jdk location.
jdk_home=distribution.underlying_home,
is_nailgunnable=True,
)
res = self.context.execute_process_synchronously_without_raising(
epr, self.name(), [WorkUnitLabel.COMPILER])
if res.exit_code != 0:
raise TaskError(res.stderr, exit_code=res.exit_code)
# TODO: parse the output of -Xprint:timings for rsc and write it to self._record_target_stats()!
res.output_digest.dump(ctx.rsc_jar_file.path)
self.context._scheduler.materialize_directory(
DirectoryToMaterialize(res.output_digest), )
ctx.rsc_jar_file.hydrate_missing_directory_digest(res.output_digest)
return res
def environment_dict(self) -> Dict[str, str]:
return {
"CPPFLAGS": safe_shlex_join(self.cpp_flags),
"LDFLAGS": safe_shlex_join(self.ld_flags),
}
def as_environment(self):
ret = {}
if self.setup_requires_site_dir:
ret['PYTHONPATH'] = self.setup_requires_site_dir.site_dir
# FIXME(#5951): the below is a lot of error-prone repeated logic -- we need a way to compose
# executables more hygienically. We should probably be composing each datatype's members, and
# only creating an environment at the very end.
native_tools = self.setup_py_native_tools
if native_tools:
# An as_tuple() method for datatypes could make this destructuring cleaner! Alternatively,
# constructing this environment could be done more compositionally instead of requiring all of
# these disparate fields together at once.
plat = native_tools.platform
c_toolchain = native_tools.c_toolchain
c_compiler = c_toolchain.c_compiler
c_linker = c_toolchain.c_linker
cpp_toolchain = native_tools.cpp_toolchain
cpp_compiler = cpp_toolchain.cpp_compiler
cpp_linker = cpp_toolchain.cpp_linker
all_path_entries = (c_compiler.path_entries +
c_linker.path_entries +
cpp_compiler.path_entries +
cpp_linker.path_entries)
ret['PATH'] = create_path_env_var(all_path_entries)
all_library_dirs = (c_compiler.library_dirs +
c_linker.library_dirs +
cpp_compiler.library_dirs +
cpp_linker.library_dirs)
joined_library_dirs = create_path_env_var(all_library_dirs)
dynamic_lib_env_var = plat.resolve_platform_specific({
'darwin':
lambda: 'DYLD_LIBRARY_PATH',
'linux':
lambda: 'LD_LIBRARY_PATH',
})
ret[dynamic_lib_env_var] = joined_library_dirs
all_linking_library_dirs = (c_linker.linking_library_dirs +
cpp_linker.linking_library_dirs)
ret['LIBRARY_PATH'] = create_path_env_var(all_linking_library_dirs)
all_include_dirs = cpp_compiler.include_dirs + c_compiler.include_dirs
ret['CPATH'] = create_path_env_var(all_include_dirs)
shared_compile_flags = safe_shlex_join(
plat.resolve_platform_specific({
'darwin':
lambda: [MIN_OSX_VERSION_ARG],
'linux':
lambda: [],
}))
ret['CFLAGS'] = shared_compile_flags
ret['CXXFLAGS'] = shared_compile_flags
ret['CC'] = c_compiler.exe_filename
ret['CXX'] = cpp_compiler.exe_filename
ret['LDSHARED'] = cpp_linker.exe_filename
all_new_ldflags = cpp_linker.extra_args + plat.resolve_platform_specific(
self._SHARED_CMDLINE_ARGS)
ret['LDFLAGS'] = safe_shlex_join(all_new_ldflags)
return ret
def _compile_hermetic(self, jvm_options, ctx, classes_dir, zinc_args,
compiler_bridge_classpath_entry, dependency_classpath,
scalac_classpath_entries):
zinc_relpath = fast_relpath(self._zinc.zinc, get_buildroot())
snapshots = [
self._zinc.snapshot(self.context._scheduler),
ctx.target.sources_snapshot(self.context._scheduler),
]
relevant_classpath_entries = dependency_classpath + [compiler_bridge_classpath_entry]
directory_digests = tuple(
entry.directory_digest for entry in relevant_classpath_entries if entry.directory_digest
)
if len(directory_digests) != len(relevant_classpath_entries):
for dep in relevant_classpath_entries:
if dep.directory_digest is None:
logger.warning(
"ClasspathEntry {} didn't have a Digest, so won't be present for hermetic "
"execution".format(dep)
)
snapshots.extend(
classpath_entry.directory_digest for classpath_entry in scalac_classpath_entries
)
if self._zinc.use_native_image:
if jvm_options:
raise ValueError(
"`{}` got non-empty jvm_options when running with a graal native-image, but this is "
"unsupported. jvm_options received: {}".format(self.options_scope, safe_shlex_join(jvm_options))
)
native_image_path, native_image_snapshot = self._zinc.native_image(self.context)
additional_snapshots = (native_image_snapshot.directory_digest,)
scala_boot_classpath = [
classpath_entry.path for classpath_entry in scalac_classpath_entries
] + [
# We include rt.jar on the scala boot classpath because the compiler usually gets its
# contents from the VM it is executing in, but not in the case of a native image. This
# resolves a `object java.lang.Object in compiler mirror not found.` error.
'.jdk/jre/lib/rt.jar',
# The same goes for the rce.jar, which provides javax.crypto.
'.jdk/jre/lib/jce.jar',
]
image_specific_argv = [
native_image_path,
'-java-home', '.jdk',
'-Dscala.boot.class.path={}'.format(os.pathsep.join(scala_boot_classpath)),
'-Dscala.usejavacp=true',
]
else:
additional_snapshots = ()
image_specific_argv = ['.jdk/bin/java'] + jvm_options + [
'-cp', zinc_relpath,
Zinc.ZINC_COMPILE_MAIN
]
# TODO: Extract something common from Executor._create_command to make the command line
# TODO: Lean on distribution for the bin/java appending here
merged_input_digest = self.context._scheduler.merge_directories(
tuple(s.directory_digest for s in snapshots) +
directory_digests +
additional_snapshots
)
argv = image_specific_argv + zinc_args
# TODO(#6071): Our ExecuteProcessRequest expects a specific string type for arguments,
# which py2 doesn't default to. This can be removed when we drop python 2.
argv = [text_type(arg) for arg in argv]
req = ExecuteProcessRequest(
argv=tuple(argv),
input_files=merged_input_digest,
output_directories=(classes_dir,),
description="zinc compile for {}".format(ctx.target.address.spec),
jdk_home=self._zinc.underlying_dist.home,
)
res = self.context.execute_process_synchronously_or_raise(
req, self.name(), [WorkUnitLabel.COMPILER])
# TODO: Materialize as a batch in do_compile or somewhere
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(get_buildroot(), res.output_directory_digest),
))
# TODO: This should probably return a ClasspathEntry rather than a Digest
return res.output_directory_digest
def _compile_hermetic(self, jvm_options, ctx, classes_dir, jar_file,
compiler_bridge_classpath_entry,
dependency_classpath, scalac_classpath_entries):
zinc_relpath = fast_relpath(self._zinc.zinc.path, get_buildroot())
snapshots = [
ctx.target.sources_snapshot(self.context._scheduler),
]
# scala_library() targets with java_sources have circular dependencies on those java source
# files, and we provide them to the same zinc command line that compiles the scala, so we need
# to make sure those source files are available in the hermetic execution sandbox.
java_sources_targets = getattr(ctx.target, 'java_sources', [])
java_sources_snapshots = [
tgt.sources_snapshot(self.context._scheduler)
for tgt in java_sources_targets
]
snapshots.extend(java_sources_snapshots)
# Ensure the dependencies and compiler bridge jars are available in the execution sandbox.
relevant_classpath_entries = (
dependency_classpath + [
compiler_bridge_classpath_entry,
self._nailgun_server_classpath_entry(
), # We include nailgun-server, to use it to start servers when needed from the hermetic execution case.
])
directory_digests = [
entry.directory_digest for entry in relevant_classpath_entries
if entry.directory_digest
]
if len(directory_digests) != len(relevant_classpath_entries):
for dep in relevant_classpath_entries:
if not dep.directory_digest:
raise AssertionError(
"ClasspathEntry {} didn't have a Digest, so won't be present for hermetic "
"execution of zinc".format(dep))
directory_digests.extend(
classpath_entry.directory_digest
for classpath_entry in scalac_classpath_entries)
if self._zinc.use_native_image:
if jvm_options:
raise ValueError(
"`{}` got non-empty jvm_options when running with a graal native-image, but this is "
"unsupported. jvm_options received: {}".format(
self.options_scope, safe_shlex_join(jvm_options)))
native_image_path, native_image_snapshot = self._zinc.native_image(
self.context)
native_image_snapshots = [
native_image_snapshot.directory_digest,
]
scala_boot_classpath = [
classpath_entry.path
for classpath_entry in scalac_classpath_entries
] + [
# We include rt.jar on the scala boot classpath because the compiler usually gets its
# contents from the VM it is executing in, but not in the case of a native image. This
# resolves a `object java.lang.Object in compiler mirror not found.` error.
'.jdk/jre/lib/rt.jar',
# The same goes for the jce.jar, which provides javax.crypto.
'.jdk/jre/lib/jce.jar',
]
image_specific_argv = [
native_image_path,
'-java-home',
'.jdk',
'-Dscala.boot.class.path={}'.format(
os.pathsep.join(scala_boot_classpath)),
'-Dscala.usejavacp=true',
]
else:
native_image_snapshots = []
# TODO: Lean on distribution for the bin/java appending here
image_specific_argv = ['.jdk/bin/java'] + jvm_options + [
'-cp', zinc_relpath, Zinc.ZINC_COMPILE_MAIN
]
argfile_snapshot, = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([fast_relpath(ctx.args_file, get_buildroot())]),
get_buildroot(),
),
])
relpath_to_analysis = fast_relpath(ctx.analysis_file, get_buildroot())
merged_local_only_scratch_inputs = self._compute_local_only_inputs(
classes_dir, relpath_to_analysis, jar_file)
# TODO: Extract something common from Executor._create_command to make the command line
argv = image_specific_argv + ['@{}'.format(argfile_snapshot.files[0])]
merged_input_digest = self.context._scheduler.merge_directories(
[self._zinc.zinc.directory_digest] +
[s.directory_digest for s in snapshots] + directory_digests +
native_image_snapshots + [
self.post_compile_extra_resources_digest(ctx),
argfile_snapshot.directory_digest
])
# NB: We always capture the output jar, but if classpath jars are not used, we additionally
# capture loose classes from the workspace. This is because we need to both:
# 1) allow loose classes as an input to dependent compiles
# 2) allow jars to be materialized at the end of the run.
output_directories = () if self.get_options().use_classpath_jars else (
classes_dir, )
req = ExecuteProcessRequest(
argv=tuple(argv),
input_files=merged_input_digest,
output_files=(jar_file, relpath_to_analysis),
output_directories=output_directories,
description="zinc compile for {}".format(ctx.target.address.spec),
unsafe_local_only_files_because_we_favor_speed_over_correctness_for_this_rule
=merged_local_only_scratch_inputs,
jdk_home=self._zinc.underlying_dist.home,
is_nailgunnable=True,
)
res = self.context.execute_process_synchronously_or_raise(
req, self.name(), [WorkUnitLabel.COMPILER])
# TODO: Materialize as a batch in do_compile or somewhere
self.context._scheduler.materialize_directory(
DirectoryToMaterialize(res.output_directory_digest))
# TODO: This should probably return a ClasspathEntry rather than a Digest
return res.output_directory_digest
def as_environment(self):
ret = {}
if self.setup_requires_site_dir:
ret['PYTHONPATH'] = self.setup_requires_site_dir.site_dir
# FIXME(#5951): the below is a lot of error-prone repeated logic -- we need a way to compose
# executables more hygienically. We should probably be composing each datatype's members, and
# only creating an environment at the very end.
native_tools = self.setup_py_native_tools
if native_tools:
# An as_tuple() method for datatypes could make this destructuring cleaner! Alternatively,
# constructing this environment could be done more compositionally instead of requiring all of
# these disparate fields together at once.
plat = native_tools.platform
c_toolchain = native_tools.c_toolchain
c_compiler = c_toolchain.c_compiler
c_linker = c_toolchain.c_linker
cpp_toolchain = native_tools.cpp_toolchain
cpp_compiler = cpp_toolchain.cpp_compiler
cpp_linker = cpp_toolchain.cpp_linker
all_path_entries = (
c_compiler.path_entries +
c_linker.path_entries +
cpp_compiler.path_entries +
cpp_linker.path_entries)
ret['PATH'] = create_path_env_var(all_path_entries)
all_library_dirs = (
c_compiler.library_dirs +
c_linker.library_dirs +
cpp_compiler.library_dirs +
cpp_linker.library_dirs)
joined_library_dirs = create_path_env_var(all_library_dirs)
dynamic_lib_env_var = plat.resolve_platform_specific({
'darwin': lambda: 'DYLD_LIBRARY_PATH',
'linux': lambda: 'LD_LIBRARY_PATH',
})
ret[dynamic_lib_env_var] = joined_library_dirs
all_linking_library_dirs = (c_linker.linking_library_dirs + cpp_linker.linking_library_dirs)
ret['LIBRARY_PATH'] = create_path_env_var(all_linking_library_dirs)
all_include_dirs = cpp_compiler.include_dirs + c_compiler.include_dirs
ret['CPATH'] = create_path_env_var(all_include_dirs)
shared_compile_flags = safe_shlex_join(plat.resolve_platform_specific({
'darwin': lambda: [MIN_OSX_VERSION_ARG],
'linux': lambda: [],
}))
ret['CFLAGS'] = shared_compile_flags
ret['CXXFLAGS'] = shared_compile_flags
ret['CC'] = c_compiler.exe_filename
ret['CXX'] = cpp_compiler.exe_filename
ret['LDSHARED'] = cpp_linker.exe_filename
all_new_ldflags = cpp_linker.extra_args + plat.resolve_platform_specific(
self._SHARED_CMDLINE_ARGS)
ret['LDFLAGS'] = safe_shlex_join(all_new_ldflags)
return ret
def invocation_environment_dict(self):
return {
'CPPFLAGS': safe_shlex_join(self.cpp_flags),
'LDFLAGS': safe_shlex_join(self.ld_flags),
}
def _pretty_cmdline(self, args): return safe_shlex_join(self._pex.cmdline(args))