def test_override_single_variable(self):
with temporary_file() as output:
# test that the override takes place
with environment_as(HORK='BORK'):
subprocess.Popen([sys.executable, '-c', 'import os; print(os.environ["HORK"])'],
stdout=output).wait()
output.seek(0)
self.assertEquals('BORK\n', output.read())
# test that the variable is cleared
with temporary_file() as new_output:
subprocess.Popen([sys.executable, '-c', 'import os; print("HORK" in os.environ)'],
stdout=new_output).wait()
new_output.seek(0)
self.assertEquals('False\n', new_output.read())
def test_communicate_teeing_retrieves_stdout_and_stderr(self):
process = subprocess.Popen([
"/bin/bash", "-c", """
echo "1out"
echo >&2 "1err"
sleep 0.05
echo >&2 "2err"
echo "2out"
sleep 0.05
echo "3out"
sleep 0.05
echo >&2 "3err"
sleep 0.05
exit 1
"""
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
process_handler = SubprocessProcessHandler(process)
self.assertEquals(
process_handler.communicate_teeing_stdout_and_stderr(), (b"""1out
2out
3out
""", b"""1err
2err
3err
"""))
def execute(self):
# Heuristics to guess whether user tries to load a python project,
# in which case intellij project sdk has to be set up manually.
jvm_target_num = len(
filter(lambda x: isinstance(x, JvmTarget),
self.context.target_roots))
python_target_num = len(
filter(lambda x: isinstance(x, PythonTarget),
self.context.target_roots))
if python_target_num > jvm_target_num:
logging.warn(
'This is likely a python project. Please make sure to '
'select the proper python interpreter as Project SDK in IntelliJ.'
)
ide_file = self.generate_project()
if ide_file and self.get_options().open:
open_with = self.get_options().open_with
if open_with:
null = open(os.devnull, 'w')
subprocess.Popen([open_with, ide_file],
stdout=null,
stderr=null)
else:
try:
desktop.ui_open(ide_file)
except desktop.OpenError as e:
raise TaskError(e)
def _get_system_properties(self, java):
if not self._system_properties:
with temporary_dir() as classpath:
with open(os.path.join(classpath, 'SystemProperties.class'),
'w+') as fp:
fp.write(
pkgutil.get_data(__name__, 'SystemProperties.class'))
cmd = [java, '-cp', classpath, 'SystemProperties']
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
if process.returncode != 0:
raise self.Error(
'Failed to determine java system properties for {} with {} - exit code'
' {}: {}'.format(java, ' '.join(cmd),
process.returncode, stderr))
props = {}
for line in stdout.split(os.linesep):
key, _, val = line.partition('=')
props[key] = val
self._system_properties = props
return self._system_properties
def _spawn(self,
cmd,
cwd=None,
stdout=None,
stderr=None,
stdin=None,
**subprocess_args):
cwd = cwd or os.getcwd()
# NB: Only stdout and stderr have non-None defaults: callers that want to capture
# stdin should pass it explicitly.
stdout = stdout or sys.stdout
stderr = stderr or sys.stderr
with self._maybe_scrubbed_env():
logger.debug('Executing: {cmd} args={args} at cwd={cwd}'.format(
cmd=' '.join(cmd), args=subprocess_args, cwd=cwd))
try:
return subprocess.Popen(cmd,
cwd=cwd,
stdin=stdin,
stdout=stdout,
stderr=stderr,
**subprocess_args)
except OSError as e:
raise self.Error(
'Problem executing {0} at cwd={1}: {2}'.format(
self._distribution.java, cwd, e))
def test_bundle_protobuf_unpacked_jars(self):
with self.pants_results([
'bundle.jvm', '--deployjar',
'examples/src/java/org/pantsbuild/example/protobuf/unpacked_jars'
]) as pants_run:
self.assertEqual(
pants_run.returncode, self.PANTS_SUCCESS_CODE,
"goal bundle run expected success, got {0}\n"
"got stderr:\n{1}\n"
"got stdout:\n{2}\n".format(pants_run.returncode,
pants_run.stderr_data,
pants_run.stdout_data))
out_path = os.path.join(get_buildroot(), 'dist', (
'examples.src.java.org.pantsbuild.example.protobuf.unpacked_jars'
'.unpacked_jars-bundle'))
args = [
'java', '-cp', 'protobuf-unpacked-jars-example.jar',
'org.pantsbuild.example.protobuf.unpacked_jars.ExampleProtobufExternalArchive'
]
java_run = subprocess.Popen(args,
stdout=subprocess.PIPE,
cwd=out_path)
java_retcode = java_run.wait()
java_out = java_run.stdout.read()
self.assertEqual(java_retcode, 0)
self.assertIn("Message is: Hello World!", java_out)
def _invoke_xz(self, xz_input_file):
"""Run the xz command and yield a file object for its stdout.
This allows streaming the decompressed tar archive directly into a tar decompression stream,
which is significantly faster in practice than making a temporary file.
"""
# FIXME: --threads=0 is supposed to use "the number of processor cores on the machine", but I
# see no more than 100% cpu used at any point. This seems like it could be a bug? If performance
# is an issue, investigate further.
cmd = [
self._xz_binary_path, '--decompress', '--stdout', '--keep',
'--threads=0', xz_input_file
]
try:
# Pipe stderr to our own stderr, but leave stdout open so we can yield it.
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=sys.stderr)
except OSError as e:
raise self.XZArchiverError(
"Error invoking xz with command {} for input file {}: {}".
format(cmd, xz_input_file, e), e)
# This is a file object.
yield process.stdout
rc = process.wait()
if rc != 0:
raise self.XZArchiverError(
"Error decompressing xz input with command {} for input file {}. Exit code was: {}. "
.format(cmd, xz_input_file, rc))
def console_output(self, _targets):
if not self.context.options.target_specs:
raise TaskError("No targets specified.")
# Heuristics to guess whether user tries to load a python project,
# in which case intellij project sdk has to be set up manually.
jvm_target_num = len(
[x for x in self.context.target_roots if isinstance(x, JvmTarget)])
python_target_num = len([
x for x in self.context.target_roots
if isinstance(x, PythonTarget)
])
if python_target_num > jvm_target_num:
logging.warn(
'This is likely a python project. Please make sure to '
'select the proper python interpreter as Project SDK in IntelliJ.'
)
ide_file = self.generate_project()
yield self.gen_project_workdir
if ide_file and self.get_options().open:
open_with = self.get_options().open_with
if open_with:
null = open(os.devnull, 'wb')
subprocess.Popen([open_with, ide_file],
stdout=null,
stderr=null)
else:
try:
desktop.ui_open(ide_file)
except desktop.OpenError as e:
raise TaskError(e)
def test_bundle_wire_dependent_targets(self):
with self.pants_results([
'bundle.jvm',
'examples/src/java/org/pantsbuild/example/wire/element'
]) as pants_run:
self.assert_success(pants_run)
out_path = os.path.join(
get_buildroot(), 'dist',
'examples.src.java.org.pantsbuild.example.wire.element.element-bundle'
)
java_run = subprocess.Popen([
'java', '-cp', 'wire-element-example.jar',
'org.pantsbuild.example.wire.element.WireElementExample'
],
stdout=subprocess.PIPE,
cwd=out_path)
java_retcode = java_run.wait()
java_out = java_run.stdout.read()
self.assertEquals(java_retcode, 0)
self.assertIn(
'Element{symbol=Hg, name=Mercury, atomic_number=80, '
'melting_point=Temperature{unit=celsius, number=-39}, '
'boiling_point=Temperature{unit=celsius, number=357}}',
java_out)
self.assertIn(
'Compound{name=Water, primary_element=Element{symbol=O, name=Oxygen, '
'atomic_number=8}, secondary_element=Element{symbol=H, name=Hydrogen, '
'atomic_number=1}}', java_out)
def test_intellij_integration(self):
with self.temporary_workdir() as workdir:
exported_file = os.path.join(workdir, "export_file.json")
p = subprocess.Popen([
'build-support/pants-intellij.sh',
'--export-output-file=' + exported_file
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p.communicate()
self.assertEqual(p.returncode, 0)
with open(exported_file) as data_file:
json_data = json.load(data_file)
python_setup = json_data['python_setup']
self.assertIsNotNone(python_setup)
self.assertIsNotNone(python_setup['interpreters'])
default_interpreter = python_setup['default_interpreter']
self.assertIsNotNone(default_interpreter)
self.assertIsNotNone(
python_setup['interpreters'][default_interpreter])
self.assertTrue(
os.path.exists(python_setup['interpreters']
[default_interpreter]['binary']))
self.assertTrue(
os.path.exists(python_setup['interpreters']
[default_interpreter]['chroot']))
python_target = json_data['targets'][
'src/python/pants/backend/python/targets:targets']
self.assertIsNotNone(python_target)
self.assertEquals(default_interpreter,
python_target['python_interpreter'])
def _test_bundle_existences(self, args, bundles, config=None):
all_bundles = set(bundle.spec for bundle in Bundles.all_bundles)
all_paths = [self._bundle_path(bundle) for bundle in all_bundles]
names = [bundle.spec for bundle in bundles]
outputs = [bundle.text for bundle in bundles]
with self.pants_results(['bundle'] + args, config=config) as pants_run:
self.assert_success(pants_run)
with self._handle_bundles(names) as (paths, jars):
for path, jar, expected in zip(paths, jars, outputs):
java_run = subprocess.Popen(['java', '-jar', jar],
stdout=subprocess.PIPE,
cwd=path)
java_retcode = java_run.wait()
java_out = java_run.stdout.read()
self.assertEquals(java_retcode, 0)
self.assertTrue(
expected in java_out,
"Expected '{output}' from {jar}, not '{stdout}'.".
format(output=expected, jar=jar, stdout=java_out))
lingering = [path for path in all_paths if os.path.exists(path)]
self.assertTrue(
not lingering, "Left with unexpected bundles! {bundles}".format(
bundles=', '.join(lingering)))
def git_version():
"""Get a Version() based on installed command-line git's version"""
process = subprocess.Popen(['git', '--version'], stdout=subprocess.PIPE)
(stdout, stderr) = process.communicate()
assert process.returncode == 0, "Failed to determine git version."
# stdout is like 'git version 1.9.1.598.g9119e8b\n' We want '1.9.1.598'
matches = re.search(r'\s(\d+(?:\.\d+)*)[\s\.]', stdout.decode('utf-8'))
return Revision.lenient(matches.group(1))
def create_jvmdoc(command, gendir):
try:
safe_mkdir(gendir, clean=True)
process = subprocess.Popen(command)
result = process.wait()
return result, gendir
except OSError:
return 1, gendir
def _execute_link_request(self, link_request):
object_files = link_request.object_files
if len(object_files) == 0:
raise self.LinkSharedLibrariesError(
"No object files were provided in request {}!".format(
link_request))
linker = link_request.linker
native_artifact = link_request.native_artifact
output_dir = link_request.output_dir
resulting_shared_lib_path = os.path.join(
output_dir, native_artifact.as_shared_lib(self.platform))
self.context.log.debug(
"resulting_shared_lib_path: {}".format(resulting_shared_lib_path))
# We are executing in the results_dir, so get absolute paths for everything.
cmd = ([linker.exe_filename] + self.platform.resolve_platform_specific(
self._SHARED_CMDLINE_ARGS) + linker.extra_args +
['-o', os.path.abspath(resulting_shared_lib_path)] + [
'-L{}'.format(lib_dir)
for lib_dir in link_request.external_lib_dirs
] + [
'-l{}'.format(lib_name)
for lib_name in link_request.external_lib_names
] + [os.path.abspath(obj) for obj in object_files])
self.context.log.debug("linker command: {}".format(cmd))
env = linker.as_invocation_environment_dict
self.context.log.debug("linker invocation environment: {}".format(env))
with self.context.new_workunit(name='link-shared-libraries',
labels=[WorkUnitLabel.LINKER
]) as workunit:
try:
process = subprocess.Popen(cmd,
cwd=output_dir,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'),
env=env)
except OSError as e:
workunit.set_outcome(WorkUnit.FAILURE)
raise self.LinkSharedLibrariesError(
"Error invoking the native linker with command {cmd} and environment {env} "
"for request {req}: {err}.".format(cmd=cmd,
env=env,
req=link_request,
err=e), e)
rc = process.wait()
if rc != 0:
workunit.set_outcome(WorkUnit.FAILURE)
raise self.LinkSharedLibrariesError(
"Error linking native objects with command {cmd} and environment {env} "
"for request {req}. Exit code was: {rc}.".format(
cmd=cmd, env=env, req=link_request, rc=rc))
return SharedLibrary(name=native_artifact.lib_name,
path=resulting_shared_lib_path)
def exe_path(name):
process = subprocess.Popen(['which', name],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, _ = process.communicate()
if process.returncode != 0:
return None
path = stdout.strip()
return path if os.path.exists(path) and os.access(path, os.X_OK) else None
def execute_with_go_env(self, go_path, import_paths, args, **kwargs):
cmd = ' '.join(args)
env = os.environ.copy()
env.update(GOROOT=self.go_dist.goroot, GOPATH=go_path)
process = subprocess.Popen(cmd, shell=True, env=env, **kwargs)
result = process.wait()
if result != 0:
raise TaskError('{} failed with exit code {}'.format(cmd, result),
exit_code=result)
def run(self, **kwargs):
"""Runs this command.
:param **kwargs: Any extra keyword arguments to pass along to `subprocess.Popen`.
:returns: A handle to the running command.
:rtype: :class:`subprocess.Popen`
"""
env, kwargs = self._prepare_env(kwargs)
return subprocess.Popen(self.cmd, env=env, **kwargs)
def test_environment_negation(self):
with temporary_file() as output:
with environment_as(HORK='BORK'):
with environment_as(HORK=None):
# test that the variable is cleared
subprocess.Popen([sys.executable, '-c', 'import os; print("HORK" in os.environ)'],
stdout=output).wait()
output.seek(0)
self.assertEquals('False\n', output.read())
def bundle_and_run(self, target, bundle_name, bundle_jar_name=None, bundle_options=None,
args=None,
expected_bundle_jar_content=None,
expected_bundle_content=None,
library_jars_are_symlinks=True):
"""Creates the bundle with pants, then does java -jar {bundle_name}.jar to execute the bundle.
:param target: target name to compile
:param bundle_name: resulting bundle filename (minus .zip extension)
:param bundle_jar_name: monolithic jar filename (minus .jar extension), if None will be the
same as bundle_name
:param bundle_options: additional options for bundle
:param args: optional arguments to pass to executable
:param expected_bundle_content: verify the bundle zip content
:param expected_bundle_jar_content: verify the bundle jar content
:param library_jars_are_symlinks: verify library jars are symlinks if True, and actual
files if False. Default `True` because we always create symlinks for both external and internal
dependencies, only exception is when shading is used.
:return: stdout as a string on success, raises an Exception on error
"""
bundle_jar_name = bundle_jar_name or bundle_name
bundle_options = bundle_options or []
bundle_options = ['bundle.jvm'] + bundle_options + ['--archive=zip', target]
with self.pants_results(bundle_options) as pants_run:
self.assert_success(pants_run)
self.assertTrue(check_symlinks('dist/{bundle_name}-bundle/libs'.format(bundle_name=bundle_name),
library_jars_are_symlinks))
# TODO(John Sirois): We need a zip here to suck in external library classpath elements
# pointed to by symlinks in the run_pants ephemeral tmpdir. Switch run_pants to be a
# contextmanager that yields its results while the tmpdir workdir is still active and change
# this test back to using an un-archived bundle.
with temporary_dir() as workdir:
ZIP.extract('dist/{bundle_name}.zip'.format(bundle_name=bundle_name), workdir)
if expected_bundle_content:
self.assertTrue(contains_exact_files(workdir, expected_bundle_content))
if expected_bundle_jar_content:
with temporary_dir() as check_bundle_jar_dir:
bundle_jar = os.path.join(workdir, '{bundle_jar_name}.jar'
.format(bundle_jar_name=bundle_jar_name))
ZIP.extract(bundle_jar, check_bundle_jar_dir)
self.assertTrue(contains_exact_files(check_bundle_jar_dir, expected_bundle_jar_content))
optional_args = []
if args:
optional_args = args
java_run = subprocess.Popen(['java',
'-jar',
'{bundle_jar_name}.jar'.format(bundle_jar_name=bundle_jar_name)]
+ optional_args,
stdout=subprocess.PIPE,
cwd=workdir)
stdout, _ = java_run.communicate()
java_returncode = java_run.returncode
self.assertEqual(java_returncode, 0)
return stdout.decode('utf-8')
def test_namespace_effective(self):
self.create_file('src/thrift/com/foo/one.thrift',
contents=dedent("""
namespace py foo.bar
struct One {}
"""))
one = self.make_target(spec='src/thrift/com/foo:one',
target_type=PythonThriftLibrary,
sources=['one.thrift'])
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(
one)
self.create_file('src/thrift2/com/foo/two.thrift',
contents=dedent("""
namespace py foo.baz
struct Two {}
"""))
two = self.make_target(spec='src/thrift2/com/foo:two',
target_type=PythonThriftLibrary,
sources=['two.thrift'])
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base,
synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
python_repos = global_subsystem_instance(PythonRepos)
python_setup = global_subsystem_instance(PythonSetup)
interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
pythonpath = [
os.path.join(get_buildroot(), t.target_base) for t in targets
]
for dist in resolve(
['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
interpreter=interpreter,
context=python_repos.get_network_context(),
fetchers=python_repos.get_fetchers()):
pythonpath.append(dist.location)
process = subprocess.Popen([
interpreter.binary, '-c',
'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'
],
env={
'PYTHONPATH':
os.pathsep.join(pythonpath)
},
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
def _spawn(self, pex, workunit, args, setsid=False):
# NB: We don't use pex.run(...) here since it makes a point of running in a clean environment,
# scrubbing all `PEX_*` environment overrides and we use overrides when running pexes in this
# task.
process = subprocess.Popen(pex.cmdline(args),
preexec_fn=os.setsid if setsid else None,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'))
return SubprocessProcessHandler(process)
def spawn(self, env=None, **kwargs):
"""
:param dict env: A custom environment to launch the Go command in. If `None` the current
environment is used.
:param kwargs: Keyword arguments to pass through to `subprocess.Popen`.
:returns: A handle to the spawned go command subprocess.
:rtype: :class:`subprocess.Popen`
"""
env = (env or os.environ).copy()
env.update(self.env)
return subprocess.Popen(self.cmdline, env=env, **kwargs)
def _run_command(binary, sandbox_dir, process_request):
command = binary.prefix_of_command() + tuple(process_request.args)
logger.debug('Running command: "{}" in {}'.format(command, sandbox_dir))
popen = subprocess.Popen(command,
stderr=subprocess.PIPE,
stdout=subprocess.PIPE,
cwd=sandbox_dir)
# TODO: At some point, we may want to replace this blocking wait with a timed one that returns
# some kind of in progress state.
popen.wait()
logger.debug('Done running command in {}'.format(sandbox_dir))
return popen
def execute(self):
if self.goal not in PrepCommand.allowed_goals():
raise AssertionError('Got goal "{}". Expected goal to be one of {}'.format(
self.goal, PrepCommand.goals()))
targets = self.context.targets(postorder=True, predicate=self.runnable_prep_cmd)
Cmdline = namedtuple('Cmdline', ['cmdline', 'environ'])
def make_cmdline(target):
executable = target.payload.get_field_value('prep_command_executable')
args = target.payload.get_field_value('prep_command_args', [])
prep_environ = target.payload.get_field_value('prep_environ')
cmdline = [executable]
cmdline.extend(args)
return Cmdline(cmdline=tuple(cmdline), environ=prep_environ)
def has_prep(target):
return target.payload.get_field_value('prep_command_executable')
cmdlines = [make_cmdline(target) for target in targets if has_prep(target)]
if not cmdlines:
return
with self.context.new_workunit(name='prep_command', labels=[WorkUnitLabel.PREP]) as workunit:
completed_cmdlines = set()
for item in cmdlines:
cmdline = item.cmdline
environ = item.environ
if not cmdline in completed_cmdlines:
completed_cmdlines.add(cmdline)
stderr = workunit.output('stderr') if workunit else None
try:
process = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=stderr)
except OSError as e:
workunit.set_outcome(WorkUnit.FAILURE)
raise TaskError('RunPrepCommand failed to execute {cmdline}: {error}'.format(
cmdline=cmdline, error=e))
stdout, _ = process.communicate()
if environ:
if not process.returncode:
environment_vars = stdout.split('\0')
for kvpair in environment_vars:
var, value = kvpair.split('=', 1)
os.environ[var] = value
else:
if workunit:
workunit.output('stdout').write(stdout)
workunit.set_outcome(WorkUnit.FAILURE if process.returncode else WorkUnit.SUCCESS)
if process.returncode:
raise TaskError('RunPrepCommand failed to run {cmdline}'.format(cmdline=cmdline))
def _assert_subprocess_error(self, worktree, cmd, expected_excerpt):
proc = subprocess.Popen(
cmd,
cwd=worktree,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
(stdout_data, stderr_data) = proc.communicate()
stdout_data = stdout_data.decode('utf-8')
stderr_data = stderr_data.decode('utf-8')
self.assertNotEqual(0, proc.returncode)
all_output = '{}\n{}'.format(stdout_data, stderr_data)
self.assertIn(expected_excerpt, all_output)
def _compile(self, compile_request):
"""Perform the process of compilation, writing object files to the request's 'output_dir'.
NB: This method must arrange the output files so that `collect_cached_objects()` can collect all
of the results (or vice versa)!
"""
sources = compile_request.sources
if len(sources) == 0:
# TODO: do we need this log message? Should we still have it for intentionally header-only
# libraries (that might be a confusing message to see)?
self.context.log.debug(
"no sources in request {}, skipping".format(compile_request))
return
compiler = compile_request.compiler
output_dir = compile_request.output_dir
argv = self._make_compile_argv(compile_request)
env = compiler.as_invocation_environment_dict
with self.context.new_workunit(name=self.workunit_label,
labels=[WorkUnitLabel.COMPILER
]) as workunit:
try:
process = subprocess.Popen(argv,
cwd=output_dir,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'),
env=env)
except OSError as e:
workunit.set_outcome(WorkUnit.FAILURE)
raise self.NativeCompileError(
"Error invoking '{exe}' with command {cmd} and environment {env} for request {req}: {err}"
.format(exe=compiler.exe_filename,
cmd=argv,
env=env,
req=compile_request,
err=e))
rc = process.wait()
if rc != 0:
workunit.set_outcome(WorkUnit.FAILURE)
raise self.NativeCompileError(
"Error in '{section_name}' with command {cmd} and environment {env} for request {req}. "
"Exit code was: {rc}.".format(
section_name=self.workunit_label,
cmd=argv,
env=env,
req=compile_request,
rc=rc))
def test_namespace_effective(self):
self.create_file('src/thrift/com/foo/one.thrift', contents=dedent("""
namespace py foo.bar
struct One {}
"""))
one = self.make_target(spec='src/thrift/com/foo:one',
target_type=PythonThriftLibrary,
sources=['one.thrift'])
apache_thrift_gen, synthetic_target_one = self.generate_single_thrift_target(one)
self.create_file('src/thrift2/com/foo/two.thrift', contents=dedent("""
namespace py foo.baz
struct Two {}
"""))
two = self.make_target(spec='src/thrift2/com/foo:two',
target_type=PythonThriftLibrary,
sources=['two.thrift'])
_, synthetic_target_two = self.generate_single_thrift_target(two)
# Confirm separate PYTHONPATH entries, which we need to test namespace packages.
self.assertNotEqual(synthetic_target_one.target_base, synthetic_target_two.target_base)
targets = (synthetic_target_one, synthetic_target_two)
python_repos = global_subsystem_instance(PythonRepos)
python_setup = global_subsystem_instance(PythonSetup)
interpreter_cache = PythonInterpreterCache(python_setup, python_repos)
interpreter = interpreter_cache.select_interpreter_for_targets(targets)
# We need setuptools to import namespace packages (via pkg_resources), so we prime the
# PYTHONPATH with interpreter extras, which Pants always populates with setuptools and wheel.
# TODO(John Sirois): We really should be emitting setuptools in a
# `synthetic_target_extra_dependencies` override in `ApacheThriftPyGen`:
# https://github.com/pantsbuild/pants/issues/5975
pythonpath = interpreter.extras.values()
pythonpath.extend(os.path.join(get_buildroot(), t.target_base) for t in targets)
for dist in resolve(['thrift=={}'.format(self.get_thrift_version(apache_thrift_gen))],
interpreter=interpreter,
context=python_repos.get_network_context(),
fetchers=python_repos.get_fetchers()):
pythonpath.append(dist.location)
process = subprocess.Popen([interpreter.binary,
'-c',
'from foo.bar.ttypes import One; from foo.baz.ttypes import Two'],
env={'PYTHONPATH': os.pathsep.join(pythonpath)},
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode, stderr)
def _echo_version(self, version):
with temporary_dir() as distdir:
config = {'GLOBAL': {'pants_distdir': distdir}}
binary_name = 'echo_interpreter_version_{}'.format(version)
binary_target = '{}:{}'.format(self.testproject, binary_name)
pants_run = self._build_pex(binary_target, config)
self.assert_success(
pants_run,
'Failed to build {binary}.'.format(binary=binary_target))
# Run the built pex.
exe = os.path.join(distdir, binary_name + '.pex')
proc = subprocess.Popen([exe], stdout=subprocess.PIPE)
(stdout_data, _) = proc.communicate()
return stdout_data
def _compile(self, compile_request):
"""Perform the process of compilation, writing object files to the request's 'output_dir'.
NB: This method must arrange the output files so that `collect_cached_objects()` can collect all
of the results (or vice versa)!
"""
try:
argv = self._make_compile_argv(compile_request)
except self._HeaderOnlyLibrary:
self.context.log.debug(
'{} is a header-only library'.format(compile_request))
return
compiler = compile_request.compiler
output_dir = compile_request.output_dir
env = compiler.as_invocation_environment_dict
with self.context.new_workunit(name=self.workunit_label,
labels=[WorkUnitLabel.COMPILER
]) as workunit:
try:
process = subprocess.Popen(argv,
cwd=output_dir,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'),
env=env)
except OSError as e:
workunit.set_outcome(WorkUnit.FAILURE)
raise self.NativeCompileError(
"Error invoking '{exe}' with command {cmd} and environment {env} for request {req}: {err}"
.format(exe=compiler.exe_filename,
cmd=argv,
env=env,
req=compile_request,
err=e))
rc = process.wait()
if rc != 0:
workunit.set_outcome(WorkUnit.FAILURE)
raise self.NativeCompileError(
"Error in '{section_name}' with command {cmd} and environment {env} for request {req}. "
"Exit code was: {rc}.".format(
section_name=self.workunit_label,
cmd=argv,
env=env,
req=compile_request,
rc=rc))
def _invoke(cls, cmd):
"""Invoke the given command, and return a tuple of process and raw binary output.
stderr flows to wherever its currently mapped for the parent process - generally to
the terminal where the user can see the error.
:param list cmd: The command in the form of a list of strings
:returns: The completed process object and its standard output.
:raises: Scm.LocalException if there was a problem exec'ing the command at all.
"""
try:
process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
except OSError as e:
# Binary DNE or is not executable
raise cls.LocalException('Failed to execute command {}: {}'.format(' '.join(cmd), e))
out, _ = process.communicate()
return process, out
