def test_pants_binary_dep_isolation_with_multiple_targets(self):
pex1 = os.path.join(get_buildroot(), 'dist', 'main_with_no_conflict.pex')
pex2 = os.path.join(get_buildroot(), 'dist', 'main_with_no_pycountry.pex')
try:
command=['binary', '{}:main_with_no_conflict'.format(self.fasthello_install_requires),
'{}:main_with_no_pycountry'.format(self.fasthello_install_requires)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex1))
self.assertTrue(os.path.isfile(pex2))
# Check that the pex 1 runs.
output = subprocess.check_output(pex1)
self._assert_native_greeting(output)
# Check that the pex 2 fails due to no python_dists leaked into it.
try:
output = subprocess.check_output(pex2)
except subprocess.CalledProcessError as e:
self.assertNotEquals(0, e.returncode)
finally:
# Cleanup
if os.path.exists(pex1):
os.remove(pex1)
if os.path.exists(pex2):
os.remove(pex2)
def setUp(self):
self.origin = safe_mkdtemp()
with pushd(self.origin):
subprocess.check_call(['git', 'init', '--bare'])
self.gitdir = safe_mkdtemp()
self.worktree = safe_mkdtemp()
self.readme_file = os.path.join(self.worktree, 'README')
with environment_as(GIT_DIR=self.gitdir, GIT_WORK_TREE=self.worktree):
self.init_repo('depot', self.origin)
touch(self.readme_file)
subprocess.check_call(['git', 'add', 'README'])
safe_mkdir(os.path.join(self.worktree, 'dir'))
with open(os.path.join(self.worktree, 'dir', 'f'), 'w') as f:
f.write("file in subdir")
# Make some symlinks
os.symlink('f', os.path.join(self.worktree, 'dir', 'relative-symlink'))
os.symlink('no-such-file', os.path.join(self.worktree, 'dir', 'relative-nonexistent'))
os.symlink('dir/f', os.path.join(self.worktree, 'dir', 'not-absolute\u2764'))
os.symlink('../README', os.path.join(self.worktree, 'dir', 'relative-dotdot'))
os.symlink('dir', os.path.join(self.worktree, 'link-to-dir'))
os.symlink('README/f', os.path.join(self.worktree, 'not-a-dir'))
os.symlink('loop1', os.path.join(self.worktree, 'loop2'))
os.symlink('loop2', os.path.join(self.worktree, 'loop1'))
subprocess.check_call(['git', 'add', 'README', 'dir', 'loop1', 'loop2',
'link-to-dir', 'not-a-dir'])
subprocess.check_call(['git', 'commit', '-am', 'initial commit with decode -> \x81b'])
self.initial_rev = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
subprocess.check_call(['git', 'tag', 'first'])
subprocess.check_call(['git', 'push', '--tags', 'depot', 'master'])
subprocess.check_call(['git', 'branch', '--set-upstream-to', 'depot/master'])
with safe_open(self.readme_file, 'w') as readme:
readme.write('Hello World.\u2764'.encode('utf-8'))
subprocess.check_call(['git', 'commit', '-am', 'Update README.'])
self.current_rev = subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
self.clone2 = safe_mkdtemp()
with pushd(self.clone2):
self.init_repo('origin', self.origin)
subprocess.check_call(['git', 'pull', '--tags', 'origin', 'master:master'])
with safe_open(os.path.realpath('README'), 'a') as readme:
readme.write('--')
subprocess.check_call(['git', 'commit', '-am', 'Update README 2.'])
subprocess.check_call(['git', 'push', '--tags', 'origin', 'master'])
self.git = Git(gitdir=self.gitdir, worktree=self.worktree)
def test_awslambda_bundle(self):
with temporary_dir() as distdir:
config = {
'GLOBAL': {
'pants_distdir': distdir,
'pythonpath': ['%(buildroot)s/contrib/awslambda/python/src/python'],
'backend_packages': ['pants.backend.python', 'pants.contrib.awslambda.python'],
}
}
command = [
'bundle',
'contrib/awslambda/python/src/python/pants/contrib/awslambda/python/examples:hello-lambda',
]
pants_run = self.run_pants(command=command, config=config)
self.assert_success(pants_run)
# Now run the lambda via the wrapper handler injected by lambdex (note that this
# is distinct from the pex's entry point - a handler must be a function with two arguments,
# whereas the pex entry point is a module).
awslambda = os.path.join(distdir, 'hello-lambda.pex')
output = subprocess.check_output(env={'PEX_INTERPRETER': '1'}, args=[
'{} -c "from lambdex_handler import handler; handler(None, None)"'.format(awslambda)
], shell=True)
self.assertEquals(b'Hello from the United States!', output.strip())
def test_pants_resolves_local_dists_for_current_platform_only(self):
# Test that pants will override pants.ini platforms config when building
# or running a target that depends on native (c or cpp) sources.
with temporary_dir() as tmp_dir:
pex = os.path.join(tmp_dir, 'main.pex')
pants_ini_config = {
'python-setup': {
'platforms': ['current', 'linux-x86_64']
}
}
# Clean all to rebuild requirements pex.
command = [
'--pants-distdir={}'.format(tmp_dir), 'run',
'{}:main'.format(self.fasthello_project)
]
pants_run = self.run_pants(command=command,
config=pants_ini_config)
self.assert_success(pants_run)
command = ['binary', '{}:main'.format(self.fasthello_project)]
pants_run = self.run_pants(command=command,
config=pants_ini_config)
self.assert_success(pants_run)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex))
# Check that the pex runs.
output = subprocess.check_output(pex)
self._assert_native_greeting(output)
def get_pyenv_root():
try:
return subprocess.check_output(['pyenv',
'root']).decode('utf-8').strip()
except (OSError, subprocess.CalledProcessError):
logger.info('No pyenv binary found. Will not use pyenv interpreters.')
return None
def jvm_locations(self):
# OSX will have a java_home tool that can be used to locate a unix-compatible java home dir.
#
# See:
# https://developer.apple.com/library/mac/documentation/Darwin/Reference/ManPages/man1/java_home.1.html
#
# The `--xml` output looks like so:
# <?xml version="1.0" encoding="UTF-8"?>
# <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
# "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
# <plist version="1.0">
# <array>
# <dict>
# ...
# <key>JVMHomePath</key>
# <string>/Library/Java/JavaVirtualMachines/jdk1.7.0_45.jdk/Contents/Home</string>
# ...
# </dict>
# ...
# </array>
# </plist>
if os.path.exists(self._osx_java_home_exe):
try:
plist = subprocess.check_output(
[self._osx_java_home_exe, '--failfast', '--xml'])
plist_results = plistlib.loads(
plist) if PY3 else plistlib.readPlistFromString(plist)
for distribution in plist_results:
home = distribution['JVMHomePath']
yield self.Location.from_home(home)
except subprocess.CalledProcessError:
pass
def test_pants_resolves_local_dists_for_current_platform_only(self):
# Test that pants will override pants.ini platforms config when building
# or running a target that depends on native (c or cpp) sources.
with temporary_dir() as tmp_dir:
pex = os.path.join(tmp_dir, 'main.pex')
pants_ini_config = {
'python-setup': {
'platforms': ['current', 'this-platform-does_not-exist'],
},
}
command=[
'--pants-distdir={}'.format(tmp_dir),
'run',
'{}:main'.format(self.fasthello_project)]
pants_run = self.run_pants(command=command, config=pants_ini_config)
self.assert_success(pants_run)
command=['binary', '{}:main'.format(self.fasthello_project)]
pants_run = self.run_pants(command=command, config=pants_ini_config)
self.assert_success(pants_run)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex))
# Check that the pex runs.
output = subprocess.check_output(pex)
self._assert_native_greeting(output)
def commit_contents_to_files(message, encoding, content, *files):
for path in files:
with safe_open(os.path.join(self.worktree, path),
'w') as fp:
fp.write(content)
subprocess.check_call(['git', 'add', '.'])
subprocess.check_call([
'git', 'config', '--local', '--add', 'i18n.commitencoding',
encoding
])
subprocess.check_call(
['git', 'config', '--local', 'commit.gpgSign', 'false'])
try:
subprocess.check_call(
['git', 'commit', '-m',
message.encode(encoding)])
finally:
subprocess.check_call([
'git', 'config', '--local', '--unset-all',
'i18n.commitencoding'
])
return subprocess.check_output(['git', 'rev-parse',
'HEAD']).strip()
def assert_no_gopath(self):
go_distribution = self.distribution()
go_env = go_distribution.go_env()
# As of go 1.8, when GOPATH is unset (set to ''), it defaults to ~/go (assuming HOME is set -
# and we can't unset that since it might legitmately be used by the subcommand) - so we manually
# fetch the "unset" default value here as our expected value for tests below.
# The key thing to note here is this default value is used only when `gopath` passed to
# `GoDistribution` is None, implying the command to be run does not need or use a GOPATH.
cmd = [
os.path.join(go_distribution.goroot, 'bin', 'go'), 'env', 'GOPATH'
]
env = os.environ.copy()
env.update(go_env)
default_gopath = subprocess.check_output(cmd, env=env).strip()
go_cmd = go_distribution.create_go_cmd(cmd='env', args=['GOPATH'])
self.assertEqual(go_env, go_cmd.env)
self.assertEqual('go', os.path.basename(go_cmd.cmdline[0]))
self.assertEqual(['env', 'GOPATH'], go_cmd.cmdline[1:])
self.assertRegexpMatches(
str(go_cmd),
r'^GOROOT=[^ ]+ GOPATH={} .*/go env GOPATH'.format(default_gopath))
self.assertEqual(default_gopath, go_cmd.check_output().strip())
def commit_contents_to_files(content, *files):
for path in files:
with safe_open(os.path.join(self.worktree, path), 'w') as fp:
fp.write(content)
subprocess.check_call(['git', 'add', '.'])
subprocess.check_call(['git', 'commit', '-m', 'change {}'.format(files)])
return subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
def test_hermetic_environment_subprocesses(self):
self.assertIn('USER', os.environ)
with hermetic_environment_as(**dict(AAA='333')):
output = subprocess.check_output('env', shell=True).decode('utf-8')
self.assertNotIn('USER='******'AAA', os.environ)
self.assertEquals(os.environ['AAA'], '333')
self.assertIn('USER', os.environ)
self.assertNotIn('AAA', os.environ)
def test_go_crosscompile(self):
# We assume that targeting windows is cross-compiling.
output_file = "dist/go/bin/hello.exe"
safe_delete(output_file)
args = ['binary', 'contrib/go/examples/src/go/hello']
pants_run = self.run_pants(args, extra_env={"GOOS": "windows"})
self.assert_success(pants_run)
self.assertIn("for MS Windows",
subprocess.check_output(["file", output_file]))
def test_go_crosscompile(self):
# We assume that targeting windows is cross-compiling.
output_file = "dist/go/bin/hello.exe"
safe_delete(output_file)
args = ['binary',
'contrib/go/examples/src/go/hello']
pants_run = self.run_pants(args, extra_env={"GOOS": "windows"})
self.assert_success(pants_run)
self.assertIn(b"for MS Windows", subprocess.check_output(["file", output_file]))
def test_hermetic_environment_subprocesses(self):
self.assertIn('USER', os.environ)
with hermetic_environment_as(**dict(AAA='333')):
output = subprocess.check_output('env', shell=True)
self.assertNotIn('USER='******'AAA', os.environ)
self.assertEquals(os.environ['AAA'], '333')
self.assertIn('USER', os.environ)
self.assertNotIn('AAA', os.environ)
def test_hermetic_environment_subprocesses(self):
with self.ensure_user_defined_in_environment():
with hermetic_environment_as(AAA='333'):
output = subprocess.check_output('env',
shell=True).decode('utf-8')
self.assertNotIn('USER='******'AAA', os.environ)
self.assertEqual(os.environ['AAA'], '333')
self.assertIn('USER', os.environ)
self.assertNotIn('AAA', os.environ)
def check_output(self, **kwargs):
"""Runs this command returning its captured stdout.
:param **kwargs: Any extra keyword arguments to pass along to `subprocess.Popen`.
:returns: The captured standard output stream of the command.
:rtype: string
:raises: :class:`subprocess.CalledProcessError` if the command fails.
"""
env, kwargs = self._prepare_env(kwargs)
return subprocess.check_output(self.cmd, env=env, **kwargs)
def commit_contents_to_files(content, *files):
for path in files:
with safe_open(os.path.join(self.worktree, path),
'w') as fp:
fp.write(content)
subprocess.check_call(['git', 'add', '.'])
subprocess.check_call(
['git', 'commit', '-m', 'change {}'.format(files)])
return subprocess.check_output(['git', 'rev-parse',
'HEAD']).strip()
def check_output(self, **kwargs):
"""Runs this command returning its captured stdout.
:param kwargs: Any extra keyword arguments to pass along to `subprocess.Popen`.
:returns: The captured standard output stream of the command.
:rtype: string
:raises: :class:`subprocess.CalledProcessError` if the command fails.
"""
env, kwargs = self._prepare_env(kwargs)
return subprocess.check_output(self.cmd, env=env, **kwargs)
def ensure_conan_remote_configuration(self, conan_binary):
"""
Ensure that the conan registry.txt file is sanitized and loaded with
a pants-specific remote for package fetching.
:param conan_binary: The conan client pex to use for manipulating registry.txt.
"""
# Conan will prepend the conan-center remote to the remote registry when
# bootstrapped for the first time, so we want to delete it from the registry
# and replace it with Pants-controlled remotes.
remove_conan_center_remote_cmdline = self._remove_conan_center_remote_cmdline(
conan_binary)
try:
# Slice the command line because subprocess errors when the first element in the
# list of command strings is the setting of an environment variable.
stdout = subprocess.check_output(
remove_conan_center_remote_cmdline.split()[1:])
self.context.log.debug(stdout)
except subprocess.CalledProcessError as e:
if not "'conan-center' not found in remotes" in e.output:
raise TaskError(
'Error deleting conan-center from conan registry: {}'.
format(e.output))
# Add the pants-specific conan remote.
index_num = 0
for remote_url in reversed(self.get_options().conan_remotes):
index_num += 1
# NB: --insert prepends a remote to conan's remote list. We reverse the options remote
# list to maintain a sensible default for conan emote search order.
add_pants_conan_remote_cmdline = self._add_pants_conan_remote_cmdline(
conan_binary, index_num, remote_url)
try:
stdout = subprocess.check_output(
add_pants_conan_remote_cmdline.split()[1:])
self.context.log.debug(stdout)
except subprocess.CalledProcessError as e:
if not "already exists in remotes" in e.output:
raise TaskError(
'Error adding pants-specific conan remote: {}'.format(
e.output))
def test_pants_binary(self):
command = ['binary', '{}:main'.format(self.fasthello_project)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
# Check that the pex was built.
pex = os.path.join(get_buildroot(), 'dist', 'main.pex')
self.assertTrue(os.path.isfile(pex))
# Check that the pex runs.
output = subprocess.check_output(pex)
self.assertIn('Super hello', output)
# Cleanup
os.remove(pex)
def python_interpreter_path(cls, version):
"""Returns the interpreter path if the current system has the specified version of python.
:param version: A python version string, such as 2.7, 3.
"""
try:
py_path = subprocess.check_output(['python%s' % version,
'-c',
'import sys; print(sys.executable)']).strip()
return os.path.realpath(py_path)
except OSError:
return None
def test_pants_binary(self):
command=['binary', '{}:main'.format(self.fasthello_project)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
# Check that the pex was built.
pex = os.path.join(get_buildroot(), 'dist', 'main.pex')
self.assertTrue(os.path.isfile(pex))
# Check that the pex runs.
output = subprocess.check_output(pex)
self.assertIn('Super hello', output)
# Cleanup
os.remove(pex)
def check_output(self, env=None, **kwargs):
"""Returns the output of the executed Go command.
:param dict env: A custom environment to launch the Go command in. If `None` the current
environment is used.
:param kwargs: Keyword arguments to pass through to `subprocess.check_output`.
:return str: Output of Go command.
:raises subprocess.CalledProcessError: Raises if Go command fails.
"""
env = (env or os.environ).copy()
env.update(self.env)
return subprocess.check_output(self.cmdline, env=env, **kwargs)
def test_with_install_requires(self):
command=['run', '{}:main_with_no_conflict'.format(self.fasthello_install_requires)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
self.assertIn('United States', pants_run.stdout_data)
command=['binary', '{}:main_with_no_conflict'.format(self.fasthello_install_requires)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
pex = os.path.join(get_buildroot(), 'dist', 'main_with_no_conflict.pex')
output = subprocess.check_output(pex)
self.assertIn('United States', output)
os.remove(pex)
def check_output(self, env=None, **kwargs):
"""Returns the output of the executed Go command.
:param dict env: A custom environment to launch the Go command in. If `None` the current
environment is used.
:param kwargs: Keyword arguments to pass through to `subprocess.check_output`.
:return str: Output of Go command.
:raises subprocess.CalledProcessError: Raises if Go command fails.
"""
env = (env or os.environ).copy()
env.update(self.env)
return subprocess.check_output(self.cmdline, env=env, **kwargs)
def test_changed_diffspec_and_files(self):
with create_isolated_git_repo():
git_sha = subprocess.check_output(['git', 'rev-parse', 'HEAD^^']).strip()
stdout = self.assert_changed_new_equals_old(['--changed-diffspec={}'.format(git_sha), '--files'])
# The output should be the files added in the last 2 commits.
self.assertEqual(
lines_to_set(stdout),
{'src/python/python_targets/BUILD',
'src/python/python_targets/test_binary.py',
'src/python/python_targets/test_library.py',
'src/python/python_targets/test_unclaimed_src.py'}
)
def test_pants_binary_dep_isolation_with_multiple_targets(self):
with temporary_dir() as tmp_dir:
pex1 = os.path.join(tmp_dir, 'main_with_no_conflict.pex')
pex2 = os.path.join(tmp_dir, 'main_with_no_pycountry.pex')
command=[
'--pants-distdir={}'.format(tmp_dir),
'binary',
'{}:main_with_no_conflict'.format(self.fasthello_install_requires_dir),
'{}:main_with_no_pycountry'.format(self.fasthello_install_requires_dir)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex1))
self.assertTrue(os.path.isfile(pex2))
# Check that the pex 1 runs.
output = subprocess.check_output(pex1)
self._assert_native_greeting(output)
# Check that the pex 2 fails due to no python_dists leaked into it.
try:
output = subprocess.check_output(pex2)
except subprocess.CalledProcessError as e:
self.assertNotEquals(0, e.returncode)
def python_interpreter_path(version):
"""Returns the interpreter path if the current system has the specified version of python.
:param string version: A python version string, such as 2.7, 3.
:returns: the normalized path to the interpreter binary if found; otherwise `None`
:rtype: string
"""
try:
command = ['python{}'.format(version), '-c', 'import sys; print(sys.executable)']
py_path = subprocess.check_output(command).decode('utf-8').strip()
return os.path.realpath(py_path)
except subprocess.CalledProcessError:
return None
def test_binary_dep_isolation_with_multiple_targets(self):
with temporary_dir() as tmp_dir:
pex1 = os.path.join(tmp_dir, 'main_with_no_conflict.pex')
pex2 = os.path.join(tmp_dir, 'main_with_no_pycountry.pex')
command=[
'--pants-distdir={}'.format(tmp_dir),
'binary',
'{}:main_with_no_conflict'.format(self.fasthello_install_requires_dir),
'{}:main_with_no_pycountry'.format(self.fasthello_install_requires_dir)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex1))
self.assertTrue(os.path.isfile(pex2))
# Check that the pex 1 runs.
output = subprocess.check_output(pex1).decode('utf-8')
self._assert_native_greeting(output)
# Check that the pex 2 fails due to no python_dists leaked into it.
try:
subprocess.check_output(pex2)
except subprocess.CalledProcessError as e:
self.assertNotEqual(0, e.returncode)
def _run_rakudobrew_command(self, argv):
subproc_env = os.environ.copy()
subproc_env['PATH'] = create_path_env_var(self.path_entries,
subproc_env,
prepend=True)
all_argv = ['rakudobrew'] + argv
pretty_printed_argv = safe_shlex_join(all_argv)
try:
return subprocess.check_output(all_argv, env=subproc_env)
except (OSError, subprocess.CalledProcessError) as e:
raise self.RakudoBrewBootstrapError(
"Error with rakudobrew command '{}': {}".format(
pretty_printed_argv, e), e)
def test_pants_binary(self):
pex = os.path.join(get_buildroot(), 'dist', 'main.pex')
try:
command=['binary', '{}:main'.format(self.fasthello_project)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex))
# Check that the pex runs.
output = subprocess.check_output(pex)
self._assert_native_greeting(output)
finally:
if os.path.exists(pex):
os.remove(pex)
def commit_contents_to_files(message, encoding, content, *files):
for path in files:
with safe_open(os.path.join(self.worktree, path), 'w') as fp:
fp.write(content)
subprocess.check_call(['git', 'add', '.'])
subprocess.check_call(['git', 'config', '--local', '--add', 'i18n.commitencoding',
encoding])
try:
subprocess.check_call(['git', 'commit', '-m', message.encode(encoding)])
finally:
subprocess.check_call(['git', 'config', '--local', '--unset-all', 'i18n.commitencoding'])
return subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
def test_python_distribution_with_setup_requires(self):
# Validate that setup_requires dependencies are present and functional.
# PANTS_TEST_SETUP_REQUIRES triggers test functionality in this particular setup.py.
with environment_as(PANTS_TEST_SETUP_REQUIRES='1'):
command = ['run', '{}:main'.format(self.hello_setup_requires)]
pants_run = self.run_pants(command=command)
self.assertRaises(Exception)
# Indicates the pycountry package is available to setup.py script.
self.assertIn('current/setup_requires_site/pycountry/__init__.py',
pants_run.stderr_data)
# Indicates that the pycountry wheel has been installed on PYTHONPATH correctly.
self.assertIn('pycountry-18.5.20.dist-info', pants_run.stderr_data)
# Valdiate the run task. Use clean-all to invalidate cached python_dist wheels from
# previous test run. Use -ldebug to get debug info on setup_requires functionality.
command = [
'-ldebug', 'clean-all', 'run',
'{}:main'.format(self.hello_setup_requires)
]
pants_run = self.run_pants(command=command)
self.assertIn("Installing setup requirements: ['pycountry']",
pants_run.stdout_data)
self.assertIn("Setting PYTHONPATH with setup_requires site directory",
pants_run.stdout_data)
# Validate that the binary can build and run properly. Use clean-all to invalidate cached
# python_dist wheels from previous test run. Use -ldebug to get debug info on setup_requires
# functionality.
pex = os.path.join(get_buildroot(), 'dist', 'main.pex')
try:
command = [
'-ldebug', 'clean-all', 'binary',
'{}:main'.format(self.hello_setup_requires)
]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
self.assertIn("Installing setup requirements: ['pycountry']",
pants_run.stdout_data)
self.assertIn(
"Setting PYTHONPATH with setup_requires site directory",
pants_run.stdout_data)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex))
# Check that the pex runs.
output = subprocess.check_output(pex).decode('utf-8')
self.assertIn('Hello, world!', output)
finally:
if os.path.exists(pex):
# Cleanup.
os.remove(pex)
def python_interpreter_path(version):
"""Returns the interpreter path if the current system has the specified version of python.
:param string version: A python version string, such as 2.7, 3.
:returns: the normalized path to the interpreter binary if found; otherwise `None`
:rtype: string
"""
if PY2:
FileNotFoundError = IOError
try:
command = ['python{}'.format(version), '-c', 'import sys; print(sys.executable)']
py_path = subprocess.check_output(command).decode('utf-8').strip()
return os.path.realpath(py_path)
except (subprocess.CalledProcessError, FileNotFoundError):
return None
def test_with_install_requires(self):
with temporary_dir() as tmp_dir:
pex = os.path.join(tmp_dir, 'main_with_no_conflict.pex')
command=[
'--pants-distdir={}'.format(tmp_dir),
'run',
'{}:main_with_no_conflict'.format(self.fasthello_install_requires_dir)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
self.assertIn('United States', pants_run.stdout_data)
command=['binary', '{}:main_with_no_conflict'.format(self.fasthello_install_requires_dir)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
output = subprocess.check_output(pex)
self.assertIn('United States', output)
def test_with_install_requires(self):
with temporary_dir() as tmp_dir:
pex = os.path.join(tmp_dir, 'main_with_no_conflict.pex')
command=[
'--pants-distdir={}'.format(tmp_dir),
'run',
'{}:main_with_no_conflict'.format(self.fasthello_install_requires)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
self.assertIn('United States', pants_run.stdout_data)
command=['binary', '{}:main_with_no_conflict'.format(self.fasthello_install_requires)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
output = subprocess.check_output(pex)
self.assertIn('United States', output)
def go_fmt_invalid_targets(self, flags):
targets = self.get_targets(self.is_checked)
with self.invalidated(targets) as invalidation_check:
invalid_targets = [vt.target for vt in invalidation_check.invalid_vts]
sources = self.calculate_sources(invalid_targets)
if sources:
args = [os.path.join(self.go_dist.goroot, 'bin', 'gofmt')] + flags + list(sources)
try:
output = subprocess.check_output(args)
except subprocess.CalledProcessError as e:
raise TaskError('{} failed with exit code {}'.format(' '.join(args), e.returncode),
exit_code=e.returncode)
yield output
else:
yield None
def test_pants_binary(self):
with temporary_dir() as tmp_dir:
pex = os.path.join(tmp_dir, 'main.pex')
wheel_glob = os.path.join(tmp_dir, 'fasthello-1.0.0-*.whl')
command=[
'--pants-distdir={}'.format(tmp_dir), 'binary', '{}:main'.format(self.fasthello_project)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex))
# Check that the pex runs.
output = subprocess.check_output(pex)
self._assert_native_greeting(output)
# Check that we have exact one wheel output
self.assertEqual(len(glob.glob(wheel_glob)), 1)
def test_with_install_requires(self):
pex = os.path.join(get_buildroot(), 'dist', 'main_with_no_conflict.pex')
try:
command=['run', '{}:main_with_no_conflict'.format(self.fasthello_install_requires)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
self.assertIn('United States', pants_run.stdout_data)
command=['binary', '{}:main_with_no_conflict'.format(self.fasthello_install_requires)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
output = subprocess.check_output(pex)
self.assertIn('United States', output)
finally:
if os.path.exists(pex):
os.remove(pex)
def _invoke_capturing_output(self, cmd, cwd=None):
if cwd is None:
cwd = self.build_root
toolchain_dirs = self.toolchain.path_entries()
isolated_toolchain_path = get_joined_path(toolchain_dirs)
try:
with environment_as(PATH=isolated_toolchain_path):
return subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
raise Exception(
"Command failed while invoking the native toolchain "
"with code '{code}', cwd='{cwd}', cmd='{cmd}'. Combined stdout and stderr:\n{out}"
.format(code=e.returncode, cwd=cwd, cmd=' '.join(cmd), out=e.output),
e)
def get_subprocess_output(cls, command, ignore_stderr=True, **kwargs):
"""Get the output of an executed command.
:param command: An iterable representing the command to execute (e.g. ['ls', '-al']).
:param ignore_stderr: Whether or not to ignore stderr output vs interleave it with stdout.
:raises: `ProcessManager.ExecutionError` on `OSError` or `CalledProcessError`.
:returns: The output of the command.
"""
if ignore_stderr is False:
kwargs.setdefault('stderr', subprocess.STDOUT)
try:
return subprocess.check_output(command, **kwargs).decode('utf-8').strip()
except (OSError, subprocess.CalledProcessError) as e:
subprocess_output = getattr(e, 'output', '').strip()
raise cls.ExecutionError(str(e), subprocess_output)
def get_subprocess_output(cls, command, ignore_stderr=True, **kwargs):
"""Get the output of an executed command.
:param command: An iterable representing the command to execute (e.g. ['ls', '-al']).
:param ignore_stderr: Whether or not to ignore stderr output vs interleave it with stdout.
:raises: `ProcessManager.ExecutionError` on `OSError` or `CalledProcessError`.
:returns: The output of the command.
"""
if ignore_stderr is False:
kwargs.setdefault('stderr', subprocess.STDOUT)
try:
return subprocess.check_output(command, **kwargs).decode('utf-8').strip()
except (OSError, subprocess.CalledProcessError) as e:
subprocess_output = getattr(e, 'output', '').strip()
raise cls.ExecutionError(str(e), subprocess_output)
def test_pants_binary(self):
with temporary_dir() as tmp_dir:
pex = os.path.join(tmp_dir, 'main.pex')
command=[
'--pants-distdir={}'.format(tmp_dir), 'binary', '{}:main'.format(self.fasthello_project)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex))
# Check that the pex runs.
output = subprocess.check_output(pex).decode('utf-8')
self._assert_native_greeting(output)
# Check that we have exactly one wheel output.
single_wheel_output = assert_single_element(glob.glob(os.path.join(tmp_dir, '*.whl')))
self.assertRegexpMatches(os.path.basename(single_wheel_output),
r'\A{}'.format(re.escape('fasthello-1.0.0+')))
def test_pants_binary(self):
with temporary_dir() as tmp_dir:
pex = os.path.join(tmp_dir, 'main.pex')
wheel_glob = os.path.join(tmp_dir, 'fasthello-1.0.0-*.whl')
command = [
'--pants-distdir={}'.format(tmp_dir), 'binary',
'{}:main'.format(self.fasthello_project)
]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex))
# Check that the pex runs.
output = subprocess.check_output(pex)
self._assert_native_greeting(output)
# Check that we have exact one wheel output
self.assertEqual(len(glob.glob(wheel_glob)), 1)
def _invoke_compiler_exe(self, cmd, env):
try:
# Get stderr interspersed in the error message too -- this should not affect output parsing.
compiler_output = subprocess.check_output(
cmd, env=env, stderr=subprocess.STDOUT).decode('utf-8')
except OSError as e:
# We use `safe_shlex_join` here to pretty-print the command.
raise self.ParseSearchDirsError(
"Process invocation with argv '{}' and environment {!r} failed."
.format(safe_shlex_join(cmd), env), e)
except subprocess.CalledProcessError as e:
raise self.ParseSearchDirsError(
"Process invocation with argv '{}' and environment {!r} exited with non-zero code {}. "
"output:\n{}".format(safe_shlex_join(cmd), env, e.returncode,
e.output), e)
return compiler_output
def _invoke_capturing_output(self, cmd, env=None):
env = env or {}
try:
with environment_as(**env):
return subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode('utf-8')
except subprocess.CalledProcessError as e:
raise Exception(
"Command failed while invoking the native toolchain "
"with code '{code}', cwd='{cwd}', cmd='{cmd}', env='{env}'. "
"Combined stdout and stderr:\n{out}"
.format(code=e.returncode,
cwd=os.getcwd(),
# safe_shlex_join() is just for pretty-printing.
cmd=safe_shlex_join(cmd),
env=env,
out=e.output),
e)
def test_pants_binary(self):
with temporary_dir() as tmp_dir:
pex = os.path.join(tmp_dir, 'main.pex')
# The + is because we append the target's fingerprint to the version. We test this version
# string in test_build_local_python_distributions.py.
wheel_glob = os.path.join(tmp_dir, 'fasthello-1.0.0+*.whl')
command=[
'--pants-distdir={}'.format(tmp_dir), 'binary', '{}:main'.format(self.fasthello_project)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex))
# Check that the pex runs.
output = subprocess.check_output(pex)
self._assert_native_greeting(output)
# Check that we have exact one wheel output
self.assertEqual(len(glob.glob(wheel_glob)), 1)
def _invoke_capturing_output(self, cmd, env=None):
if env is None:
env = os.environ.copy()
try:
with environment_as(**env):
return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
raise Exception(
"Command failed while invoking the native toolchain "
"with code '{code}', cwd='{cwd}', cmd='{cmd}', env='{env}'. "
"Combined stdout and stderr:\n{out}".format(
code=e.returncode,
cwd=os.getcwd(),
# safe_shlex_join() is just for pretty-printing.
cmd=safe_shlex_join(cmd),
env=env,
out=e.output),
e)
def _invoke_compiler_exe(self, cmd, env):
try:
# Get stderr interspersed in the error message too -- this should not affect output parsing.
compiler_output = subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT).decode('utf-8')
except OSError as e:
# We use `safe_shlex_join` here to pretty-print the command.
raise self.ParseSearchDirsError(
"Process invocation with argv '{}' and environment {!r} failed."
.format(safe_shlex_join(cmd), env),
e)
except subprocess.CalledProcessError as e:
raise self.ParseSearchDirsError(
"Process invocation with argv '{}' and environment {!r} exited with non-zero code {}. "
"output:\n{}"
.format(safe_shlex_join(cmd), env, e.returncode, e.output),
e)
return compiler_output
def test_pydist_binary(self):
with temporary_dir() as tmp_dir:
pex = os.path.join(tmp_dir, 'main_with_no_conflict.pex')
command = [
'--pants-distdir={}'.format(tmp_dir),
'binary',
'{}:main_with_no_conflict'.format(self.hello_install_requires_dir),
]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex))
# Check that the pex runs.
output = subprocess.check_output(pex).decode('utf-8')
self._assert_nation_and_greeting(output)
# Check that we have exactly one wheel output.
single_wheel_output = assert_single_element(glob.glob(os.path.join(tmp_dir, '*.whl')))
assertRegex(self, os.path.basename(single_wheel_output),
r'\A{}'.format(re.escape('hello_with_install_requires-1.0.0+')))
def _check_products(self, context, binary, expected_output=None, expected_shebang=None):
pex_name = '{}.pex'.format(binary.address.target_name)
products = context.products.get('deployable_archives')
self.assertIsNotNone(products)
product_data = products.get(binary)
product_basedir = list(product_data.keys())[0]
self.assertEqual(product_data[product_basedir], [pex_name])
# Check pex copy.
pex_copy = os.path.join(self.build_root, 'dist', pex_name)
self.assertTrue(os.path.isfile(pex_copy))
# Check that the pex runs.
output = subprocess.check_output(pex_copy).decode('utf-8')
if expected_output:
self.assertEqual(expected_output, output)
# Check that the pex has the expected shebang.
if expected_shebang:
with open(pex_copy, 'rb') as pex:
line = pex.readline()
self.assertEqual(expected_shebang, line)
def test_python_distribution_with_setup_requires(self):
# Validate that setup_requires dependencies are present and functional.
# PANTS_TEST_SETUP_REQUIRES triggers test functionality in this particular setup.py.
with environment_as(PANTS_TEST_SETUP_REQUIRES='1'):
command=['run', '{}:main'.format(self.hello_setup_requires)]
pants_run = self.run_pants(command=command)
self.assertRaises(Exception)
# Indicates the pycountry package is available to setup.py script.
self.assertIn('current/setup_requires_site/pycountry/__init__.py', pants_run.stderr_data)
# Indicates that the pycountry wheel has been installed on PYTHONPATH correctly.
self.assertIn('pycountry-18.5.20.dist-info', pants_run.stderr_data)
# Valdiate the run task. Use clean-all to invalidate cached python_dist wheels from
# previous test run. Use -ldebug to get debug info on setup_requires functionality.
command=['-ldebug', 'clean-all', 'run', '{}:main'.format(self.hello_setup_requires)]
pants_run = self.run_pants(command=command)
self.assertIn("Installing setup requirements: ['pycountry']", pants_run.stdout_data)
self.assertIn("Setting PYTHONPATH with setup_requires site directory", pants_run.stdout_data)
# Validate that the binary can build and run properly. Use clean-all to invalidate cached
# python_dist wheels from previous test run. Use -ldebug to get debug info on setup_requires
# functionality.
pex = os.path.join(get_buildroot(), 'dist', 'main.pex')
try:
command=['-ldebug', 'clean-all', 'binary', '{}:main'.format(self.hello_setup_requires)]
pants_run = self.run_pants(command=command)
self.assert_success(pants_run)
self.assertIn("Installing setup requirements: ['pycountry']", pants_run.stdout_data)
self.assertIn("Setting PYTHONPATH with setup_requires site directory", pants_run.stdout_data)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex))
# Check that the pex runs.
output = subprocess.check_output(pex)
self.assertIn('Hello, world!', output)
finally:
if os.path.exists(pex):
# Cleanup.
os.remove(pex)
def test_pants_resolves_local_dists_for_current_platform_only(self):
# Test that pants will override pants.ini platforms config when building
# or running a target that depends on native (c or cpp) sources.
pex = os.path.join(get_buildroot(), 'dist', 'main.pex')
pants_ini_config = {'python-setup': {'platforms': ['current', 'linux-x86_64']}}
try:
# Clean all to rebuild requirements pex.
command=['clean-all', 'run', '{}:main'.format(self.fasthello_project)]
pants_run = self.run_pants(command=command, config=pants_ini_config)
self.assert_success(pants_run)
command=['binary', '{}:main'.format(self.fasthello_project)]
pants_run = self.run_pants(command=command, config=pants_ini_config)
self.assert_success(pants_run)
# Check that the pex was built.
self.assertTrue(os.path.isfile(pex))
# Check that the pex runs.
output = subprocess.check_output(pex)
self._assert_native_greeting(output)
finally:
if os.path.exists(pex):
# Cleanup
os.remove(pex)
def assert_no_gopath(self):
go_distribution = self.distribution()
go_env = go_distribution.go_env()
# As of go 1.8, when GOPATH is unset (set to ''), it defaults to ~/go (assuming HOME is set -
# and we can't unset that since it might legitmately be used by the subcommand) - so we manually
# fetch the "unset" default value here as our expected value for tests below.
# The key thing to note here is this default value is used only when `gopath` passed to
# `GoDistribution` is None, implying the command to be run does not need or use a GOPATH.
cmd = [os.path.join(go_distribution.goroot, 'bin', 'go'), 'env', 'GOPATH']
env = os.environ.copy()
env.update(go_env)
default_gopath = subprocess.check_output(cmd, env=env).strip()
go_cmd = go_distribution.create_go_cmd(cmd='env', args=['GOPATH'])
self.assertEqual(go_env, go_cmd.env)
self.assertEqual('go', os.path.basename(go_cmd.cmdline[0]))
self.assertEqual(['env', 'GOPATH'], go_cmd.cmdline[1:])
self.assertRegexpMatches(str(go_cmd),
r'^GOROOT=[^ ]+ GOPATH={} .*/go env GOPATH'.format(default_gopath))
self.assertEqual(default_gopath, go_cmd.check_output().strip())
def test_changelog_utf8(self):
with environment_as(GIT_DIR=self.gitdir, GIT_WORK_TREE=self.worktree):
def commit_contents_to_files(message, encoding, content, *files):
for path in files:
with safe_open(os.path.join(self.worktree, path), 'w') as fp:
fp.write(content)
subprocess.check_call(['git', 'add', '.'])
subprocess.check_call(['git', 'config', '--local', '--add', 'i18n.commitencoding',
encoding])
try:
subprocess.check_call(['git', 'commit', '-m', message.encode(encoding)])
finally:
subprocess.check_call(['git', 'config', '--local', '--unset-all', 'i18n.commitencoding'])
return subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip()
# Mix in a non-UTF-8 author to all commits to exercise the corner described here does not
# adversely impact the ability to render the changelog (even if rendering for certain
# characters is incorrect): http://comments.gmane.org/gmane.comp.version-control.git/262685
# NB: This method of override requires we include `user.name` and `user.email` even though we
# only use `user.name` to exercise non-UTF-8. Without `user.email`, it will be unset and
# commits can then fail on machines without a proper hostname setup for git to fall back to
# when concocting a last-ditch `user.email`.
non_utf8_config = dedent("""
[user]
name = Noralf Trønnes
email = [email protected]
""").encode('iso-8859-1')
with open(os.path.join(self.gitdir, 'config'), 'wb') as fp:
fp.write(non_utf8_config)
# Note the copyright symbol is used as the non-ascii character in the next 3 commits
commit_contents_to_files('START1 © END', 'iso-8859-1', '1', 'foo')
commit_contents_to_files('START2 © END', 'latin1', '1', 'bar')
commit_contents_to_files('START3 © END', 'utf-8', '1', 'baz')
commit_contents_to_files('START4 ~ END', 'us-ascii', '1', 'bip')
# Prove our non-utf-8 encodings were stored in the commit metadata.
log = subprocess.check_output(['git', 'log', '--format=%e'])
self.assertEqual(['us-ascii', 'latin1', 'iso-8859-1'], filter(None, log.strip().splitlines()))
# And show that the git log successfully transcodes all the commits none-the-less to utf-8
changelog = self.git.changelog()
# The ascii commit should combine with the iso-8859-1 author an fail to transcode the
# o-with-stroke character, and so it should be replaced with the utf-8 replacement character
# \uFFF or �.
self.assertIn('Noralf Tr�nnes', changelog)
self.assertIn('Noralf Tr\uFFFDnnes', changelog)
# For the other 3 commits, each of iso-8859-1, latin1 and utf-8 have an encoding for the
# o-with-stroke character - \u00F8 or ø - so we should find it;
self.assertIn('Noralf Trønnes', changelog)
self.assertIn('Noralf Tr\u00F8nnes', changelog)
self.assertIn('START1 © END', changelog)
self.assertIn('START2 © END', changelog)
self.assertIn('START3 © END', changelog)
self.assertIn('START4 ~ END', changelog)