def test_pex_builder():
# test w/ and w/o zipfile dists
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
write_pex(td, exe_main, dists=[p1])
success_txt = os.path.join(td, 'success.txt')
PEX(td).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
# test w/ and w/o zipfile dists
with nested(temporary_dir(), temporary_dir(), make_bdist('p1', zipped=True)) as (
td1, td2, p1):
target_egg_dir = os.path.join(td2, os.path.basename(p1.location))
safe_mkdir(target_egg_dir)
with closing(zipfile.ZipFile(p1.location, 'r')) as zf:
zf.extractall(target_egg_dir)
p1 = DistributionHelper.distribution_from_path(target_egg_dir)
write_pex(td1, exe_main, dists=[p1])
success_txt = os.path.join(td1, 'success.txt')
PEX(td1).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def test_nested_requirements():
with temporary_dir() as td1:
with temporary_dir() as td2:
with open(os.path.join(td1, 'requirements.txt'), 'w') as fp:
fp.write(dedent('''
requirement1
requirement2
-r %s
-r %s
''' % (
os.path.join(td2, 'requirements_nonrelative.txt'),
os.path.join('relative', 'requirements_relative.txt'))
))
with open(os.path.join(td2, 'requirements_nonrelative.txt'), 'w') as fp:
fp.write(dedent('''
requirement3
requirement4
'''))
os.mkdir(os.path.join(td1, 'relative'))
with open(os.path.join(td1, 'relative', 'requirements_relative.txt'), 'w') as fp:
fp.write(dedent('''
requirement5
requirement6
'''))
def rr(req):
return ResolvableRequirement.from_string(req, ResolverOptionsBuilder())
reqs = requirements_from_file(os.path.join(td1, 'requirements.txt'))
assert reqs == [rr('requirement%d' % k) for k in (1, 2, 3, 4, 5, 6)]
def test_resolve_prereleases_cached():
stable_dep = make_sdist(name='dep', version='2.0.0')
prerelease_dep = make_sdist(name='dep', version='3.0.0rc3')
with temporary_dir() as td:
for sdist in (stable_dep, prerelease_dep):
safe_copy(sdist, os.path.join(td, os.path.basename(sdist)))
fetchers = [Fetcher([td])]
with temporary_dir() as cd:
def assert_resolve(dep, expected_version, **resolve_kwargs):
dists = list(
resolve_multi([dep], cache=cd, cache_ttl=1000, **resolve_kwargs)
)
assert 1 == len(dists)
dist = dists[0]
assert expected_version == dist.version
Crawler.reset_cache()
# First do a run to load it into the cache.
assert_resolve('dep>=1,<4', '3.0.0rc3', allow_prereleases=True, fetchers=fetchers)
# This simulates running from another pex command. The Crawler cache actually caches an empty
# cache so this fails in the same "process".
Crawler.reset_cache()
# Now assert that we can get it from the cache by removing the source.
assert_resolve('dep>=1,<4', '3.0.0rc3', allow_prereleases=True, fetchers=[])
# It should also be able to resolve without allow_prereleases, if explicitly requested.
Crawler.reset_cache()
assert_resolve('dep>=1.rc1,<4', '3.0.0rc3', fetchers=[])
def test_pex_builder_compilation():
with nested(temporary_dir(), temporary_dir(), temporary_dir()) as (td1, td2, td3):
src = os.path.join(td1, 'src.py')
with open(src, 'w') as fp:
fp.write(exe_main)
exe = os.path.join(td1, 'exe.py')
with open(exe, 'w') as fp:
fp.write(exe_main)
def build_and_check(path, precompile):
pb = PEXBuilder(path)
pb.add_source(src, 'lib/src.py')
pb.set_executable(exe, 'exe.py')
pb.freeze(bytecode_compile=precompile)
for pyc_file in ('exe.pyc', 'lib/src.pyc', '__main__.pyc'):
pyc_exists = os.path.exists(os.path.join(path, pyc_file))
if precompile:
assert pyc_exists
else:
assert not pyc_exists
bootstrap_dir = os.path.join(path, PEXBuilder.BOOTSTRAP_DIR)
bootstrap_pycs = []
for _, _, files in os.walk(bootstrap_dir):
bootstrap_pycs.extend(f for f in files if f.endswith('.pyc'))
if precompile:
assert len(bootstrap_pycs) > 0
else:
assert 0 == len(bootstrap_pycs)
build_and_check(td2, False)
build_and_check(td3, True)
def test_access_zipped_assets_integration():
test_executable = dedent('''
import os
from _pex.util import DistributionHelper
temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule')
with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp:
for line in fp:
print(line)
''')
with nested(temporary_dir(), temporary_dir()) as (td1, td2):
pb = PEXBuilder(path=td1)
with open(os.path.join(td1, 'exe.py'), 'w') as fp:
fp.write(test_executable)
pb.set_executable(fp.name)
submodule = os.path.join(td1, 'my_package', 'submodule')
safe_mkdir(submodule)
mod_path = os.path.join(submodule, 'mod.py')
with open(mod_path, 'w') as fp:
fp.write('accessed')
pb.add_source(fp.name, 'my_package/submodule/mod.py')
pex = os.path.join(td2, 'app.pex')
pb.build(pex)
output, returncode = run_simple_pex(pex)
try:
output = output.decode('UTF-8')
except ValueError:
pass
assert output == 'accessed\n'
assert returncode == 0
def test_source_packages():
for ext in ('.tar.gz', '.tar', '.tgz', '.zip', '.tar.bz2'):
sl = SourcePackage('a_p_r-3.1.3' + ext)
assert sl._name == 'a_p_r'
assert sl.name == 'a-p-r'
assert sl.raw_version == '3.1.3'
assert sl.version == parse_version(sl.raw_version)
for req in ('a_p_r', 'a_p_r>2', 'a_p_r>3', 'a_p_r>=3.1.3', 'a_p_r==3.1.3', 'a_p_r>3,<3.5'):
assert sl.satisfies(req)
assert sl.satisfies(Requirement.parse(req))
for req in ('foo', 'a_p_r==4.0.0', 'a_p_r>4.0.0', 'a_p_r>3.0.0,<3.0.3', 'a==3.1.3'):
assert not sl.satisfies(req)
sl = SourcePackage('python-dateutil-1.5.tar.gz')
assert sl.name == 'python-dateutil'
assert sl.raw_version == '1.5'
with temporary_dir() as td:
dateutil_base = 'python-dateutil-1.5'
dateutil = '%s.zip' % dateutil_base
with contextlib.closing(ZipFile(os.path.join(td, dateutil), 'w')) as zf:
zf.writestr(os.path.join(dateutil_base, 'file1.txt'), 'junk1')
zf.writestr(os.path.join(dateutil_base, 'file2.txt'), 'junk2')
sl = SourcePackage('file://' + os.path.join(td, dateutil), opener=Web())
with temporary_dir() as td2:
sl.fetch(location=td2)
print(os.listdir(td2))
assert set(os.listdir(os.path.join(td2, dateutil_base))) == set(['file1.txt', 'file2.txt'])
def test_import_export(self):
with temporary_dir() as state_dir:
with temporary_dir() as server_dir1:
with TestServer(config=dict(serverdir=server_dir1)) as devpi:
self.assertEqual(200, requests.get(devpi.url).status_code)
export_state(server_dir1, state_dir)
with temporary_dir() as server_dir2:
import_state(server_dir2, state_dir)
with TestServer(config=dict(serverdir=server_dir2)) as devpi:
self.assertEqual(200, requests.get(devpi.url).status_code)
def test_diamond_local_resolve_cached():
# This exercises the issue described here: https://github.com/pantsbuild/pex/issues/120
project1_sdist = make_sdist(name='project1', install_reqs=['project2<1.0.0'])
project2_sdist = make_sdist(name='project2')
with temporary_dir() as dd:
for sdist in (project1_sdist, project2_sdist):
safe_copy(sdist, os.path.join(dd, os.path.basename(sdist)))
fetchers = [Fetcher([dd])]
with temporary_dir() as cd:
dists = resolve(['project1', 'project2'], fetchers=fetchers, cache=cd, cache_ttl=1000)
assert len(dists) == 2
def test_nested_pushd():
pre_cwd = os.getcwd()
with temporary_dir() as tempdir1:
with pushd(tempdir1) as path1:
assert os.getcwd() == os.path.realpath(tempdir1)
with temporary_dir(root_dir=tempdir1) as tempdir2:
with pushd(tempdir2) as path2:
assert os.getcwd() == os.path.realpath(tempdir2)
assert os.getcwd() == os.path.realpath(tempdir1)
assert os.getcwd() == os.path.realpath(tempdir1)
assert os.getcwd() == pre_cwd
assert os.getcwd() == pre_cwd
def test_find_root_thrifts(self):
with temporary_dir() as dir:
root_1 = self.write(os.path.join(dir, 'root_1.thrift'), '# noop')
root_2 = self.write(os.path.join(dir, 'root_2.thrift'), '# noop')
self.assertEquals(set([root_1, root_2]),
find_root_thrifts(basedirs=[], sources=[root_1, root_2]))
with temporary_dir() as dir:
root_1 = self.write(os.path.join(dir, 'root_1.thrift'), 'include "mid_1.thrift"')
self.write(os.path.join(dir, 'mid_1.thrift'), 'include "leaf_1.thrift"')
self.write(os.path.join(dir, 'leaf_1.thrift'), '# noop')
root_2 = self.write(os.path.join(dir, 'root_2.thrift'), 'include "root_1.thrift"')
self.assertEquals(set([root_2]), find_root_thrifts(basedirs=[], sources=[root_1, root_2]))
def test_round_trip(prefix=None):
with temporary_dir() as fromdir:
safe_mkdir(os.path.join(fromdir, 'a/b/c'))
touch(os.path.join(fromdir, 'a/b/d/e.txt'))
with temporary_dir() as archivedir:
archive = archiver.create(fromdir, archivedir, 'archive', prefix=prefix)
with temporary_dir() as todir:
archiver.extract(archive, todir)
fromlisting = listtree(fromdir)
if prefix:
fromlisting = set(os.path.join(prefix, x) for x in fromlisting)
if empty_dirs:
fromlisting.add(prefix)
self.assertEqual(fromlisting, listtree(todir))
def test_pex_root():
with temporary_dir() as tmp_home:
with environment_as(HOME=tmp_home):
with temporary_dir() as td:
with temporary_dir() as output_dir:
env = os.environ.copy()
env['PEX_INTERPRETER'] = '1'
output_path = os.path.join(output_dir, 'pex.pex')
args = ['pex', '-o', output_path, '--not-zip-safe', '--pex-root={0}'.format(td)]
results = run_pex_command(args=args, env=env)
results.assert_success()
assert ['pex.pex'] == os.listdir(output_dir), 'Expected built pex file.'
assert [] == os.listdir(tmp_home), 'Expected empty temp home dir.'
assert 'build' in os.listdir(td), 'Expected build directory in tmp pex root.'
def test_create_ioerror(chown):
chown.side_effect = IOError('Disk is borked')
with temporary_dir() as d:
ds = DirectorySandbox(d)
with pytest.raises(DirectorySandbox.CreationError):
ds.create()
def test_killTask_during_runner_initialize(self): # noqa
proxy_driver = ProxyDriver()
task = make_task(HELLO_WORLD_MTI)
with temporary_dir() as td:
te = FastThermosExecutor(
runner_provider=make_provider(td),
sandbox_provider=SlowSandboxProvider())
te.launchTask(proxy_driver, task)
te.sandbox_initialized.wait()
te.killTask(proxy_driver, mesos_pb2.TaskID(value=task.task_id.value))
assert te.runner_aborted.is_set()
assert not te.sandbox_created.is_set()
# we've simulated a "slow" initialization by blocking it until the killTask was sent - so now,
# trigger the initialization to complete
te._sandbox._init_start.set()
# however, wait on the runner to definitely finish its initialization before continuing
# (otherwise, this function races ahead too fast)
te._sandbox._init_done.wait()
te.sandbox_created.wait()
assert te.sandbox_initialized.is_set()
assert te.sandbox_created.is_set()
proxy_driver.wait_stopped()
updates = proxy_driver.method_calls['sendStatusUpdate']
assert len(updates) == 2
assert updates[-1][0][0].state == mesos_pb2.TASK_KILLED
def test_filesystem_image_containerizer_not_executable(self):
proxy_driver = ProxyDriver()
with temporary_dir() as tempdir:
tempfile = os.path.join(tempdir, 'fake-containierizer')
with open(tempfile, 'a'):
os.utime(tempfile, None)
te = FastThermosExecutor(
runner_provider=make_provider(tempdir, mesos_containerizer_path=tempfile),
sandbox_provider=FileSystemImageTestSandboxProvider())
te.SANDBOX_INITIALIZATION_TIMEOUT = Amount(1, Time.MILLISECONDS)
te.START_TIMEOUT = Amount(10, Time.MILLISECONDS)
te.STOP_TIMEOUT = Amount(10, Time.MILLISECONDS)
te.launchTask(proxy_driver, make_task(HELLO_WORLD_MTI))
proxy_driver.wait_stopped()
updates = proxy_driver.method_calls['sendStatusUpdate']
assert len(updates) == 2
assert updates[0][0][0].state == mesos_pb2.TASK_STARTING
assert updates[1][0][0].state == mesos_pb2.TASK_FAILED
def test_sibling_references(self):
with temporary_dir() as root_dir:
buildfile = create_buildfile(root_dir, 'a', name='BUILD',
content=dedent("""
dependencies(name='util',
dependencies=[
jar(org='com.twitter', name='util', rev='0.0.1')
]
)
""").strip()
)
sibling = create_buildfile(root_dir, 'a', name='BUILD.sibling',
content=dedent("""
dependencies(name='util-ex',
dependencies=[
pants(':util'),
jar(org='com.twitter', name='util-ex', rev='0.0.1')
]
)
""").strip()
)
ParseContext(buildfile).parse()
utilex = Target.get(Address.parse(root_dir, 'a:util-ex', is_relative=False))
utilex_deps = set(utilex.resolve())
util = Target.get(Address.parse(root_dir, 'a:util', is_relative=False))
util_deps = set(util.resolve())
self.assertEquals(util_deps, util_deps.intersection(utilex_deps))
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
def test_verify_group_match(mock_check_output):
with temporary_dir() as d:
sandbox = FileSystemImageSandbox(d, user='******', sandbox_mount_point='/some/path')
mock_check_output.return_value = 'test-group:x:2:'
# valid case
sandbox._verify_group_match_in_taskfs(2, 'test-group')
mock_check_output.assert_called_with(
['chroot', sandbox._task_fs_root, 'getent', 'group', 'test-group'])
# invalid group id
with pytest.raises(FileSystemImageSandbox.CreationError):
sandbox._verify_group_match_in_taskfs(3, 'test-group')
# invalid group name
with pytest.raises(FileSystemImageSandbox.CreationError):
sandbox._verify_group_match_in_taskfs(2, 'invalid-group')
# exception case
exception = subprocess.CalledProcessError(
returncode=1,
cmd='some command',
output=None)
mock_check_output.side_effect = exception
with pytest.raises(FileSystemImageSandbox.CreationError):
sandbox._verify_group_match_in_taskfs(2, 'test-group')
def TestServer(users={}, indices={}, config={}, fail_on_output=['Traceback']):
"""
Starts a devpi server to be used within tests.
"""
with temporary_dir() as server_dir:
server_options = {
'port': 2414,
'serverdir': server_dir}
server_options.update(config)
if 'serverdir' not in config:
prefill_serverdir(server_options)
with DevpiServer(server_options) as url:
with DevpiClient(url, 'root', '') as client:
for user, kwargs in iteritems(users):
client.create_user(user, **kwargs)
for index, kwargs in iteritems(indices):
client.create_index(index, **kwargs)
yield client
_assert_no_logged_errors(fail_on_output, server_options['serverdir'] + '/.xproc/devpi-server/xprocess.log')
def test_empty_resolve():
empty_resolve = resolve([])
assert empty_resolve == []
with temporary_dir() as td:
empty_resolve = resolve([], cache=td)
assert empty_resolve == []
def test_empty_resolve():
empty_resolve_multi = list(resolve_multi([]))
assert empty_resolve_multi == []
with temporary_dir() as td:
empty_resolve_multi = list(resolve_multi([], cache=td))
assert empty_resolve_multi == []
def test_resolve_prereleases_and_no_version():
prerelease_dep = make_sdist(name='dep', version='3.0.0rc3')
with temporary_dir() as td:
safe_copy(prerelease_dep, os.path.join(td, os.path.basename(prerelease_dep)))
fetchers = [Fetcher([td])]
def assert_resolve(deps, expected_version, **resolve_kwargs):
dists = list(
resolve_multi(deps, fetchers=fetchers, **resolve_kwargs)
)
assert 1 == len(dists)
dist = dists[0]
assert expected_version == dist.version
# When allow_prereleases is specified, the requirement (from two dependencies)
# for a specific pre-release version and no version specified, accepts the pre-release
# version correctly.
assert_resolve(['dep==3.0.0rc3', 'dep'], '3.0.0rc3', allow_prereleases=True)
# Without allow_prereleases set, the pre-release version is rejected.
# This used to be an issue when a command-line use did not pass the `--pre` option
# correctly into the API call for resolve_multi() from build_pex() in pex.py.
with pytest.raises(Unsatisfiable):
assert_resolve(['dep==3.0.0rc3', 'dep'], '3.0.0rc3')
def test_log_tee():
with temporary_dir() as td:
taskpath = make_taskpath(td)
sandbox = setup_sandbox(td, taskpath)
# Create file stdout for capturing output. We can't use StringIO mock
# because TestProcess is running fork.
with open(os.path.join(td, "sys_stdout"), "w+") as stdout:
with open(os.path.join(td, "sys_stderr"), "w+") as stderr:
with mutable_sys():
sys.stdout, sys.stderr = stdout, stderr
p = TestProcess(
"process",
"echo hello world; echo >&2 hello stderr",
0,
taskpath,
sandbox,
logger_destination=LoggerDestination.BOTH,
)
p.start()
rc = wait_for_rc(taskpath.getpath("process_checkpoint"))
assert rc == 0
# Check log files were created in std path with correct content
assert_log_content(taskpath, "stdout", "hello world\n")
assert_log_content(taskpath, "stderr", "hello stderr\n")
# Check mock stdout
stdout.seek(0)
assert stdout.read() == "hello world\n"
# Check mock stderr
stderr.seek(0)
assert stderr.read() == "hello stderr\n"
def test_simple_process_filesystem_isolator():
with temporary_dir() as td:
taskpath = make_taskpath(td)
sandbox = setup_sandbox(td, taskpath)
test_isolator_path = os.path.join(td, "fake-mesos-containerier")
with open(test_isolator_path, "w") as fd:
# We use a fake version of the mesos-containerizer binary that just echoes out its args so
# we can assert on them in the process's output.
fd.write("\n".join(["#!/bin/sh", 'echo "$@"']))
fd.close()
chmod_plus_x(test_isolator_path)
p = TestProcess(
"process", "echo hello world", 0, taskpath, sandbox, mesos_containerizer_path=test_isolator_path
)
p.start()
rc = wait_for_rc(taskpath.getpath("process_checkpoint"))
assert rc == 0
assert_log_content(
taskpath,
"stdout",
"launch --unshare_namespace_mnt --rootfs=/some/path/taskfs --user=None "
'--command={"shell":true,"value":"echo hello world"}\n',
)
def test_verify_user_match(mock_check_output):
with temporary_dir() as d:
sandbox = FileSystemImageSandbox(os.path.join(d, "sandbox"), user="******", sandbox_mount_point="/some/path")
mock_check_output.return_value = "uid=1(test-user) gid=2(test-group) groups=2(test-group)"
# valid case
sandbox._verify_user_match_in_taskfs(1, "test-user", 2, "test-group")
mock_check_output.assert_called_with(["chroot", sandbox._task_fs_root, "id", "test-user"])
# invalid user id
with pytest.raises(FileSystemImageSandbox.CreationError):
sandbox._verify_user_match_in_taskfs(0, "test-user", 2, "test-group")
# invalid user name
with pytest.raises(FileSystemImageSandbox.CreationError):
sandbox._verify_user_match_in_taskfs(1, "invalid-user", 2, "test-group")
# invalid group id
with pytest.raises(FileSystemImageSandbox.CreationError):
sandbox._verify_user_match_in_taskfs(1, "test-user", 0, "test-group")
# invalid group name
with pytest.raises(FileSystemImageSandbox.CreationError):
sandbox._verify_user_match_in_taskfs(1, "test-user", 2, "invalid-group")
# exception case
exception = subprocess.CalledProcessError(returncode=1, cmd="some command", output=None)
mock_check_output.side_effect = exception
with pytest.raises(FileSystemImageSandbox.CreationError):
sandbox._verify_user_match_in_taskfs(1, "test-user", 2, "test-group")
def test_load_invalid_syntax():
with temporary_dir() as td:
# bad json
clusters_json = os.path.join(td, 'clusters.json')
with open(clusters_json, 'w') as fp:
fp.write('This is not json')
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_json)
# bad yaml
clusters_yml = os.path.join(td, 'clusters.yml')
with open(clusters_yml, 'w') as fp:
fp.write('L{}L')
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_yml)
# bad layout
clusters_yml = os.path.join(td, 'clusters.yml')
with open(clusters_yml, 'w') as fp:
fp.write('just a string')
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_yml)
# not a dict
clusters_json = os.path.join(td, 'clusters.json')
with open(clusters_json, 'w') as fp:
fp.write(json.dumps({'cluster1': ['not', 'cluster', 'values']}))
with pytest.raises(Clusters.ParseError):
Clusters.from_file(clusters_json)
def test_crawler_local():
FL = ('a.txt', 'b.txt', 'c.txt')
with temporary_dir() as td:
for fn in FL:
with open(os.path.join(td, fn), 'w') as fp:
pass
for dn in (1, 2):
os.mkdir(os.path.join(td, 'dir%d' % dn))
for fn in FL:
with open(os.path.join(td, 'dir%d' % dn, fn), 'w') as fp:
pass
# basic file / dir rel splitting
links, rels = Crawler(enable_cache=False).execute(td)
assert set(links) == set(os.path.join(td, fn) for fn in FL)
assert set(rels) == set(os.path.join(td, 'dir%d' % n) for n in (1, 2))
# recursive crawling, single vs multi-threaded
for caching in (False, True):
for threads in (1, 2, 3):
links = Crawler(enable_cache=caching, threads=threads).crawl([td])
expect_links = (set(os.path.join(td, fn) for fn in FL) |
set(os.path.join(td, 'dir1', fn) for fn in FL) |
set(os.path.join(td, 'dir2', fn) for fn in FL))
assert set(links) == expect_links
def relativize_analysis_file(self, src, dst):
# Make an analysis cache portable. Work on a tmpfile, for safety.
#
# NOTE: We can't port references to deps on the Java home. This is because different JVM
# implementations on different systems have different structures, and there's not
# necessarily a 1-1 mapping between Java jars on different systems. Instead we simply
# drop those references from the analysis file.
#
# In practice the JVM changes rarely, and it should be fine to require a full rebuild
# in those rare cases.
with temporary_dir() as tmp_analysis_dir:
stripped_src = os.path.join(tmp_analysis_dir, 'analysis.nojava')
tmp_analysis_file = os.path.join(tmp_analysis_dir, 'analysis.relativized')
# Strip all lines containing self._java_home.
with open(src, 'r') as infile:
with open (stripped_src, 'w') as outfile:
for line in infile:
if not self._java_home in line:
outfile.write(line)
rebasings = [
(self._ivy_home, ZincUtils.IVY_HOME_PLACEHOLDER),
(self._pants_home, ZincUtils.PANTS_HOME_PLACEHOLDER),
]
Analysis.rebase(stripped_src, tmp_analysis_file, rebasings)
shutil.move(tmp_analysis_file, dst)
def yield_runner(self, runner_class, **bindings):
with contextlib.nested(temporary_dir(), temporary_dir()) as (td1, td2):
sandbox = DirectorySandbox(td1)
checkpoint_root = td2
task_runner = runner_class(
runner_pex=os.path.join("dist", "thermos_runner.pex"),
task_id="hello_world",
task=TASK.bind(**bindings).task(),
role=getpass.getuser(),
portmap={},
sandbox=sandbox,
checkpoint_root=checkpoint_root,
)
yield task_runner
def test_log_console():
with temporary_dir() as td:
taskpath = make_taskpath(td)
sandbox = setup_sandbox(td, taskpath)
# Create file stdout for capturing output. We can't use StringIO mock
# because TestProcess is running fork.
with open(os.path.join(td, 'sys_stdout'), 'w+') as stdout:
with open(os.path.join(td, 'sys_stderr'), 'w+') as stderr:
with mutable_sys():
sys.stdout, sys.stderr = stdout, stderr
p = TestProcess('process', 'echo hello world; echo >&2 hello stderr', 0,
taskpath, sandbox, logger_destination=LoggerDestination.CONSOLE)
p.start()
rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
assert rc == 0
# Check no log files were created in std path
assert_log_dne(taskpath, 'stdout')
assert_log_dne(taskpath, 'stderr')
# Check mock stdout
stdout.seek(0)
assert stdout.read() == 'hello world\n'
# Check mock stderr
stderr.seek(0)
assert stderr.read() == 'hello stderr\n'
def inherit_path(inherit_path):
with temporary_dir() as output_dir:
exe = os.path.join(output_dir, 'exe.py')
body = "import sys ; print('\\n'.join(sys.path))"
with open(exe, 'w') as f:
f.write(body)
pex_path = os.path.join(output_dir, 'pex.pex')
results = run_pex_command([
'--disable-cache',
'msgpack_python',
'--inherit-path{}'.format(inherit_path),
'-o',
pex_path,
])
results.assert_success()
env = os.environ.copy()
env["PYTHONPATH"] = "/doesnotexist"
stdout, rc = run_simple_pex(
pex_path,
args=(exe,),
env=env,
)
assert rc == 0
stdout_lines = stdout.decode().split('\n')
requests_paths = tuple(i for i, l in enumerate(stdout_lines) if 'msgpack_python' in l)
sys_paths = tuple(i for i, l in enumerate(stdout_lines) if 'doesnotexist' in l)
assert len(requests_paths) == 1
assert len(sys_paths) == 1
if inherit_path == "=fallback":
assert requests_paths[0] < sys_paths[0]
else:
assert requests_paths[0] > sys_paths[0]
def test_simple_process_filesystem_isolator_launch_info():
with temporary_dir() as td:
taskpath = make_taskpath(td)
sandbox = setup_sandbox(td, taskpath)
test_isolator_path = os.path.join(td, 'fake-mesos-containerier')
with open(test_isolator_path, 'w') as fd:
# We use a fake version of the mesos-containerizer binary that just echoes out its args so
# we can assert on them in the process's output. Also imitates a failure when there are not
# enough arguments, this is used to find the version of the binary (by checking the failure
# message)
fd.write('\n'.join([
'#!/bin/sh', 'if [ "$#" -eq 1 ]; then',
' { echo "launch_info" >&2; };', 'else', ' echo "$@";', 'fi'
]))
fd.close()
chmod_plus_x(test_isolator_path)
p = FakeProcess('process',
'echo hello world',
0,
taskpath,
sandbox,
mesos_containerizer_path=test_isolator_path,
container_sandbox=sandbox)
p.start()
rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
assert rc == 0
assert_log_content(
taskpath, 'stdout',
'launch --unshare_namespace_mnt --launch_info={"command": {"shell": false, "arguments": '
'["/bin/bash", "-c", "echo hello world"], "value": "/bin/bash"}, '
'"working_directory": "%s", "user": "******", "rootfs": "/some/path/taskfs"}\n'
% (sandbox))
def _render_jar_tool_args(self):
args = []
if self._main:
args.append('-main=%s' % self._main)
if self._classpath:
args.append('-classpath=%s' % ','.join(self._classpath))
with temporary_dir() as stage_dir:
if self._manifest:
args.append('-manifest=%s' % self._manifest.materialize(stage_dir))
if self._entries:
def as_cli_entry(entry):
src = entry.materialize(stage_dir)
return '%s=%s' % (src, entry.dest) if entry.dest else src
args.append('-files=%s' % ','.join(map(as_cli_entry, self._entries)))
if self._jars:
args.append('-jars=%s' % ','.join(self._jars))
yield args
def test_pex_exec_with_pex_python_path_only():
with temporary_dir() as td:
pexrc_path = os.path.join(td, '.pexrc')
with open(pexrc_path, 'w') as pexrc:
# set pex python path
pex_python_path = ':'.join([
ensure_python_interpreter('2.7.10'),
ensure_python_interpreter('3.6.3')
])
pexrc.write("PEX_PYTHON_PATH=%s" % pex_python_path)
pex_out_path = os.path.join(td, 'pex.pex')
res = run_pex_command([
'--disable-cache',
'--rcfile=%s' % pexrc_path, '-o', pex_out_path
])
res.assert_success()
# test that pex bootstrapper selects lowest version interpreter
# in pex python path (python2.7)
stdin_payload = b'import sys; print(sys.executable); sys.exit(0)'
stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload)
assert rc == 0
assert str(pex_python_path.split(':')[0]).encode() in stdout
def test_filesystem_image_containerizer_not_executable(self):
proxy_driver = ProxyDriver()
with temporary_dir() as tempdir:
tempfile = os.path.join(tempdir, 'fake-containierizer')
with open(tempfile, 'a'):
os.utime(tempfile, None)
te = FastThermosExecutor(
runner_provider=make_provider(tempdir, mesos_containerizer_path=tempfile),
sandbox_provider=FileSystemImageTestSandboxProvider(), stop_timeout_in_secs=1)
te.SANDBOX_INITIALIZATION_TIMEOUT = Amount(1, Time.MILLISECONDS)
te.START_TIMEOUT = Amount(10, Time.MILLISECONDS)
te.launchTask(proxy_driver, make_task(HELLO_WORLD_MTI))
proxy_driver.wait_stopped()
updates = proxy_driver.method_calls['sendStatusUpdate']
assert len(updates) == 2
assert updates[0][0][0].state == mesos_pb2.TASK_STARTING
assert updates[1][0][0].state == mesos_pb2.TASK_FAILED
def test_shutdown_order(self):
proxy_driver = ProxyDriver()
with contextlib.nested(
temporary_dir(),
mock.patch.object(ChainedStatusChecker, 'stop'),
mock.patch.object(ThermosTaskRunner, 'stop')) as (
checkpoint_root,
status_check_stop,
runner_stop):
parent = mock.MagicMock()
parent.attach_mock(status_check_stop, 'status_check_stop')
parent.attach_mock(runner_stop, 'runner_stop')
_, executor = make_executor(proxy_driver,
checkpoint_root,
SLEEP60_MTI)
executor.shutdown(proxy_driver)
executor.terminated.wait()
parent.assert_has_calls(
[mock.call.status_check_stop(), mock.call.runner_stop()],
any_order=False)
def test_resolve_prereleases_multiple_set():
stable_dep = make_sdist(name='dep', version='2.0.0')
prerelease_dep1 = make_sdist(name='dep', version='3.0.0rc3')
prerelease_dep2 = make_sdist(name='dep', version='3.0.0rc4')
prerelease_dep3 = make_sdist(name='dep', version='3.0.0rc5')
with temporary_dir() as td:
for sdist in (stable_dep, prerelease_dep1, prerelease_dep2, prerelease_dep3):
safe_copy(sdist, os.path.join(td, os.path.basename(sdist)))
fetchers = [Fetcher([td])]
def assert_resolve(expected_version, **resolve_kwargs):
dists = list(
resolve_multi(['dep>=3.0.0rc1', 'dep==3.0.0rc4'],
fetchers=fetchers,
**resolve_kwargs)
)
assert 1 == len(dists)
dist = dists[0]
assert expected_version == dist.version
# This should resolve with explicit prerelease being set or implicitly.
assert_resolve('3.0.0rc4', allow_prereleases=True)
assert_resolve('3.0.0rc4')
def assert_create_user_and_group(mock_check_call, gid_exists, uid_exists):
mock_pwent = pwd.struct_passwd((
'someuser', # login name
'hunter2', # password
834, # uid
835, # gid
'Some User', # user name
'/home/someuser', # home directory
'/bin/sh')) # login shell
mock_grent = grp.struct_group((
'users', # group name
'*', # password
835, # gid
['someuser'])) # members
exception = subprocess.CalledProcessError(
returncode=FileSystemImageSandbox._USER_OR_GROUP_ID_EXISTS,
cmd='some command',
output=None)
mock_check_call.side_effect = [
None if gid_exists else exception, None if uid_exists else exception
]
with temporary_dir() as d:
with mock.patch.object(FileSystemImageSandbox,
'get_user_and_group',
return_value=(mock_pwent, mock_grent)):
sandbox = FileSystemImageSandbox(os.path.join(d, 'sandbox'),
user='******',
sandbox_mount_point='/some/path')
sandbox._create_user_and_group_in_taskfs()
assert len(mock_check_call.mock_calls) == 2
def _run_pants(self, pants_repo: pathlib.PosixPath, pants_target: str,
extension: str) -> pathlib.PosixPath:
"""Runs pants with widget UI display."""
# Version check for pants v1 vs v2 flags/behavior.
is_pants_v1 = pants_repo.joinpath('pants.ini').exists()
if is_pants_v1:
goal_name = 'binary'
tmp_root = None
else:
goal_name = 'package'
# N.B. pants v2 doesn't support `--pants-distdir` outside of the build root.
tmp_root = pants_repo.joinpath('dist')
# N.B. The dist dir must exist for temporary_dir.
tmp_root.mkdir(exist_ok=True)
with temporary_dir(root_dir=tmp_root, cleanup=False) as tmp_dir:
tmp_path = pathlib.PosixPath(tmp_dir)
title = f'[Build] ./pants {goal_name} {pants_target}'
cmd = f'cd {pants_repo} && ./pants --pants-distdir="{tmp_path}" {goal_name} {pants_target}'
return self._stream_binary_build_with_output(cmd,
title,
tmp_path,
extension=extension)
def find_java_home():
# A kind-of-insane hack to find the effective java home. On some platforms there are so
# many hard and symbolic links into the JRE dirs that it's actually quite hard to
# establish what path to use as the java home, e.g., for the purpose of rebasing.
# In practice, this seems to work fine.
#
# TODO: In the future we should probably hermeticize the Java enivronment rather than relying
# on whatever's on the shell's PATH. E.g., you either specify a path to the Java home via a
# cmd-line flag or .pantsrc, or we infer one with this method but verify that it's of a
# supported version.
with temporary_dir() as tmpdir:
with open(os.path.join(tmpdir, 'X.java'), 'w') as srcfile:
srcfile.write('''
class X {
public static void main(String[] argv) {
System.out.println(System.getProperty("java.home"));
}
}''')
subprocess.Popen(['javac', '-d', tmpdir, srcfile.name],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
return subprocess.Popen(['java', '-cp', tmpdir, 'X'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()[0]
def test_simple_process_filesystem_isolator():
with temporary_dir() as td:
taskpath = make_taskpath(td)
sandbox = setup_sandbox(td, taskpath)
test_isolator_path = os.path.join(td, 'fake-mesos-containerier')
with open(test_isolator_path, 'w') as fd:
# We use a fake version of the mesos-containerizer binary that just echoes out its args so
# we can assert on them in the process's output.
fd.write('\n'.join([
'#!/bin/sh',
'echo "$@"'
]))
fd.close()
chmod_plus_x(test_isolator_path)
p = TestProcess(
'process',
'echo hello world',
0,
taskpath,
sandbox,
mesos_containerizer_path=test_isolator_path,
container_sandbox=sandbox)
p.start()
rc = wait_for_rc(taskpath.getpath('process_checkpoint'))
assert rc == 0
assert_log_content(
taskpath,
'stdout',
'launch --unshare_namespace_mnt --working_directory=%s --rootfs=/some/path/taskfs --user=None '
'--command={"shell":true,"value":"/bin/bash -c \'echo hello world\'"}\n' % (
sandbox))
def post_process_cached_vts(cached_vts):
# Merge the localized analysis with the global one (if any).
analyses_to_merge = []
for vt in cached_vts:
for target in vt.targets:
analysis_file = JvmCompile._analysis_for_target(
self._analysis_tmpdir, target)
portable_analysis_file = JvmCompile._portable_analysis_for_target(
self._analysis_tmpdir, target)
if os.path.exists(portable_analysis_file):
self._analysis_tools.localize(portable_analysis_file,
analysis_file)
if os.path.exists(analysis_file):
analyses_to_merge.append(analysis_file)
if len(analyses_to_merge) > 0:
if os.path.exists(self._analysis_file):
analyses_to_merge.append(self._analysis_file)
with contextutil.temporary_dir() as tmpdir:
tmp_analysis = os.path.join(tmpdir, 'analysis')
with self.context.new_workunit(name='merge_analysis'):
self._analysis_tools.merge_from_paths(
analyses_to_merge, tmp_analysis)
self.move(tmp_analysis, self._analysis_file)
def test_basic_as_job(self):
proxy_driver = ProxyDriver()
with temporary_dir() as tempdir:
te = AuroraExecutor(runner_provider=make_provider(tempdir),
sandbox_provider=DefaultTestSandboxProvider())
te.launchTask(proxy_driver,
make_task(MESOS_JOB(task=HELLO_WORLD), instanceId=0))
te.runner_started.wait()
while te._status_manager is None:
time.sleep(0.1)
te.terminated.wait()
tm = TaskMonitor(tempdir, task_id=HELLO_WORLD_TASK_ID)
runner_state = tm.get_state()
assert 'hello_world_hello_world-001' in runner_state.processes, (
'Could not find processes, got: %s' %
' '.join(runner_state.processes))
updates = proxy_driver.method_calls['sendStatusUpdate']
assert len(updates) == 3
status_updates = [arg_tuple[0][0] for arg_tuple in updates]
assert status_updates[0].state == mesos_pb2.TASK_STARTING
assert status_updates[1].state == mesos_pb2.TASK_RUNNING
assert status_updates[2].state == mesos_pb2.TASK_FINISHED
def test_env(content=TEST_CONTENT):
with temporary_dir() as d:
with tempfile.NamedTemporaryFile() as f:
f.write(content)
f.flush()
yield f, CacheKeyGenerator(), BuildInvalidator(d)
def test_gc_ignore_retained_task(self):
with temporary_dir() as root:
task_id = self.setup_task(self.HELLO_WORLD, root, finished=True)
gcs = self.run_gc(root, task_id, retain=True)
assert len(gcs) == 0
def test_task_detector():
with temporary_dir() as root:
active_log_dir = os.path.join(root, 'active_log')
finished_log_dir = os.path.join(root, 'finished_log')
path = TaskPath(root=root)
detector = TaskDetector(root)
# test empty paths
assert list(detector.get_task_ids(state='active')) == []
assert list(detector.get_task_ids(state='finished')) == []
assert set(detector.get_task_ids()) == set()
assert detector.get_checkpoint(task_id='active_task') == path.given(
task_id='active_task').getpath('runner_checkpoint')
assert detector.get_checkpoint(task_id='finished_task') == path.given(
task_id='finished_task').getpath('runner_checkpoint')
assert set(detector.get_process_checkpoints('active_task')) == set()
assert set(detector.get_process_checkpoints('finished_task')) == set()
assert set(detector.get_process_runs('active_task', active_log_dir)) == set()
assert set(detector.get_process_runs('finished_task', finished_log_dir)) == set()
assert set(detector.get_process_logs('active_task', active_log_dir)) == set()
assert set(detector.get_process_logs('finished_task', finished_log_dir)) == set()
# create paths
paths = [
path.given(state='active', task_id='active_task').getpath('task_path'),
path.given(state='finished', task_id='finished_task').getpath('task_path'),
path.given(task_id='active_task').getpath('runner_checkpoint'),
path.given(task_id='finished_task').getpath('runner_checkpoint'),
path.given(
task_id='active_task',
process='hello_world',
run='0',
log_dir=active_log_dir
).with_filename('stdout').getpath('process_logdir'),
path.given(
task_id='finished_task',
process='goodbye_world',
run='1',
log_dir=finished_log_dir
).with_filename('stderr').getpath('process_logdir'),
path.given(task_id='active_task', process='hello_world').getpath('process_checkpoint'),
path.given(task_id='finished_task', process='goodbye_world').getpath('process_checkpoint'),
]
for p in paths:
touch(p)
detector = TaskDetector(root)
assert list(detector.get_task_ids(state='active')) == list([('active', 'active_task')])
assert list(detector.get_task_ids(state='finished')) == list([('finished', 'finished_task')])
assert set(detector.get_task_ids()) == set(
[('active', 'active_task'), ('finished', 'finished_task')])
assert list(detector.get_process_checkpoints('active_task')) == [
path.given(task_id='active_task', process='hello_world').getpath('process_checkpoint')]
assert list(detector.get_process_checkpoints('finished_task')) == [
path.given(task_id='finished_task', process='goodbye_world').getpath('process_checkpoint')]
assert list(detector.get_process_runs('active_task', active_log_dir)) == [
('hello_world', 0)]
assert list(detector.get_process_runs('finished_task', finished_log_dir)) == [
('goodbye_world', 1)]
assert list(detector.get_process_logs('active_task', active_log_dir)) == [
path.given(
task_id='active_task',
process='hello_world',
run='0',
log_dir=active_log_dir
).with_filename('stdout').getpath('process_logdir')]
assert list(detector.get_process_logs('finished_task', finished_log_dir)) == [
path.given(
task_id='finished_task',
process='goodbye_world',
run='1',
log_dir=finished_log_dir
).with_filename('stderr').getpath('process_logdir')]
def test_local_cache():
with temporary_dir() as artifact_root:
with temporary_dir() as cache_root:
artifact_cache = FileBasedArtifactCache(None, artifact_root,
cache_root)
do_test_artifact_cache(artifact_cache)
def test_temporary_dir_with_root_dir():
with temporary_dir() as path1:
with temporary_dir(root_dir=path1) as path2:
assert os.path.realpath(path2).startswith(os.path.realpath(path1)), \
'nested temporary dir should be created within outer dir.'
def test_env(content=TEST_CONTENT):
with temporary_dir() as d:
with tempfile.NamedTemporaryFile() as f:
f.write(content)
f.flush()
yield f, ArtifactCache(d)
def test_load_yaml():
with temporary_dir() as td:
clusters_yml = os.path.join(td, 'clusters.yml')
with open(clusters_yml, 'w') as fp:
fp.write(TEST_YAML)
validate_loaded_clusters(Clusters.from_file(clusters_yml))
def test_load():
with temporary_dir() as td:
clusters_json = os.path.join(td, 'clusters.json')
with open(clusters_json, 'w') as fp:
fp.write(CLUSTERS)
validate_loaded_clusters(Clusters.from_file(clusters_json))
def test_clp_prereleases_resolver():
prerelease_dep = make_sdist(name='dep', version='1.2.3b1')
with temporary_dir() as td:
safe_copy(prerelease_dep,
os.path.join(td, os.path.basename(prerelease_dep)))
fetcher = Fetcher([td])
# When no specific options are specified, allow_prereleases is None
parser, resolver_options_builder = configure_clp()
assert resolver_options_builder._allow_prereleases is None
# When we specify `--pre`, allow_prereleases is True
options, reqs = parser.parse_args(
args=['--pre', 'dep==1.2.3b1', 'dep'])
assert resolver_options_builder._allow_prereleases
# We need to use our own fetcher instead of PyPI
resolver_options_builder._fetchers.insert(0, fetcher)
#####
# The resolver created during processing of command line options (configure_clp)
# is not actually passed into the API call (resolve_multi) from build_pex().
# Instead, resolve_multi() calls resolve() where a new ResolverOptionsBuilder instance
# is created. The only way to supply our own fetcher to that new instance is to patch it
# here in the test so that it can fetch our test package (dep-1.2.3b1). Hence, this class
# below and the change in the `pex.resolver` module where the patched object resides.
#
import pex.resolver
class BuilderWithFetcher(ResolverOptionsBuilder):
def __init__(self,
fetchers=None,
allow_all_external=False,
allow_external=None,
allow_unverified=None,
allow_prereleases=None,
use_manylinux=None,
precedence=None,
context=None):
super(BuilderWithFetcher,
self).__init__(fetchers=fetchers,
allow_all_external=allow_all_external,
allow_external=allow_external,
allow_unverified=allow_unverified,
allow_prereleases=allow_prereleases,
use_manylinux=None,
precedence=precedence,
context=context)
self._fetchers.insert(0, fetcher)
# end stub
#####
# Without a corresponding fix in pex.py, this test failed for a dependency requirement of
# dep==1.2.3b1 from one package and just dep (any version accepted) from another package.
# The failure was an exit from build_pex() with the message:
#
# Could not satisfy all requirements for dep==1.2.3b1:
# dep==1.2.3b1, dep
#
# With a correct behavior the assert line is reached and pex_builder object created.
with mock.patch.object(pex.resolver, 'ResolverOptionsBuilder',
BuilderWithFetcher):
pex_builder = build_pex(reqs, options, resolver_options_builder)
assert pex_builder is not None
def test_local_cache(self):
with temporary_dir() as artifact_root:
with temporary_dir() as cache_root:
artifact_cache = LocalArtifactCache(None, artifact_root, cache_root)
self.do_test_artifact_cache(artifact_cache)
def test_analysis_files(self):
classes_dir = '/Users/kermit/src/acme.web/.pants.d/scalac/classes/'
parser = ZincAnalysisParser(classes_dir)
with temporary_dir() as tmpdir:
# Extract analysis files from tarball.
analysis_tarball = os.path.join(os.path.dirname(__file__), 'testdata', 'analysis.tar.bz2')
analysis_dir = os.path.join(tmpdir, 'orig')
print('Extracting %s to %s' % (analysis_tarball, analysis_dir))
os.mkdir(analysis_dir)
with contextlib.closing(tarfile.open(analysis_tarball, 'r:bz2')) as tar:
tar.extractall(analysis_dir)
# Parse them.
analysis_files = [os.path.join(analysis_dir, f)
for f in os.listdir(analysis_dir) if f.endswith('.analysis')]
num_analyses = len(analysis_files)
def parse(f):
return parser.parse_from_path(f)
analyses = self._time(lambda: [parse(f) for f in analysis_files],
'Parsed %d files' % num_analyses)
# Write them back out individually.
writeout_dir = os.path.join(tmpdir, 'write')
os.mkdir(writeout_dir)
def write(file_name, analysis):
outpath = os.path.join(writeout_dir, file_name)
analysis.write_to_path(outpath)
def _write_all():
for analysis_file, analysis in zip(analysis_files, analyses):
write(os.path.basename(analysis_file), analysis)
self._time(_write_all, 'Wrote %d files' % num_analyses)
# Merge them.
merged_analysis = self._time(lambda: ZincAnalysis.merge(analyses),
'Merged %d files' % num_analyses)
# Write merged analysis to file.
merged_analysis_path = os.path.join(tmpdir, 'analysis.merged')
self._time(lambda: merged_analysis.write_to_path(merged_analysis_path),
'Wrote merged analysis to %s' % merged_analysis_path)
# Read merged analysis from file.
merged_analysis2 = self._time(lambda: parser.parse_from_path(merged_analysis_path),
'Read merged analysis from %s' % merged_analysis_path)
# Split the merged analysis back to individual analyses.
sources_per_analysis = [a.stamps.sources.keys() for a in analyses]
split_analyses = self._time(lambda: merged_analysis2.split(sources_per_analysis, catchall=True),
'Split back into %d analyses' % num_analyses)
self.assertEquals(num_analyses + 1, len(split_analyses)) # +1 for the catchall.
catchall_analysis = split_analyses[-1]
# We expect an empty catchall.
self.assertEquals(0, len(catchall_analysis.stamps.sources))
# Diff the original analyses and the split ones.
# Write the split to the tmpdir, for ease of debugging on failure.
splits_dir = os.path.join(tmpdir, 'splits')
os.mkdir(splits_dir)
for analysis_file, analysis, split_analysis in zip(analysis_files, analyses, split_analyses):
outfile_path = os.path.join(splits_dir, os.path.basename(analysis_file))
split_analysis.write_to_path(outfile_path)
diffs = analysis.diff(split_analysis)
self.assertEquals(analysis, split_analysis, ''.join(diffs))
print('Total time: %f seconds' % self.total_time)
def yield_pex_builder(zip_safe=True):
with nested(temporary_dir(), make_distribution('p1', zipped=True, zip_safe=zip_safe)) as (td, p1):
pb = PEXBuilder(path=td)
pb.add_egg(p1.location)
yield pb
def pex_with_no_entrypoints():
with temporary_dir() as out:
pex = os.path.join(out, 'pex.pex')
run_pex_command(['setuptools==36.2.7', '-o', pex])
test_script = b'from setuptools.sandbox import run_setup; print(str(run_setup))'
yield pex, test_script, out
def test(self):
self.assertEqual(set(), self.git.changed_files())
self.assertEqual(set(['README']),
self.git.changed_files(from_commit='HEAD^'))
tip_sha = self.git.commit_id
self.assertTrue(tip_sha)
self.assertTrue(tip_sha in self.git.changelog())
self.assertTrue(self.git.tag_name.startswith('first-'),
msg='un-annotated tags should be found')
self.assertEqual('master', self.git.branch_name)
def edit_readme():
with open(self.readme_file, 'a') as readme:
readme.write('More data.')
edit_readme()
with open(os.path.join(self.worktree, 'INSTALL'), 'w') as untracked:
untracked.write('make install')
self.assertEqual(set(['README']), self.git.changed_files())
self.assertEqual(set(['README', 'INSTALL']),
self.git.changed_files(include_untracked=True))
try:
# These changes should be rejected because our branch point from origin is 1 commit behind
# the changes pushed there in clone 2.
self.git.commit('API Changes.')
except Scm.RemoteException:
with environment_as(GIT_DIR=self.gitdir,
GIT_WORK_TREE=self.worktree):
subprocess.check_call(
['git', 'reset', '--hard', 'depot/master'])
self.git.refresh()
edit_readme()
self.git.commit('''API '"' " Changes.''')
self.git.tag('second', message='''Tagged ' " Changes''')
with temporary_dir() as clone:
with pushd(clone):
self.init_repo('origin', self.origin)
subprocess.check_call(
['git', 'pull', '--tags', 'origin', 'master:master'])
with open(os.path.realpath('README')) as readme:
self.assertEqual('--More data.', readme.read())
git = Git()
# Check that we can pick up committed and uncommitted changes.
with safe_open(os.path.realpath('CHANGES'), 'w') as changes:
changes.write('none')
subprocess.check_call(['git', 'add', 'CHANGES'])
self.assertEqual(set(['README', 'CHANGES']),
git.changed_files(from_commit='first'))
self.assertEqual('master', git.branch_name)
self.assertEqual('second',
git.tag_name,
msg='annotated tags should be found')
def test_pex_python():
py2_path_interpreter = ensure_python_interpreter('2.7.10')
py3_path_interpreter = ensure_python_interpreter('3.6.3')
path = ':'.join([
os.path.dirname(py2_path_interpreter),
os.path.dirname(py3_path_interpreter)
])
with environment_as(PATH=path):
with temporary_dir() as td:
pexrc_path = os.path.join(td, '.pexrc')
with open(pexrc_path, 'w') as pexrc:
pex_python = ensure_python_interpreter('3.6.3')
pexrc.write("PEX_PYTHON=%s" % pex_python)
# test PEX_PYTHON with valid constraints
pex_out_path = os.path.join(td, 'pex.pex')
res = run_pex_command([
'--disable-cache',
'--rcfile=%s' % pexrc_path, '--interpreter-constraint=>3',
'--interpreter-constraint=<3.8', '-o', pex_out_path
])
res.assert_success()
stdin_payload = b'import sys; print(sys.executable); sys.exit(0)'
stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload)
assert rc == 0
correct_interpreter_path = pex_python.encode()
assert correct_interpreter_path in stdout
# test PEX_PYTHON with incompatible constraints
pexrc_path = os.path.join(td, '.pexrc')
with open(pexrc_path, 'w') as pexrc:
pex_python = ensure_python_interpreter('2.7.10')
pexrc.write("PEX_PYTHON=%s" % pex_python)
pex_out_path = os.path.join(td, 'pex2.pex')
res = run_pex_command([
'--disable-cache',
'--rcfile=%s' % pexrc_path, '--interpreter-constraint=>3',
'--interpreter-constraint=<3.8', '-o', pex_out_path
])
res.assert_success()
stdin_payload = b'import sys; print(sys.executable); sys.exit(0)'
stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload)
assert rc == 1
fail_str = 'not compatible with specified interpreter constraints'.encode(
)
assert fail_str in stdout
# test PEX_PYTHON with no constraints
pex_out_path = os.path.join(td, 'pex3.pex')
res = run_pex_command([
'--disable-cache',
'--rcfile=%s' % pexrc_path, '-o', pex_out_path
])
res.assert_success()
stdin_payload = b'import sys; print(sys.executable); sys.exit(0)'
stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload)
assert rc == 0
correct_interpreter_path = pex_python.encode()
assert correct_interpreter_path in stdout
def test_interpreter_selection_using_os_environ_for_bootstrap_reexec():
"""
This is a test for verifying the proper function of the
pex bootstrapper's interpreter selection logic and validate a corresponding
bugfix. More details on the nature of the bug can be found at:
https://github.com/pantsbuild/pex/pull/441
"""
with temporary_dir() as td:
pexrc_path = os.path.join(td, '.pexrc')
# Select pexrc interpreter versions based on test environemnt.
# The parent interpreter is the interpreter we expect the parent pex to
# execute with. The child interpreter is the interpreter we expect the
# child pex to execute with.
if (sys.version_info[0], sys.version_info[1]) == (3, 6):
child_pex_interpreter_version = '3.6.3'
else:
child_pex_interpreter_version = '2.7.10'
# Write parent pex's pexrc.
with open(pexrc_path, 'w') as pexrc:
pexrc.write("PEX_PYTHON=%s" % sys.executable)
test_setup_path = os.path.join(td, 'setup.py')
with open(test_setup_path, 'w') as fh:
fh.write(dedent('''
from setuptools import setup
setup(
name='tester',
version='1.0',
description='tests',
author='tester',
author_email='*****@*****.**',
packages=['testing']
)
'''))
os.mkdir(os.path.join(td, 'testing'))
test_init_path = os.path.join(td, 'testing/__init__.py')
with open(test_init_path, 'w') as fh:
fh.write(dedent('''
def tester():
from pex.testing import (
run_pex_command,
run_simple_pex
)
import os
import tempfile
import shutil
from textwrap import dedent
td = tempfile.mkdtemp()
try:
pexrc_path = os.path.join(td, '.pexrc')
with open(pexrc_path, 'w') as pexrc:
pexrc.write("PEX_PYTHON={}")
test_file_path = os.path.join(td, 'build_and_run_child_pex.py')
with open(test_file_path, 'w') as fh:
fh.write(dedent("""
import sys
print(sys.executable)
"""))
pex_out_path = os.path.join(td, 'child.pex')
res = run_pex_command(['--disable-cache',
'-o', pex_out_path])
stdin_payload = b'import sys; print(sys.executable); sys.exit(0)'
stdout, rc = run_simple_pex(pex_out_path, stdin=stdin_payload)
print(stdout)
finally:
shutil.rmtree(td)
'''.format(ensure_python_interpreter(child_pex_interpreter_version))))
pex_out_path = os.path.join(td, 'parent.pex')
res = run_pex_command(['--disable-cache',
'pex',
'{}'.format(td),
'-e', 'testing:tester',
'-o', pex_out_path])
res.assert_success()
stdout, rc = run_simple_pex(pex_out_path)
assert rc == 0
# Ensure that child pex used the proper interpreter as specified by its pexrc.
correct_interpreter_path = ensure_python_interpreter(child_pex_interpreter_version)
correct_interpreter_path = correct_interpreter_path.encode() # Py 2/3 compatibility
assert correct_interpreter_path in stdout
def merge_artifact(self, versioned_target_set):
if len(versioned_target_set.targets) <= 1:
return
with temporary_dir() as tmpdir:
dst_output_dir, dst_depfile, dst_analysis_cache = self.create_output_paths(
versioned_target_set.targets)
safe_rmtree(dst_output_dir)
safe_mkdir(dst_output_dir)
src_analysis_caches = []
analysis_args = []
analysis_args.extend(self._zinc_jar_args)
analysis_args.extend([
'-log-level',
self.context.options.log_level or 'info',
'-analysis',
])
# TODO: Do we actually need to merge deps? Zinc will stomp them anyway on success.
dst_deps = Dependencies(dst_output_dir)
for target in versioned_target_set.targets:
src_output_dir, src_depfile, src_analysis_cache = self.create_output_paths(
[target])
if os.path.exists(src_depfile):
src_deps = Dependencies(src_output_dir)
src_deps.load(src_depfile)
dst_deps.merge(src_deps)
classes_by_source = src_deps.findclasses([target]).get(
target, {})
for source, classes in classes_by_source.items():
for cls in classes:
src = os.path.join(src_output_dir, cls)
dst = os.path.join(dst_output_dir, cls)
# src may not exist if we aborted a build in the middle. That's OK: zinc will notice that
# it's missing and rebuild it.
# dst may already exist if we have overlapping targets. It's not a good idea
# to have those, but until we enforce it, we must allow it here.
if os.path.exists(src) and not os.path.exists(dst):
# Copy the class file.
safe_mkdir(os.path.dirname(dst))
os.link(src, dst)
# Use zinc to rebase a copy of the per-target analysis files prior to merging.
if os.path.exists(src_analysis_cache):
src_analysis_cache_tmp = \
os.path.join(tmpdir, os.path.relpath(src_analysis_cache, self._analysis_cache_dir))
shutil.copyfile(src_analysis_cache,
src_analysis_cache_tmp)
src_analysis_caches.append(src_analysis_cache_tmp)
rebase_args = analysis_args + [
'-cache',
src_analysis_cache_tmp,
'-rebase',
'%s:%s' % (src_output_dir, dst_output_dir),
]
if self.runjava(self._main,
classpath=self._zinc_classpath,
args=rebase_args,
jvmargs=self._jvm_args):
self.context.log.warn('In merge_artifact: zinc failed to rebase analysis file %s. ' \
'Target may require a full rebuild.' % src_analysis_cache_tmp)
dst_deps.save(dst_depfile)
# Use zinc to merge the analysis files.
merge_args = analysis_args + [
'-cache',
dst_analysis_cache,
'-merge',
':'.join(src_analysis_caches),
]
if self.runjava(self._main,
classpath=self._zinc_classpath,
args=merge_args,
jvmargs=self._jvm_args):
raise TaskError, 'zinc failed to merge analysis files %s to %s' % \
(':'.join(src_analysis_caches), dst_analysis_cache)
