def create(cls, path=None):
"""Creates a pip tool with PEX isolation at path.
:param str path: The path to build the pip tool pex at; a temporary directory by default.
"""
from pex.pex_builder import PEXBuilder
isolated_pip_builder = PEXBuilder(path=path)
pythonpath = third_party.expose(['pip', 'setuptools', 'wheel'])
isolated_pip_environment = third_party.pkg_resources.Environment(
search_path=pythonpath)
for dist_name in isolated_pip_environment:
for dist in isolated_pip_environment[dist_name]:
isolated_pip_builder.add_dist_location(dist=dist.location)
with open(os.path.join(isolated_pip_builder.path(), 'run_pip.py'),
'w') as fp:
fp.write(
dedent("""\
import os
import runpy
import sys
# Propagate un-vendored setuptools to pip for any legacy setup.py builds it needs to
# perform.
os.environ['__PEX_UNVENDORED__'] = '1'
os.environ['PYTHONPATH'] = os.pathsep.join(sys.path)
runpy.run_module('pip', run_name='__main__')
"""))
isolated_pip_builder.set_executable(fp.name)
isolated_pip_builder.freeze()
return cls(isolated_pip_builder.path())
def execute_codegen(self, target, results_dir):
self.context.log.info("Processing target {}".format(target))
requirements_pex = self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)
interpreter = self.context.products.get_data(PythonInterpreter)
pex_info = PexInfo.default(interpreter)
pex_info.pex_path = requirements_pex.path()
with temporary_dir() as source_pex_chroot:
sources_pex_builder = PEXBuilder(
path=source_pex_chroot,
interpreter=interpreter,
copy=True,
pex_info=pex_info
)
pex_build_util.dump_sources(sources_pex_builder, target, self.context.log)
sources_pex_builder.freeze()
codegen_pex = PEX(sources_pex_builder.path(), interpreter)
setup_py_paths = []
for source in target.sources_relative_to_source_root():
if os.path.basename(source) == 'setup.py':
setup_py_paths.append(source)
if len(setup_py_paths) != 1:
raise TaskError(
'Expected target {} to own exactly one setup.py, found {}'.format(
setup_py_paths,
len(setup_py_paths)
)
)
setup_py_path = setup_py_paths[0]
result_code = codegen_pex.run(
with_chroot=True,
blocking=True,
args=(setup_py_path, 'build_ext', '--inplace', '--verbose'),
# Passing PATH helps cython find the correct c++ compiler
env={'libraries': results_dir, 'PATH': os.getenv('PATH')}
)
if result_code != 0:
raise TaskError(
'creating cython library failed',
exit_code=result_code,
failed_targets=[target]
)
library_source_path = os.path.join(
sources_pex_builder.path(),
os.path.dirname(setup_py_path),
target.output
)
library_output = os.path.join(results_dir, target.output)
safe_mkdir_for(library_output)
shutil.move(library_source_path, library_output)
self.context.log.info(
'created library {}'.format(os.path.relpath(library_output, get_buildroot()))
)
def _test_runner(self, targets, workunit):
interpreter = self.select_interpreter_for_targets(targets)
builder = PEXBuilder(interpreter=interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
context=self.context,
python_setup=PythonSetup.global_instance(),
python_repos=PythonRepos.global_instance(),
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=interpreter)
try:
builder = chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=interpreter)
with self._maybe_shard() as shard_args:
with self._maybe_emit_junit_xml(targets) as junit_args:
with self._maybe_emit_coverage_data(targets,
builder.path(),
pex,
workunit) as coverage_args:
yield pex, shard_args + junit_args + coverage_args
finally:
chroot.delete()
def test_activate_interpreter_different_from_current():
with temporary_dir() as pex_root:
interp_version = '3.6.3' if PY2 else '2.7.10'
custom_interpreter = get_interpreter(
python_interpreter=ensure_python_interpreter(interp_version),
interpreter_cache_dir=os.path.join(pex_root, 'interpreters'),
repos=None, # Default to PyPI.
use_wheel=True)
pex_info = PexInfo.default(custom_interpreter)
pex_info.pex_root = pex_root
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot,
interpreter=custom_interpreter,
pex_info=pex_info)
with make_bdist(installer_impl=WheelInstaller,
interpreter=custom_interpreter) as bdist:
pex_builder.add_distribution(bdist)
pex_builder.set_entry_point('sys:exit')
pex_builder.freeze()
pex = PEX(pex_builder.path(), interpreter=custom_interpreter)
try:
pex._activate()
except SystemExit as e:
pytest.fail('PEX activation of %s failed with %s' %
(pex, e))
def execute(self):
if self.old_options.pex and self.old_options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.old_options.entry_point and self.old_options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.old_options.verbose:
print('Build operating on targets: %s' % ' '.join(str(target) for target in self.targets))
builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
pex_info=self.binary.pexinfo if self.binary else None)
if self.old_options.entry_point:
builder.set_entry_point(self.old_options.entry_point)
if self.old_options.ipython:
if not self.config.has_section('python-ipython'):
self.error('No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
if builder.info.entry_point is None:
self.error('Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython', 'requirements', default=[])
for requirement in requirements:
self.extra_requirements.append(PythonRequirement(requirement))
executor = PythonChroot(
targets=self.targets,
extra_requirements=self.extra_requirements,
builder=builder,
platforms=self.binary.platforms if self.binary else None,
interpreter=self.interpreter,
conn_timeout=self.old_options.conn_timeout)
executor.dump()
if self.old_options.pex:
pex_name = self.binary.name if self.binary else Target.maybe_readable_identify(self.targets)
pex_path = os.path.join(self.root_dir, 'dist', '%s.pex' % pex_name)
builder.build(pex_path)
print('Wrote %s' % pex_path)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def _test_runner(self, targets, stdout, stderr):
builder = PEXBuilder(interpreter=self._interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current', ),
interpreter=self._interpreter)
try:
builder = chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=self._interpreter)
with self._maybe_emit_junit_xml(targets) as junit_args:
with self._maybe_emit_coverage_data(targets, builder.path(),
pex, stdout,
stderr) as coverage_args:
yield pex, junit_args + coverage_args
finally:
chroot.delete()
def _test_runner(self, targets, stdout, stderr):
builder = PEXBuilder(interpreter=self._interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=self._interpreter)
try:
builder = chroot.dump()
builder.freeze()
pex = PEX(builder.path(), interpreter=self._interpreter)
with self._maybe_emit_junit_xml(targets) as junit_args:
with self._maybe_emit_coverage_data(targets,
builder.path(),
pex,
stdout,
stderr) as coverage_args:
yield pex, junit_args + coverage_args
finally:
chroot.delete()
def _run_python_tests(self, targets, stdout, stderr):
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current', ),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = []
test_args.extend(PythonTestBuilder.generate_junit_args(targets))
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(targets)
test_args.extend(args)
sources = list(
itertools.chain(
*[t.sources_relative_to_buildroot() for t in targets]))
pex = PEX(builder.path(), interpreter=self.interpreter)
rc = pex.run(args=test_args + sources,
blocking=True,
setsid=True,
stdout=stdout,
stderr=stderr)
# TODO(wickman): If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestResult.rc(rc)
except Exception:
import traceback
print('Failed to run test!', file=stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
return rv
def test_activate_interpreter_different_from_current():
with temporary_dir() as pex_root:
interp_version = PY36 if PY2 else PY27
custom_interpreter = PythonInterpreter.from_binary(ensure_python_interpreter(interp_version))
pex_info = PexInfo.default(custom_interpreter)
pex_info.pex_root = pex_root
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot,
interpreter=custom_interpreter,
pex_info=pex_info)
with make_bdist(interpreter=custom_interpreter) as bdist:
pex_builder.add_distribution(bdist)
pex_builder.set_entry_point('sys:exit')
pex_builder.freeze()
pex = PEX(pex_builder.path(), interpreter=custom_interpreter)
try:
pex._activate()
except SystemExit as e:
pytest.fail('PEX activation of %s failed with %s' % (pex, e))
def test_venv_symlinked_source_issues_1239(tmpdir):
# type: (Any) -> None
src = os.path.join(str(tmpdir), "src")
main = os.path.join(src, "main.py")
with safe_open(main, "w") as fp:
fp.write("import sys; sys.exit(42)")
pex_builder = PEXBuilder(copy_mode=CopyMode.SYMLINK)
pex_builder.set_executable(main)
pex_file = os.path.join(str(tmpdir), "a.pex")
pex_builder.build(pex_file, bytecode_compile=False)
assert 42 == subprocess.Popen(args=[pex_file]).wait()
venv = os.path.join(str(tmpdir), "a.venv")
subprocess.check_call(
args=[sys.executable, "-m", "pex.tools", pex_builder.path(), "venv", venv]
)
venv_pex = os.path.join(venv, "pex")
shutil.rmtree(src)
assert 42 == subprocess.Popen(args=[venv_pex]).wait()
def create(cls, path):
# type: (str) -> Pip
"""Creates a pip tool with PEX isolation at path.
:param path: The path to build the pip tool pex at.
"""
pip_pex_path = os.path.join(path, isolated().pex_hash)
with atomic_directory(pip_pex_path, exclusive=True) as chroot:
if chroot is not None:
from pex.pex_builder import PEXBuilder
isolated_pip_builder = PEXBuilder(path=chroot)
pythonpath = third_party.expose(["pip", "setuptools", "wheel"])
isolated_pip_environment = third_party.pkg_resources.Environment(
search_path=pythonpath
)
for dist_name in isolated_pip_environment:
for dist in isolated_pip_environment[dist_name]:
isolated_pip_builder.add_dist_location(dist=dist.location)
with open(os.path.join(isolated_pip_builder.path(), "run_pip.py"), "w") as fp:
fp.write(
dedent(
"""\
import os
import runpy
import sys
# Propagate un-vendored setuptools to pip for any legacy setup.py builds it needs to
# perform.
os.environ['__PEX_UNVENDORED__'] = '1'
os.environ['PYTHONPATH'] = os.pathsep.join(sys.path)
runpy.run_module('pip', run_name='__main__')
"""
)
)
isolated_pip_builder.set_executable(fp.name)
isolated_pip_builder.freeze()
return cls(pip_pex_path)
def assert_namespace_packages_warning(distribution, version, expected_warning):
# type: (str, str, bool) -> None
requirement = "{}=={}".format(distribution, version)
pb = PEXBuilder()
for resolved_dist in resolver.resolve([requirement]):
pb.add_dist_location(resolved_dist.distribution.location)
pb.freeze()
process = PEX(pb.path()).run(args=["-c", ""], blocking=False, stderr=subprocess.PIPE)
_, stderr = process.communicate()
stderr_text = stderr.decode("utf8")
partial_warning_preamble = "PEXWarning: The `pkg_resources` package was loaded"
partial_warning_detail = "{} namespace packages:".format(requirement)
if expected_warning:
assert partial_warning_preamble in stderr_text
assert partial_warning_detail in stderr_text
else:
assert partial_warning_preamble not in stderr_text
assert partial_warning_detail not in stderr_text
def _run_python_tests(self, targets, stdout, stderr):
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = []
test_args.extend(PythonTestBuilder.generate_junit_args(targets))
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(targets)
test_args.extend(args)
sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets]))
pex = PEX(builder.path(), interpreter=self.interpreter)
rc = pex.run(args=test_args + sources, blocking=True, setsid=True,
stdout=stdout, stderr=stderr)
# TODO(wickman): If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestResult.rc(rc)
except Exception:
import traceback
print('Failed to run test!', file=stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
return rv
def build_and_check(copy_mode):
# type: (CopyMode.Value) -> None
pb = PEXBuilder(copy_mode=copy_mode)
path = pb.path()
pb.add_source(src, "exe.py")
path_clone = os.path.join(path, "__clone")
pb.clone(into=path_clone)
for root in path, path_clone:
s1 = os.stat(src)
s2 = os.stat(os.path.join(root, "exe.py"))
is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV])
if copy_mode == CopyMode.COPY:
assert not is_link
else:
# Since os.stat follows symlinks; so in CopyMode.SYMLINK, this just proves
# the symlink points to the original file. Going further and checking path
# and path_clone for the presence of a symlink (an os.islink test) is
# trickier since a Linux hardlink of a symlink produces a symlink whereas a
# macOS hardlink of a symlink produces a hardlink.
assert is_link
def create_pex_repository(
interpreters=None, # type: Optional[Iterable[PythonInterpreter]]
platforms=None, # type: Optional[Iterable[Platform]]
requirements=None, # type: Optional[Iterable[str]]
requirement_files=None, # type: Optional[Iterable[str]]
constraint_files=None, # type: Optional[Iterable[str]]
manylinux=None, # type: Optional[str]
):
# type: (...) -> str
pex_builder = PEXBuilder()
for resolved_dist in resolve_multi(
interpreters=interpreters,
platforms=platforms,
requirements=requirements,
requirement_files=requirement_files,
constraint_files=constraint_files,
manylinux=manylinux,
):
pex_builder.add_distribution(resolved_dist.distribution)
if resolved_dist.direct_requirement:
pex_builder.add_requirement(resolved_dist.direct_requirement)
pex_builder.freeze()
return os.path.realpath(cast(str, pex_builder.path()))
def execute(self):
if self.options.pex and self.options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.options.entry_point and self.options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.options.verbose:
print('Build operating on targets: %s' %
' '.join(str(target) for target in self.targets))
builder = PEXBuilder(
tempfile.mkdtemp(),
interpreter=self.interpreter,
pex_info=self.binary.pexinfo if self.binary else None)
if self.options.entry_point:
builder.set_entry_point(self.options.entry_point)
if self.options.ipython:
if not self.config.has_section('python-ipython'):
self.error(
'No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython',
'entry_point')
if builder.info.entry_point is None:
self.error(
'Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython',
'requirements',
default=[])
for requirement in requirements:
self.extra_requirements.append(PythonRequirement(requirement))
executor = PythonChroot(
targets=self.targets,
extra_requirements=self.extra_requirements,
builder=builder,
platforms=self.binary.platforms if self.binary else None,
interpreter=self.interpreter,
conn_timeout=self.options.conn_timeout)
executor.dump()
if self.options.pex:
pex_name = self.binary.name if self.binary else Target.maybe_readable_identify(
self.targets)
pex_path = os.path.join(self.root_dir, 'dist', '%s.pex' % pex_name)
builder.build(pex_path)
print('Wrote %s' % pex_path)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def build_pex(pex_filename: str, local_only: bool) -> str:
pex_builder = PEXBuilder(include_tools=True)
pex_builder.info.inherit_path = InheritPath.FALLBACK
pex_builder.set_entry_point('daml_dit_if.main:main')
pex_builder.set_shebang('/usr/bin/env python3')
platforms = [parsed_platform('current')]
if local_only:
LOG.warn('Local-only build. THIS DIT WILL NOT RUN IN DAML HUB.')
else:
platforms = [
*platforms,
parsed_platform('manylinux2014_x86_64-cp-38-cp38')
]
daml_dit_if_bundled = False
try:
if os.path.isfile(PYTHON_REQUIREMENT_FILE):
LOG.info(
f'Bundling dependencies from {PYTHON_REQUIREMENT_FILE}...')
requirement_files = [PYTHON_REQUIREMENT_FILE]
else:
LOG.info(
f'No dependency file found ({PYTHON_REQUIREMENT_FILE}), no dependencies will be bundled.'
)
requirement_files = []
resolveds = resolve_multi(requirements=[],
requirement_files=requirement_files,
platforms=platforms)
for resolved_dist in resolveds:
if resolved_dist.distribution.project_name == IF_PROJECT_NAME \
and not daml_dit_if_bundled:
LOG.warn(
f'Bundling {IF_PROJECT_NAME} in output DIT file. This will'
f' override the version provided by Daml Hub, potentially'
f' compromising compatibility of this integration with'
f' future updates to Daml Hub. Use this option with caution.'
)
daml_dit_if_bundled = True
LOG.debug("req: %s", resolved_dist.distribution)
LOG.debug(" -> target: %s", resolved_dist.target)
pex_builder.add_distribution(resolved_dist.distribution)
if resolved_dist.direct_requirement:
LOG.info("direct_req: %s", resolved_dist.direct_requirement)
LOG.debug(" -> target: %s", resolved_dist.target)
pex_builder.add_requirement(resolved_dist.direct_requirement)
except Unsatisfiable as e:
die(f'Unsatifiable dependency error: {e}')
def walk_and_do(fn, src_dir):
src_dir = os.path.normpath(src_dir)
for root, dirs, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
LOG.debug("Adding source file: %r, %r", src_file_path,
dst_path)
fn(src_file_path, dst_path)
walk_and_do(pex_builder.add_source, 'src/')
pex_builder.freeze(bytecode_compile=True)
# Entry point verification is disabled because ddit does not
# formally depend on the integration framework, and it is not
# necessarily available at integration build time. Because entry
# point verification happens in ddit's environment and the
# entrypoint is in the framework, this causes entry point
# verification to fail unless some other agent has installed
# daml-dit-if into ddit's environment.
#
# Virtual environments provide a way to work around this (and are
# used in 'ddit run') but the PEX API does not allow a virtual
# environment to be specified at build time. If this ever changes,
# the build subcommand should be modified to prepare a virtual
# enviroment for the build that contains the appropriate version
# of daml-dit-if and entrypoint verification should be re-enabled.
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=False)
LOG.info('Building intermediate PEX file...')
LOG.debug('PEX info: %r', pex_builder.info)
pex_builder.build(pex_filename,
bytecode_compile=True,
deterministic_timestamp=True)
if daml_dit_if_bundled:
return 'python-direct'
else:
return 'python-direct-hub-if'
