def _test_runner(self, targets, workunit): interpreter = self.select_interpreter_for_targets(targets) builder = PEXBuilder(interpreter=interpreter) builder.info.entry_point = 'pytest' chroot = PythonChroot( context=self.context, python_setup=PythonSetup.global_instance(), python_repos=PythonRepos.global_instance(), targets=targets, extra_requirements=self._TESTING_TARGETS, builder=builder, platforms=('current',), interpreter=interpreter) try: builder = chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=interpreter) with self._maybe_shard() as shard_args: with self._maybe_emit_junit_xml(targets) as junit_args: with self._maybe_emit_coverage_data(targets, builder.path(), pex, workunit) as coverage_args: yield pex, shard_args + junit_args + coverage_args finally: chroot.delete()
def resolve_requirements(self, interpreter, req_libs): """Requirements resolution for PEX files. :param interpreter: Resolve against this :class:`PythonInterpreter`. :param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve. :returns: a PEX containing target requirements and any specified python dist targets. """ with self.invalidated(req_libs) as invalidation_check: # If there are no relevant targets, we still go through the motions of resolving # an empty set of requirements, to prevent downstream tasks from having to check # for this special case. if invalidation_check.all_vts: target_set_id = VersionedTargetSet.from_versioned_targets( invalidation_check.all_vts).cache_key.hash else: target_set_id = 'no_targets' # We need to ensure that we are resolving for only the current platform if we are # including local python dist targets that have native extensions. tgts = self.context.targets() maybe_platforms = ['current'] if build_for_current_platform_only_check(tgts) else None path = os.path.realpath(os.path.join(self.workdir, str(interpreter.identity), target_set_id)) # Note that we check for the existence of the directory, instead of for invalid_vts, # to cover the empty case. if not os.path.isdir(path): with safe_concurrent_creation(path) as safe_path: builder = PEXBuilder(path=safe_path, interpreter=interpreter, copy=True) dump_requirement_libs(builder, interpreter, req_libs, self.context.log, platforms=maybe_platforms) builder.freeze() return PEX(path, interpreter=interpreter)
def _build_pex(self, interpreter, path, req_libs): builder = PEXBuilder(path=path, interpreter=interpreter, copy=True) # Gather and de-dup all requirements. reqs = OrderedSet() for req_lib in req_libs: for req in req_lib.requirements: reqs.add(req) # See which ones we need to build. reqs_to_build = OrderedSet() find_links = OrderedSet() for req in reqs: # TODO: should_build appears to be hardwired to always be True. Get rid of it? if req.should_build(interpreter.python, Platform.current()): reqs_to_build.add(req) self.context.log.debug(" Dumping requirement: {}".format(req)) builder.add_requirement(req.requirement) if req.repository: find_links.add(req.repository) else: self.context.log.debug("Skipping {} based on version filter".format(req)) # Resolve the requirements into distributions. distributions = self._resolve_multi(interpreter, reqs_to_build, find_links) locations = set() for platform, dists in distributions.items(): for dist in dists: if dist.location not in locations: self.context.log.debug(" Dumping distribution: .../{}".format(os.path.basename(dist.location))) builder.add_distribution(dist) locations.add(dist.location) builder.freeze()
def nsutil_pex(self): interpreter = self.context.products.get_data(PythonInterpreter) chroot = os.path.join(self.workdir, 'nsutil', interpreter.version_string) if not os.path.exists(chroot): pex_info = PexInfo.default(interpreter=interpreter) with safe_concurrent_creation(chroot) as scratch: builder = PEXBuilder(path=scratch, interpreter=interpreter, pex_info=pex_info, copy=True) with temporary_file(binary_mode=False) as fp: declares_namespace_package_code = inspect.getsource(declares_namespace_package) fp.write(textwrap.dedent(""" import sys {declares_namespace_package_code} if __name__ == '__main__': for path in sys.argv[1:]: if declares_namespace_package(path): print(path) """).strip().format(declares_namespace_package_code=declares_namespace_package_code)) fp.close() builder.set_executable(filename=fp.name, env_filename='main.py') builder.freeze() return PEX(pex=chroot, interpreter=interpreter)
def temporary_chroot(self, interpreter=None, pex_info=None, targets=None, extra_requirements=None, platforms=None, pre_freeze=None): """Yields a temporary PythonChroot created with the specified args. pre_freeze is an optional function run on the chroot just before freezing its builder, to allow for any extra modification. """ path = tempfile.mkdtemp() builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info) with self.context.new_workunit('chroot'): chroot = PythonChroot( context=self.context, python_setup=PythonSetup.global_instance(), python_repos=PythonRepos.global_instance(), targets=targets, extra_requirements=extra_requirements, builder=builder, platforms=platforms, interpreter=interpreter) chroot.dump() if pre_freeze: pre_freeze(chroot) builder.freeze() yield chroot chroot.delete()
def write_pex(td, exe_contents): with open(os.path.join(td, 'exe.py'), 'w') as fp: fp.write(exe_contents) pb = PEXBuilder(path=td) pb.set_executable(os.path.join(td, 'exe.py')) pb.freeze() return pb
def execute(self): if self.old_options.pex and self.old_options.ipython: self.error('Cannot specify both --pex and --ipython!') if self.old_options.entry_point and self.old_options.ipython: self.error('Cannot specify both --entry_point and --ipython!') if self.old_options.verbose: print('Build operating on targets: %s' % ' '.join(str(target) for target in self.targets)) builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter, pex_info=self.binary.pexinfo if self.binary else None) if self.old_options.entry_point: builder.set_entry_point(self.old_options.entry_point) if self.old_options.ipython: if not self.config.has_section('python-ipython'): self.error('No python-ipython sections defined in your pants.ini!') builder.info.entry_point = self.config.get('python-ipython', 'entry_point') if builder.info.entry_point is None: self.error('Must specify entry_point for IPython in the python-ipython section ' 'of your pants.ini!') requirements = self.config.getlist('python-ipython', 'requirements', default=[]) for requirement in requirements: self.extra_requirements.append(PythonRequirement(requirement)) executor = PythonChroot( targets=self.targets, extra_requirements=self.extra_requirements, builder=builder, platforms=self.binary.platforms if self.binary else None, interpreter=self.interpreter, conn_timeout=self.old_options.conn_timeout) executor.dump() if self.old_options.pex: pex_name = self.binary.name if self.binary else Target.maybe_readable_identify(self.targets) pex_path = os.path.join(self.root_dir, 'dist', '%s.pex' % pex_name) builder.build(pex_path) print('Wrote %s' % pex_path) return 0 else: builder.freeze() pex = PEX(builder.path(), interpreter=self.interpreter) po = pex.run(args=list(self.args), blocking=False) try: return po.wait() except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def _resolve_requirements_for_versioned_target_closure(self, interpreter, vt): reqs_pex_path = os.path.realpath(os.path.join(self.workdir, str(interpreter.identity), vt.cache_key.hash)) if not os.path.isdir(reqs_pex_path): req_libs = filter(has_python_requirements, vt.target.closure()) with safe_concurrent_creation(reqs_pex_path) as safe_path: builder = PEXBuilder(safe_path, interpreter=interpreter, copy=True) dump_requirement_libs(builder, interpreter, req_libs, self.context.log) builder.freeze() return PEX(reqs_pex_path, interpreter=interpreter)
def create_pex(self, pex_info=None): """Returns a wrapped pex that "merges" the other pexes via PEX_PATH.""" relevant_targets = self.context.targets( lambda tgt: isinstance(tgt, (PythonRequirementLibrary, PythonTarget, Resources))) with self.invalidated(relevant_targets) as invalidation_check: # If there are no relevant targets, we still go through the motions of resolving # an empty set of requirements, to prevent downstream tasks from having to check # for this special case. if invalidation_check.all_vts: target_set_id = VersionedTargetSet.from_versioned_targets( invalidation_check.all_vts).cache_key.hash else: target_set_id = 'no_targets' interpreter = self.context.products.get_data(PythonInterpreter) path = os.path.join(self.workdir, str(interpreter.identity), target_set_id) extra_pex_paths_file_path = path + '.extra_pex_paths' extra_pex_paths = None # Note that we check for the existence of the directory, instead of for invalid_vts, # to cover the empty case. if not os.path.isdir(path): pexes = [ self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX), self.context.products.get_data(GatherSources.PYTHON_SOURCES) ] if self.extra_requirements(): extra_reqs = [PythonRequirement(req_str) for req_str in self.extra_requirements()] addr = Address.parse('{}_extra_reqs'.format(self.__class__.__name__)) self.context.build_graph.inject_synthetic_target( addr, PythonRequirementLibrary, requirements=extra_reqs) # Add the extra requirements first, so they take precedence over any colliding version # in the target set's dependency closure. pexes = [self.resolve_requirements([self.context.build_graph.get_target(addr)])] + pexes extra_pex_paths = [pex.path() for pex in pexes if pex] with safe_concurrent_creation(path) as safe_path: builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info) builder.freeze() with open(extra_pex_paths_file_path, 'w') as outfile: for epp in extra_pex_paths: outfile.write(epp) outfile.write(b'\n') if extra_pex_paths is None: with open(extra_pex_paths_file_path, 'r') as infile: extra_pex_paths = [p.strip() for p in infile.readlines()] return WrappedPEX(PEX(os.path.realpath(path), interpreter), extra_pex_paths, interpreter)
def merge_pexes(cls, path, pex_info, interpreter, pexes, interpeter_constraints=None): """Generates a merged pex at path.""" pex_paths = [pex.path() for pex in pexes if pex] if pex_paths: pex_info = pex_info.copy() pex_info.merge_pex_path(':'.join(pex_paths)) with safe_concurrent_creation(path) as safe_path: builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info) if interpeter_constraints: for constraint in interpeter_constraints: builder.add_interpreter_constraint(constraint) builder.freeze()
def write_pex(td, exe_contents, dists=None): dists = dists or [] with open(os.path.join(td, 'exe.py'), 'w') as fp: fp.write(exe_contents) pb = PEXBuilder(path=td) for dist in dists: pb.add_egg(dist.location) pb.set_executable(os.path.join(td, 'exe.py')) pb.freeze() return pb
def bootstrap_conan(self): pex_info = PexInfo.default() pex_info.entry_point = 'conans.conan' conan_bootstrap_dir = os.path.join(get_pants_cachedir(), 'conan_support') conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary') interpreter = PythonInterpreter.get() if not os.path.exists(conan_pex_path): with safe_concurrent_creation(conan_pex_path) as safe_path: builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info) reqs = [PythonRequirement(req) for req in self.get_options().conan_requirements] dump_requirements(builder, interpreter, reqs, logger) builder.freeze() conan_binary = PEX(conan_pex_path, interpreter) return self.ConanBinary(pex=conan_binary)
def write_simple_pex(td, exe_contents=None, dists=None, sources=None, coverage=False, interpreter=None, pex_info=None): """Write a pex file that optionally contains an executable entry point. :param str td: temporary directory path :param str exe_contents: entry point python file :param dists: distributions to include, typically sdists or bdists :type: list of :class:`pex.third_party.pkg_resources.Distribution` :param sources: sources to include, as a list of pairs (env_filename, contents) :type sources: list of (str, str) :param bool coverage: include coverage header :param interpreter: a custom interpreter to use to build the pex :type interpreter: :class:`pex.interpreter.PythonInterpreter` :param pex_info: a custom PexInfo to use to build the pex. :type pex_info: :class:`pex.pex_info.PexInfo` """ dists = dists or [] sources = sources or [] safe_mkdir(td) pb = PEXBuilder(path=td, preamble=COVERAGE_PREAMBLE if coverage else None, interpreter=interpreter, pex_info=pex_info) for dist in dists: pb.add_dist_location(dist.location) for env_filename, contents in sources: src_path = os.path.join(td, env_filename) safe_mkdir(os.path.dirname(src_path)) with open(src_path, 'w') as fp: fp.write(contents) pb.add_source(src_path, env_filename) if exe_contents: with open(os.path.join(td, 'exe.py'), 'w') as fp: fp.write(exe_contents) pb.set_executable(os.path.join(td, 'exe.py')) pb.freeze() return pb
def test_execute_interpreter_dashc_program(): with temporary_dir() as pex_chroot: pex_builder = PEXBuilder(path=pex_chroot) pex_builder.freeze() process = PEX(pex_chroot).run( args=["-c", 'import sys; print(" ".join(sys.argv))', "one"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, blocking=False, ) stdout, stderr = process.communicate() assert 0 == process.returncode assert b"-c one\n" == stdout assert b"" == stderr
def _resolve_requirements_for_versioned_target_closure( self, interpreter, vt): reqs_pex_path = os.path.realpath( os.path.join(self.workdir, str(interpreter.identity), vt.cache_key.hash)) if not os.path.isdir(reqs_pex_path): req_libs = filter(has_python_requirements, vt.target.closure()) with safe_concurrent_creation(reqs_pex_path) as safe_path: builder = PEXBuilder(safe_path, interpreter=interpreter, copy=True) dump_requirements(builder, interpreter, req_libs, self.context.log) builder.freeze() return PEX(reqs_pex_path, interpreter=interpreter)
def _add_test_hello_to_pex(ep): with temporary_dir() as td: hello_file = "\n".join([ "def hello():", " print('hello')", ]) with temporary_file(root_dir=td) as tf: with open(tf.name, 'w') as handle: handle.write(hello_file) pex_builder = PEXBuilder() pex_builder.add_source(tf.name, 'test.py') pex_builder.set_entry_point(ep) pex_builder.freeze() yield pex_builder
def test_execute_interpreter_stdin_program(): with temporary_dir() as pex_chroot: pex_builder = PEXBuilder(path=pex_chroot) pex_builder.freeze() process = PEX(pex_chroot).run(args=['-', 'one', 'two'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, blocking=False) stdout, stderr = process.communicate( input=b'import sys; print(" ".join(sys.argv))') assert 0 == process.returncode assert b'- one two\n' == stdout assert b'' == stderr
def test_pex_executable(): # Tests that pex keeps executable permissions with temporary_dir() as temp_dir: pex_dir = os.path.join(temp_dir, 'pex_dir') safe_mkdir(pex_dir) with open(os.path.join(pex_dir, 'exe.py'), 'w') as fp: fp.write(textwrap.dedent(''' import subprocess import os import sys import my_package path = os.path.join(os.path.dirname(my_package.__file__), 'bin/start.sh') sys.stdout.write(subprocess.check_output([path]).decode('utf-8')) ''')) project_content = { 'setup.py': textwrap.dedent(''' from setuptools import setup setup( name='my_project', version='0.0.0.0', zip_safe=True, packages=['my_package'], package_data={'my_package': ['bin/*']}, install_requires=[], ) '''), 'my_package/__init__.py': 0, 'my_package/bin/start.sh': ("#!/usr/bin/env bash\n" "echo 'hello world from start.sh!'"), 'my_package/my_module.py': 'def do_something():\n print("hello world!")\n', } pex_builder = PEXBuilder(path=pex_dir) with temporary_content(project_content, perms=0o755) as project_dir: installer = WheelBuilder(project_dir) bdist = installer.bdist() pex_builder.add_dist_location(bdist) pex_builder.set_executable(os.path.join(pex_dir, 'exe.py')) pex_builder.freeze() app_pex = os.path.join(os.path.join(temp_dir, 'out_pex_dir'), 'app.pex') pex_builder.build(app_pex) std_out, rc = run_simple_pex(app_pex, env={'PEX_ROOT': os.path.join(temp_dir, '.pex')}) assert rc == 0 assert std_out.decode('utf-8') == 'hello world from start.sh!\n'
def test_execute_interpreter_file_program(): with temporary_dir() as pex_chroot: pex_builder = PEXBuilder(path=pex_chroot) pex_builder.freeze() with tempfile.NamedTemporaryFile() as fp: fp.write(b'import sys; print(" ".join(sys.argv))') fp.flush() process = PEX(pex_chroot).run(args=[fp.name, 'one', 'two'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, blocking=False) stdout, stderr = process.communicate() assert 0 == process.returncode assert '{} one two\n'.format(fp.name).encode('utf-8') == stdout assert b'' == stderr
def test_execute_interpreter_dashm_module(): with temporary_dir() as pex_chroot: pex_builder = PEXBuilder(path=pex_chroot) with temporary_file(root_dir=pex_chroot) as fp: fp.write(b'import sys; print(" ".join(sys.argv))') fp.close() pex_builder.add_source(fp.name, 'foo/bar.py') pex_builder.freeze() process = PEX(pex_chroot).run(args=['-m', 'foo.bar', 'one', 'two'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, blocking=False) stdout, stderr = process.communicate() assert 0 == process.returncode assert b'foo.bar one two\n' == stdout assert b'' == stderr
def test_execute_interpreter_stdin_program(): # type: () -> None with temporary_dir() as pex_chroot: pex_builder = PEXBuilder(path=pex_chroot) pex_builder.freeze() process = PEX(pex_chroot).run( args=["-", "one", "two"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, blocking=False, ) stdout, stderr = process.communicate(input=b'import sys; print(" ".join(sys.argv))') assert 0 == process.returncode assert b"- one two\n" == stdout assert b"" == stderr
def write_and_run_simple_pex(inheriting=False): """Write a pex file that contains an executable entry point :param inheriting: whether this pex should inherit site-packages paths :type inheriting: bool """ with temporary_dir() as td: pex_path = os.path.join(td, 'show_path.pex') with open(os.path.join(td, 'exe.py'), 'w') as fp: fp.write('') # No contents, we just want the startup messages pb = PEXBuilder(path=td, preamble=None) pb.info.inherit_path = inheriting pb.set_executable(os.path.join(td, 'exe.py')) pb.freeze() pb.build(pex_path) yield run_simple_pex(pex_path, env={'PEX_VERBOSE': '1'})[0]
def bootstrap_conan(self): pex_info = PexInfo.default() pex_info.entry_point = 'conans.conan' conan_bootstrap_dir = os.path.join(get_pants_cachedir(), 'conan_support') conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary') interpreter = PythonInterpreter.get() if not os.path.exists(conan_pex_path): with safe_concurrent_creation(conan_pex_path) as safe_path: builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info) reqs = [ PythonRequirement(req) for req in self.get_options().conan_requirements ] dump_requirements(builder, interpreter, reqs, logger) builder.freeze() conan_binary = PEX(conan_pex_path, interpreter) return self.ConanBinary(pex=conan_binary)
def resolve_requirement_strings(self, interpreter, requirement_strings): """Resolve a list of pip-style requirement strings.""" requirement_strings = sorted(requirement_strings) if len(requirement_strings) == 0: req_strings_id = 'no_requirements' elif len(requirement_strings) == 1: req_strings_id = requirement_strings[0] else: req_strings_id = hash_all(requirement_strings) path = os.path.realpath(os.path.join(self.workdir, str(interpreter.identity), req_strings_id)) if not os.path.isdir(path): reqs = [PythonRequirement(req_str) for req_str in requirement_strings] with safe_concurrent_creation(path) as safe_path: builder = PEXBuilder(path=safe_path, interpreter=interpreter, copy=True) dump_requirements(builder, interpreter, reqs, self.context.log) builder.freeze() return PEX(path, interpreter=interpreter)
def write_and_run_simple_pex(inheriting): # type: (str) -> Iterator[Text] """Write a pex file that contains an executable entry point. :param inheriting: whether this pex should inherit site-packages paths. """ with temporary_dir() as td: pex_path = os.path.join(td, "show_path.pex") with open(os.path.join(td, "exe.py"), "w") as fp: fp.write(u"") # No contents, we just want the startup messages pb = PEXBuilder(path=td, preamble=None) pb.info.inherit_path = inheriting pb.set_executable(os.path.join(td, "exe.py")) pb.freeze() pb.build(pex_path) stdout, _ = run_simple_pex(pex_path, env={"PEX_VERBOSE": "1"}) yield stdout.decode("utf-8")
def execute(self): binary = self.require_single_root_target() if isinstance(binary, PythonBinary): # We can't throw if binary isn't a PythonBinary, because perhaps we were called on a # jvm_binary, in which case we have to no-op and let jvm_run do its thing. # TODO(benjy): Use MutexTask to coordinate this. interpreter = self.context.products.get_data(PythonInterpreter) with temporary_dir() as tmpdir: # Create a wrapper pex to "merge" the other pexes into via PEX_PATH. builder = PEXBuilder(tmpdir, interpreter, pex_info=binary.pexinfo) builder.freeze() pexes = [ self.context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX), self.context.products.get_data(GatherSources.PYTHON_SOURCES), ] # TODO: Expose the path as a property in pex, instead of relying on # fishing it out of the cmdline. pex_path = os.pathsep.join([pex.cmdline()[1] for pex in pexes]) pex = PEX(tmpdir, interpreter) self.context.release_lock() with self.context.new_workunit(name="run", labels=[WorkUnitLabel.RUN]): args = [] for arg in self.get_options().args: args.extend(safe_shlex_split(arg)) args += self.get_passthru_args() po = pex.run(blocking=False, args=args, env={"PEX_PATH": pex_path}) try: result = po.wait() if result != 0: msg = "{interpreter} {entry_point} {args} ... exited non-zero ({code})".format( interpreter=interpreter.binary, entry_point=binary.entry_point, args=" ".join(args), code=result, ) raise TaskError(msg, exit_code=result) except KeyboardInterrupt: po.send_signal(signal.SIGINT) raise
def build_and_check(path, precompile): pb = PEXBuilder(path) pb.add_source(src, 'lib/src.py') pb.set_executable(exe, 'exe.py') pb.freeze(bytecode_compile=precompile) for pyc_file in ('exe.pyc', 'lib/src.pyc', '__main__.pyc'): pyc_exists = os.path.exists(os.path.join(path, pyc_file)) if precompile: assert pyc_exists else: assert not pyc_exists bootstrap_dir = os.path.join(path, PEXBuilder.BOOTSTRAP_DIR) bootstrap_pycs = [] for _, _, files in os.walk(bootstrap_dir): bootstrap_pycs.extend(f for f in files if f.endswith('.pyc')) if precompile: assert len(bootstrap_pycs) > 0 else: assert 0 == len(bootstrap_pycs)
def resolve_requirements(self, interpreter, req_libs): """Requirements resolution for PEX files. :param interpreter: Resolve against this :class:`PythonInterpreter`. :param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve. :returns: a PEX containing target requirements and any specified python dist targets. """ with self.invalidated(req_libs) as invalidation_check: # If there are no relevant targets, we still go through the motions of resolving # an empty set of requirements, to prevent downstream tasks from having to check # for this special case. if invalidation_check.all_vts: target_set_id = VersionedTargetSet.from_versioned_targets( invalidation_check.all_vts).cache_key.hash else: target_set_id = 'no_targets' # We need to ensure that we are resolving for only the current platform if we are # including local python dist targets that have native extensions. tgts = self.context.targets() if self._python_native_code_settings.check_build_for_current_platform_only( tgts): maybe_platforms = ['current'] else: maybe_platforms = None path = os.path.realpath( os.path.join(self.workdir, str(interpreter.identity), target_set_id)) # Note that we check for the existence of the directory, instead of for invalid_vts, # to cover the empty case. if not os.path.isdir(path): with safe_concurrent_creation(path) as safe_path: builder = PEXBuilder(path=safe_path, interpreter=interpreter, copy=True) dump_requirement_libs(builder, interpreter, req_libs, self.context.log, platforms=maybe_platforms) builder.freeze() return PEX(path, interpreter=interpreter)
def _run_python_tests(self, targets, stdout, stderr): coverage_rc = None coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ try: builder = PEXBuilder(interpreter=self.interpreter) builder.info.entry_point = 'pytest' chroot = PythonChroot(targets=targets, extra_requirements=self._TESTING_TARGETS, builder=builder, platforms=('current', ), interpreter=self.interpreter, conn_timeout=self._conn_timeout) builder = chroot.dump() builder.freeze() test_args = [] test_args.extend(PythonTestBuilder.generate_junit_args(targets)) test_args.extend(self.args) if coverage_enabled: coverage_rc, args = self.cov_setup(targets) test_args.extend(args) sources = list( itertools.chain( *[t.sources_relative_to_buildroot() for t in targets])) pex = PEX(builder.path(), interpreter=self.interpreter) rc = pex.run(args=test_args + sources, blocking=True, setsid=True, stdout=stdout, stderr=stderr) # TODO(wickman): If coverage is enabled, write an intermediate .html that points to # each of the coverage reports generated and webbrowser.open to that page. rv = PythonTestResult.rc(rc) except Exception: import traceback print('Failed to run test!', file=stderr) traceback.print_exc() rv = PythonTestResult.exception() finally: if coverage_rc: os.unlink(coverage_rc) return rv
def _test_runner(self, targets, stdout, stderr): builder = PEXBuilder(interpreter=self._interpreter) builder.info.entry_point = 'pytest' chroot = PythonChroot(targets=targets, extra_requirements=self._TESTING_TARGETS, builder=builder, platforms=('current', ), interpreter=self._interpreter) try: builder = chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=self._interpreter) with self._maybe_emit_junit_xml(targets) as junit_args: with self._maybe_emit_coverage_data(targets, builder.path(), pex, stdout, stderr) as coverage_args: yield pex, junit_args + coverage_args finally: chroot.delete()
def write_simple_pex(td, exe_contents, dists=None, sources=None, coverage=False, interpreter=None): """Write a pex file that contains an executable entry point :param td: temporary directory path :param exe_contents: entry point python file :type exe_contents: string :param dists: distributions to include, typically sdists or bdists :param sources: sources to include, as a list of pairs (env_filename, contents) :param coverage: include coverage header :param interpreter: a custom interpreter to use to build the pex """ dists = dists or [] sources = sources or [] safe_mkdir(td) with open(os.path.join(td, 'exe.py'), 'w') as fp: fp.write(exe_contents) pb = PEXBuilder(path=td, preamble=COVERAGE_PREAMBLE if coverage else None, interpreter=interpreter) for dist in dists: pb.add_dist_location(dist.location) for env_filename, contents in sources: src_path = os.path.join(td, env_filename) safe_mkdir(os.path.dirname(src_path)) with open(src_path, 'w') as fp: fp.write(contents) pb.add_source(src_path, env_filename) pb.set_executable(os.path.join(td, 'exe.py')) pb.freeze() return pb
def test_activate_interpreter_different_from_current(): with temporary_dir() as pex_root: interp_version = PY36 if PY2 else PY27 custom_interpreter = PythonInterpreter.from_binary(ensure_python_interpreter(interp_version)) pex_info = PexInfo.default(custom_interpreter) pex_info.pex_root = pex_root with temporary_dir() as pex_chroot: pex_builder = PEXBuilder(path=pex_chroot, interpreter=custom_interpreter, pex_info=pex_info) with make_bdist(interpreter=custom_interpreter) as bdist: pex_builder.add_distribution(bdist) pex_builder.set_entry_point('sys:exit') pex_builder.freeze() pex = PEX(pex_builder.path(), interpreter=custom_interpreter) try: pex._activate() except SystemExit as e: pytest.fail('PEX activation of %s failed with %s' % (pex, e))
def build_and_check(path, precompile): # type: (str, bool) -> None pb = PEXBuilder(path=path) pb.add_source(src, "lib/src.py") pb.set_executable(exe, "exe.py") pb.freeze(bytecode_compile=precompile) for pyc_file in ("exe.pyc", "lib/src.pyc", "__main__.pyc"): pyc_exists = os.path.exists(os.path.join(path, pyc_file)) if precompile: assert pyc_exists else: assert not pyc_exists bootstrap_dir = os.path.join(path, pb.info.bootstrap) bootstrap_pycs = [] # type: List[str] for _, _, files in os.walk(bootstrap_dir): bootstrap_pycs.extend(f for f in files if f.endswith(".pyc")) if precompile: assert len(bootstrap_pycs) > 0 else: assert 0 == len(bootstrap_pycs)
def create(cls, path): # type: (str) -> Pip """Creates a pip tool with PEX isolation at path. :param path: The path to build the pip tool pex at. """ pip_pex_path = os.path.join(path, isolated().pex_hash) with atomic_directory(pip_pex_path, exclusive=True) as chroot: if chroot is not None: from pex.pex_builder import PEXBuilder isolated_pip_builder = PEXBuilder(path=chroot) pythonpath = third_party.expose(["pip", "setuptools", "wheel"]) isolated_pip_environment = third_party.pkg_resources.Environment( search_path=pythonpath ) for dist_name in isolated_pip_environment: for dist in isolated_pip_environment[dist_name]: isolated_pip_builder.add_dist_location(dist=dist.location) with open(os.path.join(isolated_pip_builder.path(), "run_pip.py"), "w") as fp: fp.write( dedent( """\ import os import runpy import sys # Propagate un-vendored setuptools to pip for any legacy setup.py builds it needs to # perform. os.environ['__PEX_UNVENDORED__'] = '1' os.environ['PYTHONPATH'] = os.pathsep.join(sys.path) runpy.run_module('pip', run_name='__main__') """ ) ) isolated_pip_builder.set_executable(fp.name) isolated_pip_builder.freeze() return cls(pip_pex_path)
def _build_chroot(self, path, interpreter, pex_info, targets, platforms, extra_requirements=None, executable_file_content=None): """Create a PythonChroot with the specified args.""" builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info, copy=True) with self.context.new_workunit('chroot'): chroot = self.create_chroot( interpreter=interpreter, builder=builder, targets=targets, platforms=platforms, extra_requirements=extra_requirements) chroot.dump() if executable_file_content is not None: with open(os.path.join(path, '{}.py'.format(self.CHROOT_EXECUTABLE_NAME)), 'w') as outfile: outfile.write(executable_file_content) # Override any user-specified entry point, under the assumption that the # executable_file_content does what the user intends (including, probably, calling that # underlying entry point). pex_info.entry_point = self.CHROOT_EXECUTABLE_NAME builder.freeze() return chroot
def assert_namespace_packages_warning(distribution, version, expected_warning): # type: (str, str, bool) -> None requirement = "{}=={}".format(distribution, version) pb = PEXBuilder() for resolved_dist in resolver.resolve([requirement]): pb.add_dist_location(resolved_dist.distribution.location) pb.freeze() process = PEX(pb.path()).run(args=["-c", ""], blocking=False, stderr=subprocess.PIPE) _, stderr = process.communicate() stderr_text = stderr.decode("utf8") partial_warning_preamble = "PEXWarning: The `pkg_resources` package was loaded" partial_warning_detail = "{} namespace packages:".format(requirement) if expected_warning: assert partial_warning_preamble in stderr_text assert partial_warning_detail in stderr_text else: assert partial_warning_preamble not in stderr_text assert partial_warning_detail not in stderr_text
def test_execute_interpreter_dashm_module(): with temporary_dir() as pex_chroot: pex_builder = PEXBuilder(path=pex_chroot) pex_builder.add_source(None, "foo/__init__.py") with tempfile.NamedTemporaryFile() as fp: fp.write(b'import sys; print(" ".join(sys.argv))') fp.flush() pex_builder.add_source(fp.name, "foo/bar.py") pex_builder.freeze() pex = PEX(pex_chroot) process = pex.run( args=["-m", "foo.bar", "one", "two"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, blocking=False, ) stdout, stderr = process.communicate() assert 0 == process.returncode assert b"foo.bar one two\n" == stdout assert b"" == stderr
def _test_runner(self, targets, stdout, stderr): builder = PEXBuilder(interpreter=self._interpreter) builder.info.entry_point = 'pytest' chroot = PythonChroot( targets=targets, extra_requirements=self._TESTING_TARGETS, builder=builder, platforms=('current',), interpreter=self._interpreter) try: builder = chroot.dump() builder.freeze() pex = PEX(builder.path(), interpreter=self._interpreter) with self._maybe_emit_junit_xml(targets) as junit_args: with self._maybe_emit_coverage_data(targets, builder.path(), pex, stdout, stderr) as coverage_args: yield pex, junit_args + coverage_args finally: chroot.delete()
def _build_pex(self, interpreter, path, req_libs): builder = PEXBuilder(path=path, interpreter=interpreter, copy=True) # Gather and de-dup all requirements. reqs = OrderedSet() for req_lib in req_libs: for req in req_lib.requirements: reqs.add(req) # See which ones we need to build. reqs_to_build = OrderedSet() find_links = OrderedSet() for req in reqs: # TODO: should_build appears to be hardwired to always be True. Get rid of it? if req.should_build(interpreter.python, Platform.current()): reqs_to_build.add(req) self.context.log.debug(' Dumping requirement: {}'.format(req)) builder.add_requirement(req.requirement) if req.repository: find_links.add(req.repository) else: self.context.log.debug( 'Skipping {} based on version filter'.format(req)) # Resolve the requirements into distributions. distributions = self._resolve_multi(interpreter, reqs_to_build, find_links) locations = set() for platform, dists in distributions.items(): for dist in dists: if dist.location not in locations: self.context.log.debug( ' Dumping distribution: .../{}'.format( os.path.basename(dist.location))) builder.add_distribution(dist) locations.add(dist.location) builder.freeze()
def _run_python_tests(self, targets, stdout, stderr): coverage_rc = None coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ try: builder = PEXBuilder(interpreter=self.interpreter) builder.info.entry_point = 'pytest' chroot = PythonChroot( targets=targets, extra_requirements=self._TESTING_TARGETS, builder=builder, platforms=('current',), interpreter=self.interpreter, conn_timeout=self._conn_timeout) builder = chroot.dump() builder.freeze() test_args = [] test_args.extend(PythonTestBuilder.generate_junit_args(targets)) test_args.extend(self.args) if coverage_enabled: coverage_rc, args = self.cov_setup(targets) test_args.extend(args) sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets])) pex = PEX(builder.path(), interpreter=self.interpreter) rc = pex.run(args=test_args + sources, blocking=True, setsid=True, stdout=stdout, stderr=stderr) # TODO(wickman): If coverage is enabled, write an intermediate .html that points to # each of the coverage reports generated and webbrowser.open to that page. rv = PythonTestResult.rc(rc) except Exception: import traceback print('Failed to run test!', file=stderr) traceback.print_exc() rv = PythonTestResult.exception() finally: if coverage_rc: os.unlink(coverage_rc) return rv
def create_pex_repository( interpreters=None, # type: Optional[Iterable[PythonInterpreter]] platforms=None, # type: Optional[Iterable[Platform]] requirements=None, # type: Optional[Iterable[str]] requirement_files=None, # type: Optional[Iterable[str]] constraint_files=None, # type: Optional[Iterable[str]] manylinux=None, # type: Optional[str] ): # type: (...) -> str pex_builder = PEXBuilder() for resolved_dist in resolve_multi( interpreters=interpreters, platforms=platforms, requirements=requirements, requirement_files=requirement_files, constraint_files=constraint_files, manylinux=manylinux, ): pex_builder.add_distribution(resolved_dist.distribution) if resolved_dist.direct_requirement: pex_builder.add_requirement(resolved_dist.direct_requirement) pex_builder.freeze() return os.path.realpath(cast(str, pex_builder.path()))
def test_pex_run_strip_env(): with temporary_dir() as pex_root: pex_env = dict(PEX_MODULE='does_not_exist_in_sub_pex', PEX_ROOT=pex_root) with environment_as(**pex_env), temporary_dir() as pex_chroot: pex_builder = PEXBuilder(path=pex_chroot) with tempfile.NamedTemporaryFile(mode="w") as fp: fp.write(dedent(""" import json import os print(json.dumps({k: v for k, v in os.environ.items() if k.startswith("PEX_")})) """)) fp.flush() pex_builder.set_executable(fp.name, 'print_pex_env.py') pex_builder.freeze() stdout, returncode = run_simple_pex(pex_chroot) assert 0 == returncode assert {} == json.loads(stdout.decode('utf-8')), ( 'Expected the entrypoint environment to be stripped of PEX_ environment variables.' ) assert pex_env == {k: v for k, v in os.environ.items() if k.startswith("PEX_")}, ( 'Expected the parent environment to be left un-stripped.' )
def _build_pex(self, interpreter, path, vts): builder = PEXBuilder(path=path, interpreter=interpreter, copy=True) for vt in vts: self._dump_sources(builder, vt.target) builder.freeze()
def _build_pex(self, interpreter, path, targets): builder = PEXBuilder(path=path, interpreter=interpreter, copy=True) for target in targets: dump_sources(builder, target, self.context.log) builder.freeze()
def _build_requirements_pex(self, interpreter, path, req_libs): builder = PEXBuilder(path=path, interpreter=interpreter, copy=True) dump_requirements(builder, interpreter, req_libs, self.context.log) builder.freeze()
def _compile_target(self, vt): """'Compiles' a python target. 'Compiling' means forming an isolated chroot of its sources and transitive deps and then attempting to import each of the target's sources in the case of a python library or else the entry point in the case of a python binary. For a library with sources lib/core.py and lib/util.py a "compiler" main file would look like: if __name__ == '__main__': import lib.core import lib.util For a binary with entry point lib.bin:main the "compiler" main file would look like: if __name__ == '__main__': from lib.bin import main In either case the main file is executed within the target chroot to reveal missing BUILD dependencies. """ target = vt.target with self.context.new_workunit(name=target.address.spec): modules = self._get_modules(target) if not modules: # Nothing to eval, so a trivial compile success. return 0 interpreter = self._get_interpreter_for_target_closure(target) reqs_pex = self._resolve_requirements_for_versioned_target_closure(interpreter, vt) srcs_pex = self._source_pex_for_versioned_target_closure(interpreter, vt) # Create the executable pex. exec_pex_parent = os.path.join(self.workdir, 'executable_pex') executable_file_content = self._get_executable_file_content(exec_pex_parent, modules) hasher = hashlib.sha1() hasher.update(reqs_pex.path().encode('utf-8')) hasher.update(srcs_pex.path().encode('utf-8')) hasher.update(executable_file_content.encode('utf-8')) exec_file_hash = hasher.hexdigest() exec_pex_path = os.path.realpath(os.path.join(exec_pex_parent, exec_file_hash)) if not os.path.isdir(exec_pex_path): with safe_concurrent_creation(exec_pex_path) as safe_path: # Write the entry point. safe_mkdir(safe_path) with open(os.path.join(safe_path, '{}.py'.format(self._EXEC_NAME)), 'w') as outfile: outfile.write(executable_file_content) pex_info = (target.pexinfo if isinstance(target, PythonBinary) else None) or PexInfo() # Override any user-specified entry point, under the assumption that the # executable_file_content does what the user intends (including, probably, calling that # underlying entry point). pex_info.entry_point = self._EXEC_NAME pex_info.pex_path = ':'.join(pex.path() for pex in (reqs_pex, srcs_pex) if pex) builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info) builder.freeze() pex = PEX(exec_pex_path, interpreter) with self.context.new_workunit(name='eval', labels=[WorkUnitLabel.COMPILER, WorkUnitLabel.RUN, WorkUnitLabel.TOOL], cmd=' '.join(pex.cmdline())) as workunit: returncode = pex.run(stdout=workunit.output('stdout'), stderr=workunit.output('stderr')) workunit.set_outcome(WorkUnit.SUCCESS if returncode == 0 else WorkUnit.FAILURE) if returncode != 0: self.context.log.error('Failed to eval {}'.format(target.address.spec)) return returncode
def main(): parser = optparse.OptionParser(usage="usage: %prog [options] output") parser.add_option("--entry-point", default="__main__") parser.add_option("--directory", action="store_true", default=False) parser.add_option( "--no-zip-safe", action="store_false", dest="zip_safe", default=True ) parser.add_option("--python", default="") parser.add_option("--python-version", default="") parser.add_option("--python-shebang", default=None) parser.add_option("--preload", action="append", default=[]) options, args = parser.parse_args() if len(args) == 1: output = args[0] else: parser.error("'output' positional argument is required") return 1 # The manifest is passed via stdin, as it can sometimes get too large # to be passed as a CLA. manifest = json.load(sys.stdin) # The version of pkg_resources.py (from setuptools) on some distros is # too old for PEX. So we keep a recent version in the buck repo and # force it into the process by constructing a custom PythonInterpreter # instance using it. if not options.python: options.python = sys.executable identity = PythonIdentity.get() elif not options.python_version: # Note: this is expensive (~500ms). prefer passing --python-version when possible. identity = PythonInterpreter.from_binary(options.python).identity else: # Convert "CPython 2.7" to "CPython 2 7 0" python_version = options.python_version.replace(".", " ").split() if len(python_version) == 3: python_version.append("0") identity = PythonIdentity.from_id_string(" ".join(python_version)) interpreter = PythonInterpreter(options.python, identity, extras={}) pex_builder = PEXBuilder( path=output if options.directory else None, interpreter=interpreter ) if options.python_shebang is not None: pex_builder.set_shebang(options.python_shebang) # Set whether this PEX as zip-safe, meaning everything will stayed zipped up # and we'll rely on python's zip-import mechanism to load modules from # the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS). pex_builder.info.zip_safe = options.zip_safe # Set the starting point for this PEX. pex_builder.info.entry_point = options.entry_point # Copy in our version of `pkg_resources` & `_markerlib`. copy_package(pex_builder, "pkg_resources", prefix=pex_builder.BOOTSTRAP_DIR) copy_package(pex_builder, "_markerlib", prefix=pex_builder.BOOTSTRAP_DIR) # Add the sources listed in the manifest. for dst, src in manifest["modules"].iteritems(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as e: raise Exception("Failed to add {}: {}".format(src, e)) # Add resources listed in the manifest. for dst, src in manifest["resources"].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Add resources listed in the manifest. for dst, src in manifest["nativeLibraries"].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) if options.directory: pex_builder.freeze(code_hash=False, bytecode_compile=False) else: pex_builder.build(output)
def assert_force_local_implicit_ns_packages_issues_598( interpreter=None, requirements=(), create_ns_packages=True ): def create_foo_bar_setup(name, **extra_args): # type: (str, **Any) -> str setup_args = dict(name=name, version="0.0.1", packages=["foo", "foo.bar"]) if create_ns_packages: setup_args.update(namespace_packages=["foo", "foo.bar"]) if requirements: setup_args.update(install_requires=list(requirements)) setup_args.update(extra_args) return dedent( """ from setuptools import setup setup(**{setup_args!r}) """.format( setup_args=setup_args ) ) def with_foo_bar_ns_packages(content): # type: (Dict[str, str]) -> Dict[str, str] ns_packages = ( { os.path.join( pkg, "__init__.py" ): '__import__("pkg_resources").declare_namespace(__name__)' for pkg in ("foo", "foo/bar") } if create_ns_packages else {} ) ns_packages.update(content) return ns_packages content1 = with_foo_bar_ns_packages( { "foo/bar/spam.py": "identify = lambda: 42", "setup.py": create_foo_bar_setup("foo-bar-spam"), } ) content2 = with_foo_bar_ns_packages( { "foo/bar/eggs.py": dedent( """ # NB: This only works when this content is unpacked loose on the filesystem! def read_self(): with open(__file__) as fp: return fp.read() """ ) } ) content3 = with_foo_bar_ns_packages( { "foobaz": dedent( """\ #!python import sys from foo.bar import baz sys.exit(baz.main()) """ ), "foo/bar/baz.py": dedent( """ import sys from foo.bar import eggs, spam def main(): assert len(eggs.read_self()) > 0 return spam.identify() """ ), "setup.py": create_foo_bar_setup("foo-bar-baz", scripts=["foobaz"]), } ) def add_requirements(builder, cache): # type: (PEXBuilder, str) -> None for resolved_dist in resolve(requirements, cache=cache, interpreter=builder.interpreter): builder.add_distribution(resolved_dist.distribution) if resolved_dist.direct_requirement: builder.add_requirement(resolved_dist.direct_requirement) def add_wheel(builder, content): # type: (PEXBuilder, Dict[str, str]) -> None with temporary_content(content) as project: dist = WheelBuilder(project, interpreter=builder.interpreter).bdist() builder.add_dist_location(dist) def add_sources(builder, content): # type: (PEXBuilder, Dict[str, str]) -> None with temporary_content(content) as project: for path in content.keys(): builder.add_source(os.path.join(project, path), path) with temporary_dir() as root, temporary_dir() as cache: pex_info1 = PexInfo.default() pex_info1.zip_safe = False pex1 = os.path.join(root, "pex1.pex") builder1 = PEXBuilder(interpreter=interpreter, pex_info=pex_info1) add_requirements(builder1, cache) add_wheel(builder1, content1) add_sources(builder1, content2) builder1.build(pex1) pex_info2 = PexInfo.default() pex_info2.pex_path = pex1 pex2 = os.path.join(root, "pex2") builder2 = PEXBuilder(path=pex2, interpreter=interpreter, pex_info=pex_info2) add_requirements(builder2, cache) add_wheel(builder2, content3) builder2.set_script("foobaz") builder2.freeze() assert 42 == PEX(pex2, interpreter=interpreter).run(env=dict(PEX_VERBOSE="9"))