def create( cls, path, # type: str interpreter=None, # type: Optional[PythonInterpreter] ): # type: (...) -> Pip """Creates a pip tool with PEX isolation at path. :param path: The path to assemble the pip tool at. :param interpreter: The interpreter to run Pip with. The current interpreter by default. :return: The path of a PEX that can be used to execute Pip in isolation. """ pip_interpreter = interpreter or PythonInterpreter.get() pip_pex_path = os.path.join(path, isolated().pex_hash) with atomic_directory(pip_pex_path, exclusive=True) as chroot: if not chroot.is_finalized: from pex.pex_builder import PEXBuilder isolated_pip_builder = PEXBuilder(path=chroot.work_dir) isolated_pip_builder.info.venv = True for dist_location in third_party.expose( ["pip", "setuptools", "wheel"]): isolated_pip_builder.add_dist_location(dist=dist_location) isolated_pip_builder.set_script("pip") isolated_pip_builder.freeze() pex_info = PexInfo.from_pex(pip_pex_path) pex_info.add_interpreter_constraint( str(pip_interpreter.identity.requirement)) return cls( ensure_venv( PEX(pip_pex_path, interpreter=pip_interpreter, pex_info=pex_info)))
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreter', V=2): interpreter = interpreter_from_options(options) if interpreter is None: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) resolver_kwargs = dict(interpreter=interpreter, platform=options.platform) if options.cache_dir: resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs) else: resolver = Resolver(**resolver_kwargs) with TRACER.timed('Resolving distributions'): try: resolveds = resolver.resolve(resolvables) except Unsatisfiable as e: die(e) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_isort_pex(cls, context, interpreter, pex_path, requirements_lib): with safe_concurrent_creation(pex_path) as chroot: builder = PEXBuilder(path=chroot, interpreter=interpreter) dump_requirement_libs(builder=builder, interpreter=interpreter, req_libs=[requirements_lib], log=context.log) builder.set_script('isort') builder.freeze()
def build_pex(args, options, resolver_option_builder, interpreter=None): if interpreter is None: with TRACER.timed('Resolving interpreter', V=2): interpreter = interpreter_from_options(options) if interpreter is None: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [ Resolvable.get(arg, resolver_option_builder) for arg in args ] for requirements_txt in options.requirement_files: resolvables.extend( requirements_from_file(requirements_txt, resolver_option_builder)) resolver_kwargs = dict(interpreter=interpreter, platform=options.platform) if options.cache_dir: resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs) else: resolver = Resolver(**resolver_kwargs) with TRACER.timed('Resolving distributions'): try: resolveds = resolver.resolve(resolvables) except Unsatisfiable as e: die(e) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder, interpreter=None): if interpreter is None: with TRACER.timed("Resolving interpreter", V=2): interpreter = interpreter_from_options(options) if interpreter is None: die("Could not find compatible interpreter", CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) resolver_kwargs = dict(interpreter=interpreter, platform=options.platform) if options.cache_dir: resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs) else: resolver = Resolver(**resolver_kwargs) with TRACER.timed("Resolving distributions"): try: resolveds = resolver.resolve(resolvables) except Unsatisfiable as e: die(e) for dist in resolveds: log(" %s" % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) if options.entry_point and options.script: die("Must specify at most one entry point or script.", INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def test_prex_builder_script_from_pex_path(tmpdir): # type: (Any) -> None pex_with_script = os.path.join(str(tmpdir), "script.pex") with built_wheel( name="my_project", entry_points={ "console_scripts": ["my_app = my_project.my_module:do_something"] }, ) as my_whl: pb = PEXBuilder() pb.add_dist_location(my_whl) pb.build(pex_with_script) pex_file = os.path.join(str(tmpdir), "app.pex") pb = PEXBuilder() pb.info.pex_path = pex_with_script pb.set_script("my_app") pb.build(pex_file) assert "hello world!\n" == subprocess.check_output( args=[pex_file]).decode("utf-8")
def build_pex(reqs, options, cache=None): interpreters = None # Default to the current interpreter. pex_python_path = options.python_path # If None, this will result in using $PATH. # TODO(#1075): stop looking at PEX_PYTHON_PATH and solely consult the `--python-path` flag. if pex_python_path is None and (options.rc_file or not ENV.PEX_IGNORE_RCFILES): rc_variables = Variables(rc=options.rc_file) pex_python_path = rc_variables.PEX_PYTHON_PATH # NB: options.python and interpreter constraints cannot be used together. if options.python: with TRACER.timed("Resolving interpreters", V=2): def to_python_interpreter(full_path_or_basename): if os.path.isfile(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interp = PythonInterpreter.from_env(full_path_or_basename) if interp is None: die("Failed to find interpreter: %s" % full_path_or_basename) return interp interpreters = [ to_python_interpreter(interp) for interp in options.python ] elif options.interpreter_constraint: with TRACER.timed("Resolving interpreters", V=2): constraints = options.interpreter_constraint validate_constraints(constraints) try: interpreters = list( iter_compatible_interpreters( path=pex_python_path, interpreter_constraints=constraints)) except UnsatisfiableInterpreterConstraintsError as e: die( e.create_message( "Could not find a compatible interpreter."), CANNOT_SETUP_INTERPRETER, ) platforms = OrderedSet(options.platforms) interpreters = interpreters or [] if options.platforms and options.resolve_local_platforms: with TRACER.timed( "Searching for local interpreters matching {}".format( ", ".join(map(str, platforms)))): candidate_interpreters = OrderedSet( iter_compatible_interpreters(path=pex_python_path)) candidate_interpreters.add(PythonInterpreter.get()) for candidate_interpreter in candidate_interpreters: resolved_platforms = candidate_interpreter.supported_platforms.intersection( platforms) if resolved_platforms: for resolved_platform in resolved_platforms: TRACER.log("Resolved {} for platform {}".format( candidate_interpreter, resolved_platform)) platforms.remove(resolved_platform) interpreters.append(candidate_interpreter) if platforms: TRACER.log( "Could not resolve a local interpreter for {}, will resolve only binary distributions " "for {}.".format( ", ".join(map(str, platforms)), "this platform" if len(platforms) == 1 else "these platforms", )) interpreter = (PythonInterpreter.latest_release_of_min_compatible_version( interpreters) if interpreters else None) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None pex_builder = PEXBuilder( path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble, copy_mode=CopyMode.SYMLINK, include_tools=options.include_tools or options.venv, ) if options.resources_directory: pex_warnings.warn( "The `-R/--resources-directory` option is deprecated. Resources should be added via " "`-D/--sources-directory` instead.") for directory in OrderedSet(options.sources_directory + options.resources_directory): src_dir = os.path.normpath(directory) for root, _, files in os.walk(src_dir): for f in files: src_file_path = os.path.join(root, f) dst_path = os.path.relpath(src_file_path, src_dir) pex_builder.add_source(src_file_path, dst_path) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.unzip = options.unzip pex_info.venv = bool(options.venv) pex_info.venv_bin_path = options.venv pex_info.venv_copies = options.venv_copies pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.emit_warnings = options.emit_warnings pex_info.inherit_path = InheritPath.for_value(options.inherit_path) pex_info.pex_root = options.runtime_pex_root pex_info.strip_pex_env = options.strip_pex_env if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) indexes = compute_indexes(options) for requirements_pex in options.requirements_pexes: pex_builder.add_from_requirements_pex(requirements_pex) with TRACER.timed( "Resolving distributions ({})".format(reqs + options.requirement_files)): if options.cache_ttl: pex_warnings.warn( "The --cache-ttl option is deprecated and no longer has any effect." ) if options.headers: pex_warnings.warn( "The --header option is deprecated and no longer has any effect." ) network_configuration = NetworkConfiguration( retries=options.retries, timeout=options.timeout, proxy=options.proxy, cert=options.cert, client_cert=options.client_cert, ) try: if options.pex_repository: with TRACER.timed("Resolving requirements from PEX {}.".format( options.pex_repository)): resolveds = resolve_from_pex( pex=options.pex_repository, requirements=reqs, requirement_files=options.requirement_files, constraint_files=options.constraint_files, network_configuration=network_configuration, transitive=options.transitive, interpreters=interpreters, platforms=list(platforms), manylinux=options.manylinux, ignore_errors=options.ignore_errors, ) else: with TRACER.timed("Resolving requirements."): resolveds = resolve_multi( requirements=reqs, requirement_files=options.requirement_files, constraint_files=options.constraint_files, allow_prereleases=options.allow_prereleases, transitive=options.transitive, interpreters=interpreters, platforms=list(platforms), indexes=indexes, find_links=options.find_links, resolver_version=ResolverVersion.for_value( options.resolver_version), network_configuration=network_configuration, cache=cache, build=options.build, use_wheel=options.use_wheel, compile=options.compile, manylinux=options.manylinux, max_parallel_jobs=options.max_parallel_jobs, ignore_errors=options.ignore_errors, ) for resolved_dist in resolveds: pex_builder.add_distribution(resolved_dist.distribution) if resolved_dist.direct_requirement: pex_builder.add_requirement( resolved_dist.direct_requirement) except Unsatisfiable as e: die(str(e)) if options.entry_point and options.script: die("Must specify at most one entry point or script.", INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(reqs, options, cache=None): interpreters = None # Default to the current interpreter. pex_python_path = None # Defaults to $PATH if options.rc_file or not ENV.PEX_IGNORE_RCFILES: rc_variables = Variables(rc=options.rc_file) pex_python_path = rc_variables.PEX_PYTHON_PATH # NB: options.python and interpreter constraints cannot be used together. if options.python: with TRACER.timed("Resolving interpreters", V=2): def to_python_interpreter(full_path_or_basename): if os.path.isfile(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interpreter = PythonInterpreter.from_env(full_path_or_basename) if interpreter is None: die("Failed to find interpreter: %s" % full_path_or_basename) return interpreter interpreters = [to_python_interpreter(interp) for interp in options.python] elif options.interpreter_constraint: with TRACER.timed("Resolving interpreters", V=2): constraints = options.interpreter_constraint validate_constraints(constraints) try: interpreters = list(iter_compatible_interpreters(pex_python_path, constraints)) except UnsatisfiableInterpreterConstraintsError as e: die( e.create_message("Could not find a compatible interpreter."), CANNOT_SETUP_INTERPRETER, ) platforms = OrderedSet(options.platforms) interpreters = interpreters or [] if options.platforms and options.resolve_local_platforms: with TRACER.timed( "Searching for local interpreters matching {}".format(", ".join(map(str, platforms))) ): candidate_interpreters = OrderedSet(iter_compatible_interpreters(pex_python_path)) candidate_interpreters.add(PythonInterpreter.get()) for candidate_interpreter in candidate_interpreters: resolved_platforms = candidate_interpreter.supported_platforms.intersection( platforms ) if resolved_platforms: for resolved_platform in resolved_platforms: TRACER.log( "Resolved {} for platform {}".format( candidate_interpreter, resolved_platform ) ) platforms.remove(resolved_platform) interpreters.append(candidate_interpreter) if platforms: TRACER.log( "Could not resolve a local interpreter for {}, will resolve only binary distributions " "for {}.".format( ", ".join(map(str, platforms)), "this platform" if len(platforms) == 1 else "these platforms", ) ) interpreter = min(interpreters) if interpreters else None if options.use_first_matching_interpreter and interpreters: if len(interpreters) > 1: unused_interpreters = set(interpreters) - {interpreter} TRACER.log( "Multiple interpreters resolved, but only using {} because " "`--use-first-matching-interpreter` was used. These interpreters were matched but " "will not be used: {}".format( interpreter.binary, ", ".join(interpreter.binary for interpreter in sorted(unused_interpreters)), ) ) interpreters = [interpreter] try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) def walk_and_do(fn, src_dir): src_dir = os.path.normpath(src_dir) for root, dirs, files in os.walk(src_dir): for f in files: src_file_path = os.path.join(root, f) dst_path = os.path.relpath(src_file_path, src_dir) fn(src_file_path, dst_path) for directory in options.sources_directory: walk_and_do(pex_builder.add_source, directory) for directory in options.resources_directory: walk_and_do(pex_builder.add_resource, directory) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.unzip = options.unzip pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.emit_warnings = options.emit_warnings pex_info.inherit_path = options.inherit_path pex_info.pex_root = options.runtime_pex_root pex_info.strip_pex_env = options.strip_pex_env # If we're only building the PEX for the first of many interpreters due to # `--use-first-matching-interpreter` selection, we do not want to enable those same interpreter # constraints at runtime, where they could lead to a different interpreter being selected # leading to a failure to execute the PEX. Instead we rely on the shebang set by that single # interpreter to pick out a similar interpreter at runtime (for a CPython interpreter, the # shebang will be `#!/usr/bin/env pythonX.Y` which should generally be enough to select a # matching interpreter. To be clear though, there are many corners this will not work for # including mismatching abi (python2.7m vs python2.7mu) when the PEX contains platform specific # wheels, etc. if options.interpreter_constraint and not options.use_first_matching_interpreter: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) # NB: `None` means use the default (pypi) index, `[]` means use no indexes. indexes = None if options.indexes != [_PYPI] and options.indexes is not None: indexes = [str(index) for index in options.indexes] for requirements_pex in options.requirements_pexes: pex_builder.add_from_requirements_pex(requirements_pex) with TRACER.timed("Resolving distributions ({})".format(reqs + options.requirement_files)): network_configuration = NetworkConfiguration.create( cache_ttl=options.cache_ttl, retries=options.retries, timeout=options.timeout, headers=options.headers, proxy=options.proxy, cert=options.cert, client_cert=options.client_cert, ) try: resolveds = resolve_multi( requirements=reqs, requirement_files=options.requirement_files, constraint_files=options.constraint_files, allow_prereleases=options.allow_prereleases, transitive=options.transitive, interpreters=interpreters, platforms=list(platforms), indexes=indexes, find_links=options.find_links, network_configuration=network_configuration, cache=cache, build=options.build, use_wheel=options.use_wheel, compile=options.compile, manylinux=options.manylinux, max_parallel_jobs=options.max_parallel_jobs, ignore_errors=options.ignore_errors, ) for resolved_dist in resolveds: log( " %s -> %s" % (resolved_dist.requirement, resolved_dist.distribution), V=options.verbosity, ) pex_builder.add_distribution(resolved_dist.distribution) pex_builder.add_requirement(resolved_dist.requirement) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die("Must specify at most one entry point or script.", INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def assert_force_local_implicit_ns_packages_issues_598( interpreter=None, requirements=(), create_ns_packages=True ): def create_foo_bar_setup(name, **extra_args): # type: (str, **Any) -> str setup_args = dict(name=name, version="0.0.1", packages=["foo", "foo.bar"]) if create_ns_packages: setup_args.update(namespace_packages=["foo", "foo.bar"]) if requirements: setup_args.update(install_requires=list(requirements)) setup_args.update(extra_args) return dedent( """ from setuptools import setup setup(**{setup_args!r}) """.format( setup_args=setup_args ) ) def with_foo_bar_ns_packages(content): # type: (Dict[str, str]) -> Dict[str, str] ns_packages = ( { os.path.join( pkg, "__init__.py" ): '__import__("pkg_resources").declare_namespace(__name__)' for pkg in ("foo", "foo/bar") } if create_ns_packages else {} ) ns_packages.update(content) return ns_packages content1 = with_foo_bar_ns_packages( { "foo/bar/spam.py": "identify = lambda: 42", "setup.py": create_foo_bar_setup("foo-bar-spam"), } ) content2 = with_foo_bar_ns_packages( { "foo/bar/eggs.py": dedent( """ # NB: This only works when this content is unpacked loose on the filesystem! def read_self(): with open(__file__) as fp: return fp.read() """ ) } ) content3 = with_foo_bar_ns_packages( { "foobaz": dedent( """\ #!python import sys from foo.bar import baz sys.exit(baz.main()) """ ), "foo/bar/baz.py": dedent( """ import sys from foo.bar import eggs, spam def main(): assert len(eggs.read_self()) > 0 return spam.identify() """ ), "setup.py": create_foo_bar_setup("foo-bar-baz", scripts=["foobaz"]), } ) def add_requirements(builder, cache): # type: (PEXBuilder, str) -> None for resolved_dist in resolve(requirements, cache=cache, interpreter=builder.interpreter): builder.add_distribution(resolved_dist.distribution) if resolved_dist.direct_requirement: builder.add_requirement(resolved_dist.direct_requirement) def add_wheel(builder, content): # type: (PEXBuilder, Dict[str, str]) -> None with temporary_content(content) as project: dist = WheelBuilder(project, interpreter=builder.interpreter).bdist() builder.add_dist_location(dist) def add_sources(builder, content): # type: (PEXBuilder, Dict[str, str]) -> None with temporary_content(content) as project: for path in content.keys(): builder.add_source(os.path.join(project, path), path) with temporary_dir() as root, temporary_dir() as cache: pex_info1 = PexInfo.default() pex_info1.zip_safe = False pex1 = os.path.join(root, "pex1.pex") builder1 = PEXBuilder(interpreter=interpreter, pex_info=pex_info1) add_requirements(builder1, cache) add_wheel(builder1, content1) add_sources(builder1, content2) builder1.build(pex1) pex_info2 = PexInfo.default() pex_info2.pex_path = pex1 pex2 = os.path.join(root, "pex2") builder2 = PEXBuilder(path=pex2, interpreter=interpreter, pex_info=pex_info2) add_requirements(builder2, cache) add_wheel(builder2, content3) builder2.set_script("foobaz") builder2.freeze() assert 42 == PEX(pex2, interpreter=interpreter).run(env=dict(PEX_VERBOSE="9"))
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): interpreters = [ get_interpreter(interpreter, options.interpreter_cache_dir, options.repos, options.use_wheel) for interpreter in options.python or [None] ] if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = _lowest_version_interpreter(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platform, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): def to_python_interpreter(full_path_or_basename): if os.path.exists(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interpreter = PythonInterpreter.from_env(full_path_or_basename) if interpreter is None: die('Failed to find interpreter: %s' % full_path_or_basename) return interpreter interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]] if options.interpreter_constraint: # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) def walk_and_do(fn, src_dir): src_dir = os.path.normpath(src_dir) for root, dirs, files in os.walk(src_dir): for f in files: src_file_path = os.path.join(root, f) dst_path = os.path.relpath(src_file_path, src_dir) fn(src_file_path, dst_path) for directory in options.sources_directory: walk_and_do(pex_builder.add_source, directory) for directory in options.resources_directory: walk_and_do(pex_builder.add_resource, directory) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter) for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, builder=resolver_option_builder, interpreter=interpreter)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, builder=resolver_option_builder, interpreter=interpreter): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platforms, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed, use_manylinux=options.use_manylinux) for resolved_dist in resolveds: log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution), V=options.verbosity) pex_builder.add_distribution(resolved_dist.distribution) pex_builder.add_requirement(resolved_dist.requirement) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(reqs, options): interpreters = None # Default to the current interpreter. # NB: options.python and interpreter constraints cannot be used together. if options.python: with TRACER.timed('Resolving interpreters', V=2): def to_python_interpreter(full_path_or_basename): if os.path.exists(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interpreter = PythonInterpreter.from_env( full_path_or_basename) if interpreter is None: die('Failed to find interpreter: %s' % full_path_or_basename) return interpreter interpreters = [ to_python_interpreter(interp) for interp in options.python ] elif options.interpreter_constraint: with TRACER.timed('Resolving interpreters', V=2): constraints = options.interpreter_constraint validate_constraints(constraints) if options.rc_file or not ENV.PEX_IGNORE_RCFILES: rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', None) else: pex_python_path = None interpreters = list( iter_compatible_interpreters(pex_python_path, constraints)) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = min(interpreters) if interpreters else None pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) def walk_and_do(fn, src_dir): src_dir = os.path.normpath(src_dir) for root, dirs, files in os.walk(src_dir): for f in files: src_file_path = os.path.join(root, f) dst_path = os.path.relpath(src_file_path, src_dir) fn(src_file_path, dst_path) for directory in options.sources_directory: walk_and_do(pex_builder.add_source, directory) for directory in options.resources_directory: walk_and_do(pex_builder.add_resource, directory) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.emit_warnings = options.emit_warnings pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) # NB: `None` means use the default (pypi) index, `[]` means use no indexes. indexes = None if options.indexes != [_PYPI] and options.indexes is not None: indexes = [str(index) for index in options.indexes] with TRACER.timed( 'Resolving distributions ({})'.format(reqs + options.requirement_files)): try: resolveds = resolve_multi( requirements=reqs, requirement_files=options.requirement_files, constraint_files=options.constraint_files, allow_prereleases=options.allow_prereleases, transitive=options.transitive, interpreters=interpreters, platforms=options.platforms, indexes=indexes, find_links=options.find_links, cache=options.cache_dir, build=options.build, use_wheel=options.use_wheel, compile=options.compile, manylinux=options.manylinux, max_parallel_jobs=options.max_parallel_jobs, ignore_errors=options.ignore_errors) for resolved_dist in resolveds: log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution), V=options.verbosity) pex_builder.add_distribution(resolved_dist.distribution) pex_builder.add_requirement(resolved_dist.requirement) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): def to_python_interpreter(full_path_or_basename): if os.path.exists(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interpreter = PythonInterpreter.from_env(full_path_or_basename) if interpreter is None: die('Failed to find interpreter: %s' % full_path_or_basename) return interpreter interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]] if options.interpreter_constraint: # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) if options.rc_file or not ENV.PEX_IGNORE_RCFILES: rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') else: pex_python_path = "" interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) def walk_and_do(fn, src_dir): src_dir = os.path.normpath(src_dir) for root, dirs, files in os.walk(src_dir): for f in files: src_file_path = os.path.join(root, f) dst_path = os.path.relpath(src_file_path, src_dir) fn(src_file_path, dst_path) for directory in options.sources_directory: walk_and_do(pex_builder.add_source, directory) for directory in options.resources_directory: walk_and_do(pex_builder.add_resource, directory) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.emit_warnings = options.emit_warnings pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter) for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, builder=resolver_option_builder, interpreter=interpreter)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, builder=resolver_option_builder, interpreter=interpreter): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platforms, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed, use_manylinux=options.use_manylinux) for resolved_dist in resolveds: log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution), V=options.verbosity) pex_builder.add_distribution(resolved_dist.distribution) pex_builder.add_requirement(resolved_dist.requirement) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): interpreters = [ get_interpreter(interpreter, options.interpreter_cache_dir, options.repos, options.use_wheel) for interpreter in options.python or [None] ] if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = _lowest_version_interpreter(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [ Resolvable.get(arg, resolver_option_builder) for arg in args ] for requirements_txt in options.requirement_files: resolvables.extend( requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platform, cache=options.cache_dir, cache_ttl=options.cache_ttl) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): interpreters = [ get_interpreter(interpreter, options.interpreter_cache_dir, options.repos, options.use_wheel) for interpreter in options.python or [None] ] if options.interpreter_constraint: # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) resolvables = [ Resolvable.get(arg, resolver_option_builder) for arg in args ] for requirements_txt in options.requirement_files: resolvables.extend( requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi( resolvables, interpreters=interpreters, platforms=options.platform, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
class PexBuilderWrapper: """Wraps PEXBuilder to provide an API that consumes targets and other BUILD file entities.""" class Factory(Subsystem): options_scope = "pex-builder-wrapper" @classmethod def register_options(cls, register): super(PexBuilderWrapper.Factory, cls).register_options(register) # TODO: make an analogy to cls.register_jvm_tool that can be overridden for python subsystems # by a python_requirement_library() target, not just via pants.ini! register( "--setuptools-version", advanced=True, default="40.6.3", fingerprint=True, help= "The setuptools version to include in the pex if namespace packages need " "to be injected.", ) register( "--pex-version", advanced=True, default=pex_version, fingerprint=True, help="The pex version to include in any generated ipex files. " "NOTE: This should ideally be the same as the pex version which pants " f"itself depends on, which right now is {pex_version}.", ) @classmethod def subsystem_dependencies(cls): return super(PexBuilderWrapper.Factory, cls).subsystem_dependencies() + ( PythonRepos, PythonSetup, ) @classmethod def create(cls, builder, log=None, generate_ipex=False): options = cls.global_instance().get_options() setuptools_requirement = f"setuptools=={options.setuptools_version}" pex_requirement = f"pex=={options.pex_version}" log = log or logging.getLogger(__name__) return PexBuilderWrapper( builder=builder, python_repos_subsystem=PythonRepos.global_instance(), python_setup_subsystem=PythonSetup.global_instance(), setuptools_requirement=PythonRequirement( setuptools_requirement), pex_requirement=PythonRequirement(pex_requirement), log=log, generate_ipex=generate_ipex, ) def __init__( self, builder: PEXBuilder, python_repos_subsystem: PythonRepos, python_setup_subsystem: PythonSetup, setuptools_requirement: PythonRequirement, pex_requirement: PythonRequirement, log, generate_ipex: bool = False, ): assert log is not None self._builder = builder self._python_repos_subsystem = python_repos_subsystem self._python_setup_subsystem = python_setup_subsystem self._setuptools_requirement = setuptools_requirement self._pex_requirement = pex_requirement self._log = log self._distributions: Dict[str, Distribution] = {} self._frozen = False self._generate_ipex = generate_ipex # If we generate a .ipex, we need to ensure all the code we copy into the underlying PEXBuilder # is also added to the new PEXBuilder created in `._shuffle_original_build_info_into_ipex()`. self._all_added_sources_resources: List[Path] = [] # If we generate a dehydrated "ipex" file, we need to make sure that it is aware of any special # find_links repos attached to any single requirement, so it can later resolve those # requirements when it is first bootstrapped, using the same resolve options. self._all_find_links: OrderedSet[str] = OrderedSet() def add_requirement_libs_from(self, req_libs, platforms=None): """Multi-platform dependency resolution for PEX files. :param builder: Dump the requirements into this builder. :param interpreter: The :class:`PythonInterpreter` to resolve requirements for. :param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve. :param log: Use this logger. :param platforms: A list of :class:`Platform`s to resolve requirements for. Defaults to the platforms specified by PythonSetup. """ reqs = [req for req_lib in req_libs for req in req_lib.requirements] self.add_resolved_requirements(reqs, platforms=platforms) class SingleDistExtractionError(Exception): pass def extract_single_dist_for_current_platform(self, reqs, dist_key) -> Distribution: """Resolve a specific distribution from a set of requirements matching the current platform. :param list reqs: A list of :class:`PythonRequirement` to resolve. :param str dist_key: The value of `distribution.key` to match for a `distribution` from the resolved requirements. :return: The single :class:`pkg_resources.Distribution` matching `dist_key`. :raises: :class:`self.SingleDistExtractionError` if no dists or multiple dists matched the given `dist_key`. """ distributions = self.resolve_distributions(reqs, platforms=["current"]) try: matched_dist = assert_single_element( dist for dists in distributions.values() for dist in dists if dist.key == dist_key) except (StopIteration, ValueError) as e: raise self.SingleDistExtractionError( f"Exactly one dist was expected to match name {dist_key} in requirements {reqs}: {e!r}" ) return matched_dist def resolve_distributions( self, reqs: List[PythonRequirement], platforms: Optional[List[Platform]] = None, ) -> Dict[str, List[Distribution]]: """Multi-platform dependency resolution. :param reqs: A list of :class:`PythonRequirement` to resolve. :param platforms: A list of platform strings to resolve requirements for. Defaults to the platforms specified by PythonSetup. :returns: A tuple `(map, transitive_reqs)`, where `map` is a dict mapping distribution name to a list of resolved distributions, and `reqs` contains all transitive == requirements needed to resolve the initial given requirements `reqs` for the given platforms. """ deduped_reqs = OrderedSet(reqs) find_links: OrderedSet[str] = OrderedSet() for req in deduped_reqs: self._log.debug(f" Dumping requirement: {req}") self._builder.add_requirement(str(req.requirement)) if req.repository: find_links.add(req.repository) # Resolve the requirements into distributions. distributions = self._resolve_multi( self._builder.interpreter, list(deduped_reqs), platforms, list(find_links), ) return distributions def add_resolved_requirements( self, reqs: List[PythonRequirement], platforms: Optional[List[Platform]] = None, override_ipex_build_do_actually_add_distribution: bool = False, ) -> None: """Multi-platform dependency resolution for PEX files. :param builder: Dump the requirements into this builder. :param interpreter: The :class:`PythonInterpreter` to resolve requirements for. :param reqs: A list of :class:`PythonRequirement` to resolve. :param log: Use this logger. :param platforms: A list of :class:`Platform`s to resolve requirements for. Defaults to the platforms specified by PythonSetup. :param bool override_ipex_build_do_actually_add_distribution: When this PexBuilderWrapper is configured with generate_ipex=True, this method won't add any distributions to the output pex. The internal implementation of this class adds a pex dependency to the output ipex file, and therefore needs to override the default behavior of this method. """ distributions = self.resolve_distributions(reqs, platforms=platforms) locations: Set[str] = set() for platform, dists in distributions.items(): for dist in dists: if dist.location not in locations: if self._generate_ipex and not override_ipex_build_do_actually_add_distribution: self._log.debug( f" *AVOIDING* dumping distribution into ipex: .../{os.path.basename(dist.location)}" ) self._register_distribution(dist) else: self._log.debug( f" Dumping distribution: .../{os.path.basename(dist.location)}" ) self.add_distribution(dist) locations.add(dist.location) def _resolve_multi( self, interpreter: PythonInterpreter, requirements: List[PythonRequirement], platforms: Optional[List[Platform]], find_links: Optional[List[str]], ) -> Dict[str, List[Distribution]]: """Multi-platform dependency resolution for PEX files. Returns a tuple containing a list of distributions that must be included in order to satisfy a set of requirements, and the transitive == requirements for those distributions. This may involve distributions for multiple platforms. :param interpreter: The :class:`PythonInterpreter` to resolve for. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param platforms: A list of :class:`Platform`s to resolve for. :param find_links: Additional paths to search for source packages during resolution. :return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed to satisfy the requirements on that platform. """ python_setup = self._python_setup_subsystem python_repos = self._python_repos_subsystem platforms = platforms or python_setup.platforms find_links = list(find_links) if find_links else [] find_links.extend(python_repos.repos) # Individual requirements from pants may have a `repository` link attached to them, which is # extracted in `self.resolve_distributions()`. When generating a .ipex file with # `generate_ipex=True`, we want to ensure these repos are known to the ipex launcher when it # tries to resolve all the requirements from BOOTSTRAP-PEX-INFO. self._all_find_links.update(OrderedSet(find_links)) distributions: Dict[str, List[Distribution]] = defaultdict(list) for platform in platforms: requirements_cache_dir = os.path.join( python_setup.resolver_cache_dir, str(interpreter.identity)) resolved_dists = resolve( requirements=[str(req.requirement) for req in requirements], interpreter=interpreter, platform=platform, indexes=python_repos.indexes, find_links=find_links, cache=requirements_cache_dir, allow_prereleases=python_setup.resolver_allow_prereleases, manylinux=python_setup.manylinux, ) for resolved_dist in resolved_dists: distributions[platform].append(resolved_dist.distribution) return distributions def _create_source_dumper(self, tgt: Target) -> Callable[[str], None]: buildroot = get_buildroot() def get_chroot_path(relpath: str) -> str: if type(tgt) == Files: # Loose `Files`, as opposed to `Resources` or `PythonTarget`s, have no (implied) package # structure and so we chroot them relative to the build root so that they can be accessed # via the normal Python filesystem APIs just as they would be accessed outside the # chrooted environment. NB: This requires we mark the pex as not zip safe so # these `Files` can still be accessed in the context of a built pex distribution. self._builder.info.zip_safe = False return relpath return str(Path(relpath).relative_to(tgt.target_base)) def dump_source(relpath: str) -> None: source_path = str(Path(buildroot, relpath)) dest_path = get_chroot_path(relpath) self._all_added_sources_resources.append(Path(dest_path)) if has_resources(tgt): self._builder.add_resource(filename=source_path, env_filename=dest_path) else: self._builder.add_source(filename=source_path, env_filename=dest_path) return dump_source def add_sources_from(self, tgt: Target) -> None: dump_source = self._create_source_dumper(tgt) self._log.debug(f" Dumping sources: {tgt}") for relpath in tgt.sources_relative_to_buildroot(): try: dump_source(relpath) except OSError: self._log.error( f"Failed to copy {relpath} for target {tgt.address.spec}") raise if getattr(tgt, "_resource_target_specs", None) or getattr( tgt, "_synthetic_resources_target", None): # No one should be on old-style resources any more. And if they are, # switching to the new python pipeline will be a great opportunity to fix that. raise TaskError( f"Old-style resources not supported for target {tgt.address.spec}. Depend on resources() " "targets instead.") def _prepare_inits(self) -> Set[str]: chroot = self._builder.chroot() sources = chroot.get("source") | chroot.get("resource") missing_init_files = identify_missing_init_files(sources) if missing_init_files: with temporary_file(permissions=0o644) as ns_package: ns_package.write( b'__import__("pkg_resources").declare_namespace(__name__) # type: ignore[attr-defined]' ) ns_package.flush() for missing_init_file in missing_init_files: self._all_added_sources_resources.append( Path(missing_init_file)) self._builder.add_source(filename=ns_package.name, env_filename=missing_init_file) return missing_init_files def set_emit_warnings(self, emit_warnings): self._builder.info.emit_warnings = emit_warnings def _set_major_minor_interpreter_constraint_for_ipex( self, info: PexInfo, identity: PythonIdentity, ) -> PexInfo: interpreter_name = identity.requirement.name major, minor, _patch = identity.version major_minor_only_constraint = f"{interpreter_name}=={major}.{minor}.*" return ipex_launcher.modify_pex_info( info, interpreter_constraints=[str(major_minor_only_constraint)]) def _shuffle_underlying_pex_builder(self) -> Tuple[PexInfo, Path]: """Replace the original builder with a new one, and just pull files from the old chroot.""" # Ensure that (the interpreter selected to resolve requirements when the ipex is first run) is # (the exact same interpreter we used to resolve those requirements here). This is the only (?) # way to ensure that the ipex bootstrap uses the *exact* same interpreter version. self._builder.info = self._set_major_minor_interpreter_constraint_for_ipex( self._builder.info, self._builder.interpreter.identity) # Remove all the original top-level requirements in favor of the transitive == requirements. self._builder.info = ipex_launcher.modify_pex_info(self._builder.info, requirements=[]) transitive_reqs = [ dist.as_requirement() for dist in self._distributions.values() ] self.add_direct_requirements(transitive_reqs) orig_info = self._builder.info.copy() orig_chroot = self._builder.chroot() # Mutate the PexBuilder object which is manipulated by this subsystem. self._builder = PEXBuilder(interpreter=self._builder.interpreter) self._builder.info = self._set_major_minor_interpreter_constraint_for_ipex( self._builder.info, self._builder.interpreter.identity) self._distributions = {} return (orig_info, Path(orig_chroot.path())) def _shuffle_original_build_info_into_ipex(self): """Create a "dehydrated" ipex file without any of its requirements, and specify that in two. *-INFO files. See ipex_launcher.py for details of how these files are used. """ orig_pex_info, orig_chroot = self._shuffle_underlying_pex_builder() # Gather information needed to create IPEX-INFO. all_code = [str(src) for src in self._all_added_sources_resources] prefixed_code_paths = [ os.path.join(ipex_launcher.APP_CODE_PREFIX, src) for src in all_code ] for src, prefixed in zip(all_code, prefixed_code_paths): # NB: Need to add under 'source' label for `self._prepare_inits()` to pick it up! self._builder.chroot().copy(os.path.join(str(orig_chroot), src), prefixed, label="source") python_repos = self._python_repos_subsystem python_setup = self._python_setup_subsystem # NB: self._all_find_links is updated on every call to self._resolve_multi(), and therefore # includes all of the links from python_repos.repos, as well as any links added within any # individual requirements from that resolve. resolver_settings = dict( indexes=list(python_repos.indexes), find_links=list(self._all_find_links), allow_prereleases=UnsetBool.coerce_bool( python_setup.resolver_allow_prereleases, default=True), manylinux=python_setup.manylinux, ) # IPEX-INFO: A json mapping interpreted in ipex_launcher.py: # { # "code": [<which source files to add to the "hydrated" pex when bootstrapped>], # "resolver_settings": {<which indices to search for requirements from when bootstrapping>}, # } ipex_info = dict( code=prefixed_code_paths, resolver_settings=resolver_settings, ) with temporary_file(permissions=0o644) as ipex_info_file: ipex_info_file.write(json.dumps(ipex_info).encode()) ipex_info_file.flush() self._builder.add_resource(filename=ipex_info_file.name, env_filename="IPEX-INFO") # BOOTSTRAP-PEX-INFO: The original PEX-INFO, which should be the PEX-INFO in the hydrated .pex # file that is generated when the .ipex is first executed. with temporary_file(permissions=0o644) as bootstrap_pex_info_file: bootstrap_pex_info_file.write(orig_pex_info.dump().encode()) bootstrap_pex_info_file.flush() self._builder.add_resource(filename=bootstrap_pex_info_file.name, env_filename="BOOTSTRAP-PEX-INFO") # ipex.py: The special bootstrap script to hydrate the .ipex with the fully resolved # requirements when it is first executed. # Extract the file contents of our custom app launcher script from the pants package. parent_module = module_dirname(module_dirname(ipex_launcher.__name__)) ipex_launcher_provider = get_provider(parent_module) ipex_launcher_script = ipex_launcher_provider.get_resource_string( parent_module, "ipex/ipex_launcher.py") with temporary_file(permissions=0o644) as ipex_launcher_file: ipex_launcher_file.write(ipex_launcher_script) ipex_launcher_file.flush() # Our .ipex file will use our custom app launcher! self._builder.set_executable(ipex_launcher_file.name, env_filename="ipex.py") # The PEX-INFO we generate shouldn't have any requirements (except pex itself), or they will # fail to bootstrap because they were unable to find those distributions. Instead, the .pex file # produced when the .ipex is first executed will read and resolve all those requirements from # the BOOTSTRAP-PEX-INFO. self.add_resolved_requirements( [self._pex_requirement, self._setuptools_requirement], override_ipex_build_do_actually_add_distribution=True, ) def freeze(self) -> None: if self._frozen: return if self._prepare_inits(): dist = self._distributions.get("setuptools") if not dist: self.add_resolved_requirements([self._setuptools_requirement]) if self._generate_ipex: self._shuffle_original_build_info_into_ipex() self._builder.freeze(bytecode_compile=False) self._frozen = True def set_entry_point(self, entry_point): self._builder.set_entry_point(entry_point) def build(self, safe_path): self.freeze() self._builder.build(safe_path, bytecode_compile=False, deterministic_timestamp=True) def set_shebang(self, shebang): self._builder.set_shebang(shebang) def add_interpreter_constraint(self, constraint): self._builder.add_interpreter_constraint(constraint) def add_interpreter_constraints_from(self, constraint_tgts): # TODO this would be a great place to validate the constraints and present a good error message # if they are incompatible because all the sources of the constraints are available. # See: https://github.com/pantsbuild/pex/blob/584b6e367939d24bc28aa9fa36eb911c8297dac8/pex/interpreter_constraints.py constraint_tuples = { self._python_setup_subsystem.compatibility_or_constraints( tgt.compatibility) for tgt in constraint_tgts } for constraint_tuple in constraint_tuples: for constraint in constraint_tuple: self.add_interpreter_constraint(constraint) def add_direct_requirements(self, reqs): for req in reqs: self._builder.add_requirement(str(req)) def add_distribution(self, dist): self._builder.add_distribution(dist) self._register_distribution(dist) def add_dist_location(self, location): self._builder.add_dist_location(location) dist = DistributionHelper.distribution_from_path(location) self._register_distribution(dist) def _register_distribution(self, dist): self._distributions[dist.key] = dist def set_script(self, script): self._builder.set_script(script)
def assert_force_local_implicit_ns_packages_issues_598(interpreter=None, requirements=(), create_ns_packages=True): def create_foo_bar_setup(name, **extra_args): setup_args = dict( name=name, version='0.0.1', packages=['foo', 'foo.bar'] ) if create_ns_packages: setup_args.update(namespace_packages=['foo', 'foo.bar']) if requirements: setup_args.update(install_requires=list(requirements)) setup_args.update(extra_args) return dedent(""" from setuptools import setup setup(**{setup_args!r}) """.format(setup_args=setup_args)) def with_foo_bar_ns_packages(content): ns_packages = { os.path.join(pkg, '__init__.py'): '__import__("pkg_resources").declare_namespace(__name__)' for pkg in ('foo', 'foo/bar') } if create_ns_packages else {} ns_packages.update(content) return ns_packages content1 = with_foo_bar_ns_packages({ 'foo/bar/spam.py': 'identify = lambda: 42', 'setup.py': create_foo_bar_setup('foo-bar-spam') }) content2 = with_foo_bar_ns_packages({ 'foo/bar/eggs.py': dedent(""" # NB: This only works when this content is unpacked loose on the filesystem! def read_self(): with open(__file__) as fp: return fp.read() """) }) content3 = with_foo_bar_ns_packages({ 'foobaz': dedent("""\ #!python import sys from foo.bar import baz sys.exit(baz.main()) """), 'foo/bar/baz.py': dedent(""" import sys from foo.bar import eggs, spam def main(): assert len(eggs.read_self()) > 0 return spam.identify() """), 'setup.py': create_foo_bar_setup('foo-bar-baz', scripts=['foobaz']) }) def add_requirements(builder, cache): for resolved_dist in resolve(requirements, cache=cache, interpreter=builder.interpreter): builder.add_requirement(resolved_dist.requirement) builder.add_distribution(resolved_dist.distribution) def add_wheel(builder, content): with temporary_content(content) as project: dist = WheelInstaller(project, interpreter=builder.interpreter).bdist() builder.add_dist_location(dist) def add_sources(builder, content): with temporary_content(content) as project: for path in content.keys(): builder.add_source(os.path.join(project, path), path) with nested(temporary_dir(), temporary_dir()) as (root, cache): pex_info1 = PexInfo.default() pex_info1.zip_safe = False pex1 = os.path.join(root, 'pex1.pex') builder1 = PEXBuilder(interpreter=interpreter, pex_info=pex_info1) add_requirements(builder1, cache) add_wheel(builder1, content1) add_sources(builder1, content2) builder1.build(pex1) pex_info2 = PexInfo.default() pex_info2.pex_path = pex1 pex2 = os.path.join(root, 'pex2') builder2 = PEXBuilder(path=pex2, interpreter=interpreter, pex_info=pex_info2) add_requirements(builder2, cache) add_wheel(builder2, content3) builder2.set_script('foobaz') builder2.freeze() assert 42 == PEX(pex2, interpreter=interpreter).run(env=dict(PEX_VERBOSE='9'))