def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreter', V=2): interpreter = interpreter_from_options(options) if interpreter is None: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) resolver_kwargs = dict(interpreter=interpreter, platform=options.platform) if options.cache_dir: resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs) else: resolver = Resolver(**resolver_kwargs) with TRACER.timed('Resolving distributions'): try: resolveds = resolver.resolve(resolvables) except Unsatisfiable as e: die(e) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def test_nested_requirements(): with temporary_dir() as td1: with temporary_dir() as td2: with open(os.path.join(td1, 'requirements.txt'), 'w') as fp: fp.write(dedent(''' requirement1 requirement2 -r %s -r %s ''' % ( os.path.join(td2, 'requirements_nonrelative.txt'), os.path.join('relative', 'requirements_relative.txt')) )) with open(os.path.join(td2, 'requirements_nonrelative.txt'), 'w') as fp: fp.write(dedent(''' requirement3 requirement4 ''')) os.mkdir(os.path.join(td1, 'relative')) with open(os.path.join(td1, 'relative', 'requirements_relative.txt'), 'w') as fp: fp.write(dedent(''' requirement5 requirement6 ''')) def rr(req): return ResolvableRequirement.from_string(req, ResolverOptionsBuilder()) reqs = requirements_from_file(os.path.join(td1, 'requirements.txt')) assert reqs == [rr('requirement%d' % k) for k in (1, 2, 3, 4, 5, 6)]
def build_pex(args, options, resolver_option_builder, interpreter=None): if interpreter is None: with TRACER.timed('Resolving interpreter', V=2): interpreter = interpreter_from_options(options) if interpreter is None: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [ Resolvable.get(arg, resolver_option_builder) for arg in args ] for requirements_txt in options.requirement_files: resolvables.extend( requirements_from_file(requirements_txt, resolver_option_builder)) resolver_kwargs = dict(interpreter=interpreter, platform=options.platform) if options.cache_dir: resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs) else: resolver = Resolver(**resolver_kwargs) with TRACER.timed('Resolving distributions'): try: resolveds = resolver.resolve(resolvables) except Unsatisfiable as e: die(e) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder, interpreter=None): if interpreter is None: with TRACER.timed("Resolving interpreter", V=2): interpreter = interpreter_from_options(options) if interpreter is None: die("Could not find compatible interpreter", CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) resolver_kwargs = dict(interpreter=interpreter, platform=options.platform) if options.cache_dir: resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs) else: resolver = Resolver(**resolver_kwargs) with TRACER.timed("Resolving distributions"): try: resolveds = resolver.resolve(resolvables) except Unsatisfiable as e: die(e) for dist in resolveds: log(" %s" % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) if options.entry_point and options.script: die("Must specify at most one entry point or script.", INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args): with TRACER.timed('Resolving interpreter', V=2): interpreter = _establish_interpreter(args) if interpreter is None: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=_PREAMBLE) pex_info = pex_builder.info pex_info.zip_safe = False pex_info.always_write_cache = True pex_info.inherit_path = False resolver_option_builder = _establish_resolver_options(args) reqs = args.reqs resolvables = [Resolvable.get(req, resolver_option_builder) for req in reqs] for requirements_txt in args.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) resolver_kwargs = dict(interpreter=interpreter, platform=args.platform) _add_spex_deps(resolvables, pex_builder, resolver_option_builder=resolver_option_builder) if not args.disable_cache: resolver = CachingResolver(args.cache_dir, args.cache_ttl, **resolver_kwargs) else: resolver = Resolver(**resolver_kwargs) resolveds = [] with TRACER.timed('Resolving distributions'): try: resolveds = resolver.resolve(resolvables) except Unsatisfiable as exception: die(exception) for dist in resolveds: log(' %s' % dist, verbose=args.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) pex_builder.set_entry_point('spex:spex') if args.python_shebang: pex_builder.set_shebang(args.python_shebang) return pex_builder
def test_nested_requirements(): with temporary_dir() as td1: with temporary_dir() as td2: with open(os.path.join(td1, "requirements.txt"), "w") as fp: fp.write( dedent( """ requirement1 requirement2 -r %s -r %s """ % ( os.path.join(td2, "requirements_nonrelative.txt"), os.path.join("relative", "requirements_relative.txt"), ) ) ) with open(os.path.join(td2, "requirements_nonrelative.txt"), "w") as fp: fp.write( dedent( """ requirement3 requirement4 """ ) ) os.mkdir(os.path.join(td1, "relative")) with open(os.path.join(td1, "relative", "requirements_relative.txt"), "w") as fp: fp.write( dedent( """ requirement5 requirement6 """ ) ) def rr(req): return ResolvableRequirement.from_string(req, ResolverOptionsBuilder()) reqs = requirements_from_file(os.path.join(td1, "requirements.txt")) assert reqs == [rr("requirement%d" % k) for k in (1, 2, 3, 4, 5, 6)]
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): def to_python_interpreter(full_path_or_basename): if os.path.exists(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interpreter = PythonInterpreter.from_env(full_path_or_basename) if interpreter is None: die('Failed to find interpreter: %s' % full_path_or_basename) return interpreter interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]] if options.interpreter_constraint: # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) def walk_and_do(fn, src_dir): src_dir = os.path.normpath(src_dir) for root, dirs, files in os.walk(src_dir): for f in files: src_file_path = os.path.join(root, f) dst_path = os.path.relpath(src_file_path, src_dir) fn(src_file_path, dst_path) for directory in options.sources_directory: walk_and_do(pex_builder.add_source, directory) for directory in options.resources_directory: walk_and_do(pex_builder.add_resource, directory) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter) for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, builder=resolver_option_builder, interpreter=interpreter)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, builder=resolver_option_builder, interpreter=interpreter): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platforms, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed, use_manylinux=options.use_manylinux) for resolved_dist in resolveds: log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution), V=options.verbosity) pex_builder.add_distribution(resolved_dist.distribution) pex_builder.add_requirement(resolved_dist.requirement) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): def to_python_interpreter(full_path_or_basename): if os.path.exists(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interpreter = PythonInterpreter.from_env(full_path_or_basename) if interpreter is None: die('Failed to find interpreter: %s' % full_path_or_basename) return interpreter interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]] if options.interpreter_constraint: # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) if options.rc_file or not ENV.PEX_IGNORE_RCFILES: rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') else: pex_python_path = "" interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) def walk_and_do(fn, src_dir): src_dir = os.path.normpath(src_dir) for root, dirs, files in os.walk(src_dir): for f in files: src_file_path = os.path.join(root, f) dst_path = os.path.relpath(src_file_path, src_dir) fn(src_file_path, dst_path) for directory in options.sources_directory: walk_and_do(pex_builder.add_source, directory) for directory in options.resources_directory: walk_and_do(pex_builder.add_resource, directory) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.emit_warnings = options.emit_warnings pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter) for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, builder=resolver_option_builder, interpreter=interpreter)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, builder=resolver_option_builder, interpreter=interpreter): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platforms, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed, use_manylinux=options.use_manylinux) for resolved_dist in resolveds: log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution), V=options.verbosity) pex_builder.add_distribution(resolved_dist.distribution) pex_builder.add_requirement(resolved_dist.requirement) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): interpreters = [ get_interpreter(interpreter, options.interpreter_cache_dir, options.repos, options.use_wheel) for interpreter in options.python or [None] ] if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = _lowest_version_interpreter(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [ Resolvable.get(arg, resolver_option_builder) for arg in args ] for requirements_txt in options.requirement_files: resolvables.extend( requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platform, cache=options.cache_dir, cache_ttl=options.cache_ttl) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): interpreters = [ get_interpreter(interpreter, options.interpreter_cache_dir, options.repos, options.use_wheel) for interpreter in options.python or [None] ] if options.interpreter_constraint: # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) resolvables = [ Resolvable.get(arg, resolver_option_builder) for arg in args ] for requirements_txt in options.requirement_files: resolvables.extend( requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi( resolvables, interpreters=interpreters, platforms=options.platform, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): interpreters = [ get_interpreter(interpreter, options.interpreter_cache_dir, options.repos, options.use_wheel) for interpreter in options.python or [None] ] if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = _lowest_version_interpreter(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platform, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder