def pack_in_pex(requirements: Dict[str, str], output: str) -> str: """ Pack current environment using a pex. :param requirements: list of requirements (ex ['tensorflow==0.1.10']) :param output: location of the pex :return: destination of the archive, name of the pex """ requirements_to_install = format_requirements(requirements) fetchers = [] if _criteo.is_criteo(): fetchers.append(PyPIFetcher(pypi_base=CRITEO_PYPI_URL)) fetchers.append(PyPIFetcher()) resolver_option_builder = ResolverOptionsBuilder(use_manylinux=True, fetchers=fetchers) resolvables = [ Resolvable.get(req, resolver_option_builder) for req in requirements_to_install ] pex_builder = PEXBuilder(copy=True) try: resolveds = resolve_multi(resolvables, use_manylinux=True) for resolved in resolveds: _logger.debug("Add requirement %s", resolved.distribution) pex_builder.add_distribution(resolved.distribution) pex_builder.add_requirement(resolved.requirement) except (Unsatisfiable, Untranslateable): _logger.exception('Cannot create pex') raise pex_builder.build(output) return output
def test_resolvable_requirement(): req = 'foo[bar]==2.3.4' resolvable = ResolvableRequirement.from_string( req, ResolverOptionsBuilder(fetchers=[])) assert resolvable.requirement == pkg_resources.Requirement.parse( 'foo[bar]==2.3.4') assert resolvable.name == 'foo' assert resolvable.exact is True assert resolvable.extras() == ['bar'] assert resolvable.options._fetchers == [] assert resolvable.packages() == [] source_pkg = SourcePackage.from_href('foo-2.3.4.tar.gz') mock_iterator = mock.create_autospec(Iterator, spec_set=True) mock_iterator.iter.return_value = iter([source_pkg]) assert resolvable.compatible(mock_iterator) == [source_pkg] assert mock_iterator.iter.mock_calls == [ mock.call(pkg_resources.Requirement.parse('foo[bar]==2.3.4')) ] # test non-exact resolvable = ResolvableRequirement.from_string('foo', ResolverOptionsBuilder()) assert resolvable.exact is False # test Resolvable.get, which should delegate to a ResolvableRequirement in this case assert Resolvable.get('foo') == ResolvableRequirement.from_string( 'foo', ResolverOptionsBuilder())
def _get_distributions(self, resolver_options_builder): eggs = self.target.get("eggs").splitlines() resolvables = [ Resolvable.get(e, resolver_options_builder) for e in eggs ] resolver = Resolver() return resolver.resolve(resolvables)
def test_resolvable_package(): builder = ResolverOptionsBuilder() source_name = 'foo-2.3.4.tar.gz' pkg = SourcePackage.from_href(source_name) resolvable = ResolvablePackage.from_string(source_name, builder) assert resolvable.packages() == [pkg] mock_iterator = mock.create_autospec(Iterator, spec_set=True) mock_iterator.iter.return_value = iter([]) # fetchers are currently unused for static packages. assert resolvable.compatible(mock_iterator) == [] assert mock_iterator.iter.mock_calls == [] assert resolvable.name == 'foo' assert resolvable.exact is True assert resolvable.extras() == [] resolvable = ResolvablePackage.from_string(source_name + '[extra1,extra2]', builder) assert resolvable.extras() == ['extra1', 'extra2'] assert Resolvable.get('foo-2.3.4.tar.gz') == ResolvablePackage.from_string( 'foo-2.3.4.tar.gz', builder) with pytest.raises(ResolvablePackage.InvalidRequirement): ResolvablePackage.from_string('foo', builder)
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreter', V=2): interpreter = interpreter_from_options(options) if interpreter is None: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) resolver_kwargs = dict(interpreter=interpreter, platform=options.platform) if options.cache_dir: resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs) else: resolver = Resolver(**resolver_kwargs) with TRACER.timed('Resolving distributions'): try: resolveds = resolver.resolve(resolvables) except Unsatisfiable as e: die(e) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def _add_spex_deps(resolvables, pex_builder, resolver_option_builder=None): spex = [pkg for pkg in pkg_resources.working_set if pkg.project_name == 'spex'] assert len(spex) == 1, 'Too many spex distributions' spex = spex[0] spex_reqs = (Resolvable.get(str(req), resolver_option_builder) for req in spex.requires()) resolvables.extend(spex_reqs) pex_builder.add_distribution(spex) pex_builder.add_requirement(spex.as_requirement())
def build_pex(args, options, resolver_option_builder, interpreter=None): if interpreter is None: with TRACER.timed('Resolving interpreter', V=2): interpreter = interpreter_from_options(options) if interpreter is None: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [ Resolvable.get(arg, resolver_option_builder) for arg in args ] for requirements_txt in options.requirement_files: resolvables.extend( requirements_from_file(requirements_txt, resolver_option_builder)) resolver_kwargs = dict(interpreter=interpreter, platform=options.platform) if options.cache_dir: resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs) else: resolver = Resolver(**resolver_kwargs) with TRACER.timed('Resolving distributions'): try: resolveds = resolver.resolve(resolvables) except Unsatisfiable as e: die(e) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder, interpreter=None): if interpreter is None: with TRACER.timed("Resolving interpreter", V=2): interpreter = interpreter_from_options(options) if interpreter is None: die("Could not find compatible interpreter", CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) resolver_kwargs = dict(interpreter=interpreter, platform=options.platform) if options.cache_dir: resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs) else: resolver = Resolver(**resolver_kwargs) with TRACER.timed("Resolving distributions"): try: resolveds = resolver.resolve(resolvables) except Unsatisfiable as e: die(e) for dist in resolveds: log(" %s" % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) if options.entry_point and options.script: die("Must specify at most one entry point or script.", INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args): with TRACER.timed('Resolving interpreter', V=2): interpreter = _establish_interpreter(args) if interpreter is None: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=_PREAMBLE) pex_info = pex_builder.info pex_info.zip_safe = False pex_info.always_write_cache = True pex_info.inherit_path = False resolver_option_builder = _establish_resolver_options(args) reqs = args.reqs resolvables = [Resolvable.get(req, resolver_option_builder) for req in reqs] for requirements_txt in args.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) resolver_kwargs = dict(interpreter=interpreter, platform=args.platform) _add_spex_deps(resolvables, pex_builder, resolver_option_builder=resolver_option_builder) if not args.disable_cache: resolver = CachingResolver(args.cache_dir, args.cache_ttl, **resolver_kwargs) else: resolver = Resolver(**resolver_kwargs) resolveds = [] with TRACER.timed('Resolving distributions'): try: resolveds = resolver.resolve(resolvables) except Unsatisfiable as exception: die(exception) for dist in resolveds: log(' %s' % dist, verbose=args.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) pex_builder.set_entry_point('spex:spex') if args.python_shebang: pex_builder.set_shebang(args.python_shebang) return pex_builder
def test_resolvable_requirement(): req = 'foo[bar]==2.3.4' resolvable = ResolvableRequirement.from_string(req, ResolverOptionsBuilder(fetchers=[])) assert resolvable.requirement == pkg_resources.Requirement.parse('foo[bar]==2.3.4') assert resolvable.name == 'foo' assert resolvable.exact is True assert resolvable.extras() == ['bar'] assert resolvable.options._fetchers == [] assert resolvable.packages() == [] source_pkg = SourcePackage.from_href('foo-2.3.4.tar.gz') mock_iterator = mock.create_autospec(Iterator, spec_set=True) mock_iterator.iter.return_value = iter([source_pkg]) assert resolvable.compatible(mock_iterator) == [source_pkg] assert mock_iterator.iter.mock_calls == [ mock.call(pkg_resources.Requirement.parse('foo[bar]==2.3.4'))] # test non-exact resolvable = ResolvableRequirement.from_string('foo', ResolverOptionsBuilder()) assert resolvable.exact is False # test Resolvable.get, which should delegate to a ResolvableRequirement in this case assert Resolvable.get('foo') == ResolvableRequirement.from_string( 'foo', ResolverOptionsBuilder())
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): interpreters = [ get_interpreter(interpreter, options.interpreter_cache_dir, options.repos, options.use_wheel) for interpreter in options.python or [None] ] if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = _lowest_version_interpreter(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [ Resolvable.get(arg, resolver_option_builder) for arg in args ] for requirements_txt in options.requirement_files: resolvables.extend( requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platform, cache=options.cache_dir, cache_ttl=options.cache_ttl) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): interpreters = [ get_interpreter(interpreter, options.interpreter_cache_dir, options.repos, options.use_wheel) for interpreter in options.python or [None] ] if options.interpreter_constraint: # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) resolvables = [ Resolvable.get(arg, resolver_option_builder) for arg in args ] for requirements_txt in options.requirement_files: resolvables.extend( requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi( resolvables, interpreters=interpreters, platforms=options.platform, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def requirements_from_lines(lines, builder=None, relpath=None, interpreter=None): relpath = relpath or os.getcwd() builder = builder.clone() if builder else ResolverOptionsBuilder() to_resolve = [] for line in lines: line = line.strip() if not line or line.startswith('#'): continue elif line.startswith('-e '): raise UnsupportedLine('Editable distributions not supported: %s' % line) elif _startswith_any(line, ('-i ', '--index-url')): builder.set_index(_get_parameter(line)) elif line.startswith('--extra-index-url'): builder.add_index(_get_parameter(line)) elif _startswith_any(line, ('-f ', '--find-links')): builder.add_repository(_get_parameter(line)) elif line.startswith('--allow-external'): builder.allow_external(_get_parameter(line)) elif line.startswith('--allow-all-external'): builder.allow_all_external() elif line.startswith('--allow-unverified'): builder.allow_unverified(_get_parameter(line)) elif line.startswith('--pre'): builder.allow_prereleases(True) elif line.startswith('--no-pre'): builder.allow_prereleases(False) elif line.startswith('--no-index'): builder.clear_indices() elif line.startswith('--no-use-wheel'): builder.no_use_wheel() # defer the conversion of strings/files to resolvables until all options defined # within the current grouping of lines has been processed. elif _startswith_any(line, ('-r ', '--requirement')): path = os.path.join(relpath, _get_parameter(line)) to_resolve.append(RequirementsTxtSentinel(path)) else: to_resolve.append(line) resolvables = [] for resolvable in to_resolve: if isinstance(resolvable, RequirementsTxtSentinel): resolvables.extend( requirements_from_file(resolvable.filename, builder=builder, interpreter=interpreter)) else: try: resolvables.append( Resolvable.get(resolvable, options_builder=builder, interpreter=interpreter)) except Resolvable.Error as e: raise UnsupportedLine('Could not resolve line: %s (%s)' % (resolvable, e)) return resolvables
def build_site_packages(): """Use PEX to resolve dependencies in a virtual environment, with some customizations to reduce the size of our build. https://www.pantsbuild.org/pex.html """ # Remove flywheel_cli from cache # If you skip this step, it doesn't automatically update the python code if os.path.isdir(PEX_BUILD_CACHE_DIR): for name in os.listdir(PEX_BUILD_CACHE_DIR): if fnmatch.fnmatch(name, 'flywheel_cli*.whl'): path = os.path.join(PEX_BUILD_CACHE_DIR, name) print('Removing {} from cache...'.format(name)) os.remove(path) # Read ignore list # See package-ignore.txt, largely we're removing test files and # Multi-megabyte dicoms from the dicom folder ignore_patterns = read_ignore_patterns() # Create resolver # Loosely based on: https://github.com/pantsbuild/pex/blob/982cb9a988949ffff3348b9bca98ae72a0bf8847/pex/bin/pex.py#L577 resolver_option_builder = ResolverOptionsBuilder() resolvables = [ Resolvable.get('flywheel-cli=={}'.format(PYTHON_CLI_VERSION), resolver_option_builder) ] resolver = CachingResolver(PEX_BUILD_CACHE_DIR, None) # Effectively we resolve (possibly from cache) The source and all of the dependency packages # Then create the virtual environment, which contains those files print('Resolving distributions') resolved = resolver.resolve(resolvables) print('Building package lists') builder = PEXBuilder() for dist in resolved: builder.add_distribution(dist) builder.add_requirement(dist.as_requirement()) # After this point, builder.chroot contains a full list of the files print('Compiling package') builder.freeze(bytecode_compile=False) site_packages_path = os.path.join(BUILD_DIR, 'site-packages.zip') # Create an uncompressed site-packages.zip and add all of the discovered files # (Except those that are filtered out) with open(site_packages_path, 'wb') as f: added_files = set() with zipfile.ZipFile(f, 'w') as zf: for filename in sorted(builder.chroot().files()): if is_ignored_file(ignore_patterns, filename): continue if not filename.startswith('.deps'): continue # Determine new path src_path = os.path.join(builder.chroot().chroot, filename) dst_path = '/'.join(filename.split('/')[2:]) # Optionally, compile the file _, ext = os.path.splitext(src_path) if ext == '.py': cfile_path = src_path + 'c' dst_path += 'c' print('Compiling: {}'.format(dst_path)) py_compile.compile(src_path, cfile=cfile_path, dfile=dst_path, optimize=1) src_path = cfile_path if not dst_path in added_files: zf.write(src_path, dst_path) added_files.add(dst_path) return site_packages_path
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): interpreters = [ get_interpreter(interpreter, options.interpreter_cache_dir, options.repos, options.use_wheel) for interpreter in options.python or [None] ] if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = _lowest_version_interpreter(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platform, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def _get_distributions(self, resolver_options_builder): eggs = self.target.get("eggs").splitlines() resolvables = [Resolvable.get(e, resolver_options_builder) for e in eggs] resolver = Resolver() return resolver.resolve(resolvables)