def pack_in_pex(requirements: Dict[str, str], output: str) -> str: """ Pack current environment using a pex. :param requirements: list of requirements (ex ['tensorflow==0.1.10']) :param output: location of the pex :return: destination of the archive, name of the pex """ requirements_to_install = format_requirements(requirements) fetchers = [] if _criteo.is_criteo(): fetchers.append(PyPIFetcher(pypi_base=CRITEO_PYPI_URL)) fetchers.append(PyPIFetcher()) resolver_option_builder = ResolverOptionsBuilder(use_manylinux=True, fetchers=fetchers) resolvables = [ Resolvable.get(req, resolver_option_builder) for req in requirements_to_install ] pex_builder = PEXBuilder(copy=True) try: resolveds = resolve_multi(resolvables, use_manylinux=True) for resolved in resolveds: _logger.debug("Add requirement %s", resolved.distribution) pex_builder.add_distribution(resolved.distribution) pex_builder.add_requirement(resolved.requirement) except (Unsatisfiable, Untranslateable): _logger.exception('Cannot create pex') raise pex_builder.build(output) return output
def test_pypifetcher(): fetcher = PyPIFetcher('https://pypi.org/simple') assert fetcher._pypi_base == 'https://pypi.org/simple/' assert fetcher.urls('setuptools') == ['https://pypi.org/simple/setuptools/'] fetcher = PyPIFetcher() assert fetcher._pypi_base == 'https://pypi.org/simple/' assert fetcher.urls('setuptools') == ['https://pypi.org/simple/setuptools/'] fetcher = PyPIFetcher('file:///srv/simple') assert fetcher._pypi_base == 'file:///srv/simple/' assert fetcher.urls('setuptools') == ['file:///srv/simple/setuptools/']
def process_index_url(option, option_str, option_value, parser, builder): indices = getattr(parser.values, option.dest, []) index = PyPIFetcher(option_value) if index not in indices: indices.append(index) setattr(parser.values, option.dest, indices) builder.add_index(option_value)
def test_clp_index_option(): with parser_pair() as (builder, parser): configure_clp_pex_resolution(parser, builder) assert len(builder._fetchers) == 1 options, _ = parser.parse_args(args=['-i', 'http://www.example.com']) assert len(builder._fetchers) == 2 assert builder._fetchers == options.repos assert builder._fetchers[1] == PyPIFetcher('http://www.example.com')
def fetchers_from_config(config): fetchers = [] fetchers.extend( Fetcher([url]) for url in config.getlist('python-repos', 'repos', [])) fetchers.extend( PyPIFetcher(url) for url in config.getlist('python-repos', 'indices', [])) return fetchers
def __init__(self, fetchers=None, crawler=None, follow_links=False, allow_prereleases=None): self._crawler = crawler or Crawler() self._fetchers = fetchers if fetchers is not None else [PyPIFetcher()] self.__follow_links = follow_links self.__allow_prereleases = allow_prereleases
def test_empty_iteration(): crawler_mock = mock.create_autospec(Crawler, spec_set=True) crawler_mock.crawl.return_value = [] iterator = Iterator(crawler=crawler_mock) assert list(iterator.iter(Requirement.parse('foo'))) == [] assert len(crawler_mock.crawl.mock_calls) == 1 _, args, kwargs = crawler_mock.crawl.mock_calls[0] assert list(args[0]) == list(PyPIFetcher().urls(Requirement.parse('foo'))) assert kwargs == {'follow_links': False}
def process_pypi_option(option, option_str, option_value, parser, builder): if option_str.startswith('--no'): setattr(parser.values, option.dest, []) builder.clear_indices() else: indices = getattr(parser.values, option.dest, []) pypi = PyPIFetcher() if pypi not in indices: indices.append(pypi) setattr(parser.values, option.dest, indices) builder.add_index(PyPIFetcher.PYPI_BASE)
def resolve_multi(config, requirements, interpreter=None, platforms=None, conn_timeout=None, ttl=3600): """Multi-platform dependency resolution for PEX files. Given a pants configuration and a set of requirements, return a list of distributions that must be included in order to satisfy them. That may involve distributions for multiple platforms. :param config: Pants :class:`Config` object. :param requirements: A list of :class:`PythonRequirement` objects to resolve. :param interpreter: :class:`PythonInterpreter` for which requirements should be resolved. If None specified, defaults to current interpreter. :param platforms: Optional list of platforms against requirements will be resolved. If None specified, the defaults from `config` will be used. :param conn_timeout: Optional connection timeout for any remote fetching. :param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g. "flask>=0.2" if a matching distribution is available on disk. Defaults to 3600. """ distributions = dict() interpreter = interpreter or PythonInterpreter.get() if not isinstance(interpreter, PythonInterpreter): raise TypeError( 'Expected interpreter to be a PythonInterpreter, got %s' % type(interpreter)) install_cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs') platforms = get_platforms( platforms or config.getlist('python-setup', 'platforms', ['current'])) for platform in platforms: translator = Translator.default(install_cache=install_cache, interpreter=interpreter, platform=platform, conn_timeout=conn_timeout) obtainer = PantsObtainer( install_cache=install_cache, crawler=crawler_from_config(config, conn_timeout=conn_timeout), fetchers=fetchers_from_config(config) or [PyPIFetcher()], translators=translator) distributions[platform] = resolve(requirements=requirements, obtainer=obtainer, interpreter=interpreter, platform=platform) return distributions
def __init__(self, fetchers=None, allow_external=False, allow_unverified=False, allow_prereleases=None, use_manylinux=None, precedence=None, context=None): self._fetchers = fetchers if fetchers is not None else [PyPIFetcher()] self._allow_external = allow_external self._allow_unverified = allow_unverified self._allow_prereleases = allow_prereleases self._use_manylinux = use_manylinux self._precedence = precedence if precedence is not None else Sorter.DEFAULT_PACKAGE_PRECEDENCE self._context = context or Context.get()
def configure_clp_pex_resolution(parser, builder): group = OptionGroup( parser, 'Resolver options', 'Tailor how to find, resolve and translate the packages that get put into the PEX ' 'environment.') group.add_option( '--pypi', '--no-pypi', '--no-index', action='callback', dest='repos', callback=process_pypi_option, callback_args=(builder, ), help='Whether to use pypi to resolve dependencies; Default: use pypi') group.add_option( '-f', '--find-links', '--repo', metavar='PATH/URL', action='callback', dest='repos', callback=process_find_links, callback_args=(builder, ), type=str, help= 'Additional repository path (directory or URL) to look for requirements.' ) group.add_option( '-i', '--index', '--index-url', metavar='URL', action='callback', dest='repos', callback=process_index_url, callback_args=(builder, ), type=str, help='Additional cheeseshop indices to use to satisfy requirements.') group.add_option('--disable-cache', action='callback', dest='cache_dir', callback=process_disable_cache, help='Disable caching in the pex tool entirely.') group.add_option( '--cache-dir', dest='cache_dir', default=os.path.expanduser('~/.pex/build'), help='The local cache directory to use for speeding up requirement ' 'lookups. [Default: %default]') group.add_option( '--cache-ttl', dest='cache_ttl', type=int, default=None, help='The cache TTL to use for inexact requirement specifications.') group.add_option( '--wheel', '--no-wheel', '--no-use-wheel', dest='use_wheel', default=True, action='callback', callback=process_precedence, callback_args=(builder, ), help='Whether to allow wheel distributions; Default: allow wheels') group.add_option( '--build', '--no-build', action='callback', callback=process_precedence, callback_args=(builder, ), help= 'Whether to allow building of distributions from source; Default: allow builds' ) # Set the pex tool to fetch from PyPI by default if nothing is specified. parser.set_default('repos', [PyPIFetcher()]) parser.add_option_group(group)
def build_pex(args, options): interpreter = interpreter_from_options(options) pex_builder = PEXBuilder( path=safe_mkdtemp(), interpreter=interpreter, ) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path installer = WheelInstaller if options.use_wheel else EggInstaller interpreter = interpreter_from_options(options) fetchers = [Fetcher(options.repos)] if options.pypi: fetchers.append(PyPIFetcher()) if options.indices: fetchers.extend(PyPIFetcher(index) for index in options.indices) translator = translator_from_options(options) if options.use_wheel: precedence = (WheelPackage, EggPackage, SourcePackage) else: precedence = (EggPackage, SourcePackage) requirements = options.requirements[:] if options.source_dirs: temporary_package_root = safe_mkdtemp() for source_dir in options.source_dirs: try: sdist = Packager(source_dir).sdist() except installer.Error: die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL) # record the requirement information sdist_pkg = Package.from_href(sdist) requirements.append('%s==%s' % (sdist_pkg.name, sdist_pkg.raw_version)) # copy the source distribution shutil.copyfile( sdist, os.path.join(temporary_package_root, os.path.basename(sdist))) # Tell pex where to find the packages fetchers.append(Fetcher([temporary_package_root])) with TRACER.timed('Resolving distributions'): resolveds = requirement_resolver(requirements, fetchers=fetchers, translator=translator, interpreter=interpreter, platform=options.platform, precedence=precedence, cache=options.cache_dir, cache_ttl=options.cache_ttl) for pkg in resolveds: log(' %s' % pkg, v=options.verbosity) pex_builder.add_distribution(pkg) pex_builder.add_requirement(pkg.as_requirement()) if options.entry_point is not None: log('Setting entry point to %s' % options.entry_point, v=options.verbosity) pex_builder.info.entry_point = options.entry_point else: log('Creating environment PEX.', v=options.verbosity) return pex_builder
def set_index(self, index): self._fetchers = [PyPIFetcher(index)] return self
def add_index(self, index): fetcher = PyPIFetcher(index) if fetcher not in self._fetchers: self._fetchers.append(fetcher) return self
def configure_clp_pex_resolution(parser, builder): group = OptionGroup( parser, 'Resolver options', 'Tailor how to find, resolve and translate the packages that get put into the PEX ' 'environment.') group.add_option( '--pypi', '--no-pypi', '--no-index', action='callback', dest='repos', callback=process_pypi_option, callback_args=(builder,), help='Whether to use pypi to resolve dependencies; Default: use pypi') group.add_option( '--pex-path', dest='pex_path', type=str, default=None, help='A colon separated list of other pex files to merge into the runtime environment.') group.add_option( '-f', '--find-links', '--repo', metavar='PATH/URL', action='callback', dest='repos', callback=process_find_links, callback_args=(builder,), type=str, help='Additional repository path (directory or URL) to look for requirements.') group.add_option( '-i', '--index', '--index-url', metavar='URL', action='callback', dest='repos', callback=process_index_url, callback_args=(builder,), type=str, help='Additional cheeseshop indices to use to satisfy requirements.') group.add_option( '--pre', '--no-pre', dest='allow_prereleases', default=None, action='callback', callback=process_prereleases, callback_args=(builder,), help='Whether to include pre-release and development versions of requirements; ' 'Default: only stable versions are used, unless explicitly requested') group.add_option( '--disable-cache', action='callback', dest='cache_dir', callback=process_disable_cache, help='Disable caching in the pex tool entirely.') group.add_option( '--cache-dir', dest='cache_dir', default='{pex_root}/build', help='The local cache directory to use for speeding up requirement ' 'lookups. [Default: ~/.pex/build]') group.add_option( '--cache-ttl', dest='cache_ttl', type=int, default=3600, help='The cache TTL to use for inexact requirement specifications.') group.add_option( '--wheel', '--no-wheel', '--no-use-wheel', dest='use_wheel', default=True, action='callback', callback=process_precedence, callback_args=(builder,), help='Whether to allow wheel distributions; Default: allow wheels') group.add_option( '--build', '--no-build', action='callback', callback=process_precedence, callback_args=(builder,), help='Whether to allow building of distributions from source; Default: allow builds') group.add_option( '--manylinux', '--no-manylinux', '--no-use-manylinux', dest='use_manylinux', default=True, action='callback', callback=process_precedence, callback_args=(builder,), help=('Whether to allow resolution of manylinux dists for linux target ' 'platforms; Default: allow manylinux')) # Set the pex tool to fetch from PyPI by default if nothing is specified. parser.set_default('repos', [PyPIFetcher()]) parser.add_option_group(group)
def get_fetchers(self): fetchers = [] fetchers.extend(Fetcher([url]) for url in self.repos) fetchers.extend(PyPIFetcher(url) for url in self.indexes) return fetchers
def build_pex(args, options): interpreter = interpreter_from_options(options) pex_builder = PEXBuilder( path=safe_mkdtemp(), interpreter=interpreter, ) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path installer = WheelInstaller if options.use_wheel else EggInstaller interpreter = interpreter_from_options(options) fetchers = [Fetcher(options.repos)] if options.pypi: fetchers.append(PyPIFetcher()) if options.indices: fetchers.extend(PyPIFetcher(index) for index in options.indices) translator = translator_from_options(options) if options.use_wheel: precedence = (WheelPackage, EggPackage, SourcePackage) else: precedence = (EggPackage, SourcePackage) with TRACER.timed('Resolving distributions'): resolveds = requirement_resolver( options.requirements, fetchers=fetchers, translator=translator, interpreter=interpreter, platform=options.platform, precedence=precedence, cache=options.cache_dir, cache_ttl=options.cache_ttl) for pkg in resolveds: log(' %s' % pkg, v=options.verbosity) pex_builder.add_distribution(pkg) pex_builder.add_requirement(pkg.as_requirement()) for source_dir in options.source_dirs: try: bdist = installer(source_dir).bdist() except installer.Error: die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL) pex_builder.add_dist_location(bdist) if options.entry_point is not None: log('Setting entry point to %s' % options.entry_point, v=options.verbosity) pex_builder.info.entry_point = options.entry_point else: log('Creating environment PEX.', v=options.verbosity) return pex_builder