def pack_in_pex(requirements: Dict[str, str], output: str) -> str:
"""
Pack current environment using a pex.
:param requirements: list of requirements (ex ['tensorflow==0.1.10'])
:param output: location of the pex
:return: destination of the archive, name of the pex
"""
requirements_to_install = format_requirements(requirements)
fetchers = []
if _criteo.is_criteo():
fetchers.append(PyPIFetcher(pypi_base=CRITEO_PYPI_URL))
fetchers.append(PyPIFetcher())
resolver_option_builder = ResolverOptionsBuilder(use_manylinux=True,
fetchers=fetchers)
resolvables = [
Resolvable.get(req, resolver_option_builder)
for req in requirements_to_install
]
pex_builder = PEXBuilder(copy=True)
try:
resolveds = resolve_multi(resolvables, use_manylinux=True)
for resolved in resolveds:
_logger.debug("Add requirement %s", resolved.distribution)
pex_builder.add_distribution(resolved.distribution)
pex_builder.add_requirement(resolved.requirement)
except (Unsatisfiable, Untranslateable):
_logger.exception('Cannot create pex')
raise
pex_builder.build(output)
return output
def test_pypifetcher():
fetcher = PyPIFetcher('https://pypi.org/simple')
assert fetcher._pypi_base == 'https://pypi.org/simple/'
assert fetcher.urls('setuptools') == ['https://pypi.org/simple/setuptools/']
fetcher = PyPIFetcher()
assert fetcher._pypi_base == 'https://pypi.org/simple/'
assert fetcher.urls('setuptools') == ['https://pypi.org/simple/setuptools/']
fetcher = PyPIFetcher('file:///srv/simple')
assert fetcher._pypi_base == 'file:///srv/simple/'
assert fetcher.urls('setuptools') == ['file:///srv/simple/setuptools/']
def process_index_url(option, option_str, option_value, parser, builder):
indices = getattr(parser.values, option.dest, [])
index = PyPIFetcher(option_value)
if index not in indices:
indices.append(index)
setattr(parser.values, option.dest, indices)
builder.add_index(option_value)
def test_clp_index_option():
with parser_pair() as (builder, parser):
configure_clp_pex_resolution(parser, builder)
assert len(builder._fetchers) == 1
options, _ = parser.parse_args(args=['-i', 'http://www.example.com'])
assert len(builder._fetchers) == 2
assert builder._fetchers == options.repos
assert builder._fetchers[1] == PyPIFetcher('http://www.example.com')
def fetchers_from_config(config):
fetchers = []
fetchers.extend(
Fetcher([url]) for url in config.getlist('python-repos', 'repos', []))
fetchers.extend(
PyPIFetcher(url)
for url in config.getlist('python-repos', 'indices', []))
return fetchers
def __init__(self,
fetchers=None,
crawler=None,
follow_links=False,
allow_prereleases=None):
self._crawler = crawler or Crawler()
self._fetchers = fetchers if fetchers is not None else [PyPIFetcher()]
self.__follow_links = follow_links
self.__allow_prereleases = allow_prereleases
def test_empty_iteration():
crawler_mock = mock.create_autospec(Crawler, spec_set=True)
crawler_mock.crawl.return_value = []
iterator = Iterator(crawler=crawler_mock)
assert list(iterator.iter(Requirement.parse('foo'))) == []
assert len(crawler_mock.crawl.mock_calls) == 1
_, args, kwargs = crawler_mock.crawl.mock_calls[0]
assert list(args[0]) == list(PyPIFetcher().urls(Requirement.parse('foo')))
assert kwargs == {'follow_links': False}
def process_pypi_option(option, option_str, option_value, parser, builder):
if option_str.startswith('--no'):
setattr(parser.values, option.dest, [])
builder.clear_indices()
else:
indices = getattr(parser.values, option.dest, [])
pypi = PyPIFetcher()
if pypi not in indices:
indices.append(pypi)
setattr(parser.values, option.dest, indices)
builder.add_index(PyPIFetcher.PYPI_BASE)
def resolve_multi(config,
requirements,
interpreter=None,
platforms=None,
conn_timeout=None,
ttl=3600):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param config: Pants :class:`Config` object.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param interpreter: :class:`PythonInterpreter` for which requirements should be resolved.
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param conn_timeout: Optional connection timeout for any remote fetching.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
"""
distributions = dict()
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError(
'Expected interpreter to be a PythonInterpreter, got %s' %
type(interpreter))
install_cache = PythonSetup(config).scratch_dir('install_cache',
default_name='eggs')
platforms = get_platforms(
platforms or config.getlist('python-setup', 'platforms', ['current']))
for platform in platforms:
translator = Translator.default(install_cache=install_cache,
interpreter=interpreter,
platform=platform,
conn_timeout=conn_timeout)
obtainer = PantsObtainer(
install_cache=install_cache,
crawler=crawler_from_config(config, conn_timeout=conn_timeout),
fetchers=fetchers_from_config(config) or [PyPIFetcher()],
translators=translator)
distributions[platform] = resolve(requirements=requirements,
obtainer=obtainer,
interpreter=interpreter,
platform=platform)
return distributions
def __init__(self,
fetchers=None,
allow_external=False,
allow_unverified=False,
allow_prereleases=None,
use_manylinux=None,
precedence=None,
context=None):
self._fetchers = fetchers if fetchers is not None else [PyPIFetcher()]
self._allow_external = allow_external
self._allow_unverified = allow_unverified
self._allow_prereleases = allow_prereleases
self._use_manylinux = use_manylinux
self._precedence = precedence if precedence is not None else Sorter.DEFAULT_PACKAGE_PRECEDENCE
self._context = context or Context.get()
def configure_clp_pex_resolution(parser, builder):
group = OptionGroup(
parser, 'Resolver options',
'Tailor how to find, resolve and translate the packages that get put into the PEX '
'environment.')
group.add_option(
'--pypi',
'--no-pypi',
'--no-index',
action='callback',
dest='repos',
callback=process_pypi_option,
callback_args=(builder, ),
help='Whether to use pypi to resolve dependencies; Default: use pypi')
group.add_option(
'-f',
'--find-links',
'--repo',
metavar='PATH/URL',
action='callback',
dest='repos',
callback=process_find_links,
callback_args=(builder, ),
type=str,
help=
'Additional repository path (directory or URL) to look for requirements.'
)
group.add_option(
'-i',
'--index',
'--index-url',
metavar='URL',
action='callback',
dest='repos',
callback=process_index_url,
callback_args=(builder, ),
type=str,
help='Additional cheeseshop indices to use to satisfy requirements.')
group.add_option('--disable-cache',
action='callback',
dest='cache_dir',
callback=process_disable_cache,
help='Disable caching in the pex tool entirely.')
group.add_option(
'--cache-dir',
dest='cache_dir',
default=os.path.expanduser('~/.pex/build'),
help='The local cache directory to use for speeding up requirement '
'lookups. [Default: %default]')
group.add_option(
'--cache-ttl',
dest='cache_ttl',
type=int,
default=None,
help='The cache TTL to use for inexact requirement specifications.')
group.add_option(
'--wheel',
'--no-wheel',
'--no-use-wheel',
dest='use_wheel',
default=True,
action='callback',
callback=process_precedence,
callback_args=(builder, ),
help='Whether to allow wheel distributions; Default: allow wheels')
group.add_option(
'--build',
'--no-build',
action='callback',
callback=process_precedence,
callback_args=(builder, ),
help=
'Whether to allow building of distributions from source; Default: allow builds'
)
# Set the pex tool to fetch from PyPI by default if nothing is specified.
parser.set_default('repos', [PyPIFetcher()])
parser.add_option_group(group)
def build_pex(args, options):
interpreter = interpreter_from_options(options)
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
installer = WheelInstaller if options.use_wheel else EggInstaller
interpreter = interpreter_from_options(options)
fetchers = [Fetcher(options.repos)]
if options.pypi:
fetchers.append(PyPIFetcher())
if options.indices:
fetchers.extend(PyPIFetcher(index) for index in options.indices)
translator = translator_from_options(options)
if options.use_wheel:
precedence = (WheelPackage, EggPackage, SourcePackage)
else:
precedence = (EggPackage, SourcePackage)
requirements = options.requirements[:]
if options.source_dirs:
temporary_package_root = safe_mkdtemp()
for source_dir in options.source_dirs:
try:
sdist = Packager(source_dir).sdist()
except installer.Error:
die('Failed to run installer for %s' % source_dir,
CANNOT_DISTILL)
# record the requirement information
sdist_pkg = Package.from_href(sdist)
requirements.append('%s==%s' %
(sdist_pkg.name, sdist_pkg.raw_version))
# copy the source distribution
shutil.copyfile(
sdist,
os.path.join(temporary_package_root, os.path.basename(sdist)))
# Tell pex where to find the packages
fetchers.append(Fetcher([temporary_package_root]))
with TRACER.timed('Resolving distributions'):
resolveds = requirement_resolver(requirements,
fetchers=fetchers,
translator=translator,
interpreter=interpreter,
platform=options.platform,
precedence=precedence,
cache=options.cache_dir,
cache_ttl=options.cache_ttl)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point,
v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
return pex_builder
def set_index(self, index): self._fetchers = [PyPIFetcher(index)] return self
def add_index(self, index):
fetcher = PyPIFetcher(index)
if fetcher not in self._fetchers:
self._fetchers.append(fetcher)
return self
def configure_clp_pex_resolution(parser, builder):
group = OptionGroup(
parser,
'Resolver options',
'Tailor how to find, resolve and translate the packages that get put into the PEX '
'environment.')
group.add_option(
'--pypi', '--no-pypi', '--no-index',
action='callback',
dest='repos',
callback=process_pypi_option,
callback_args=(builder,),
help='Whether to use pypi to resolve dependencies; Default: use pypi')
group.add_option(
'--pex-path',
dest='pex_path',
type=str,
default=None,
help='A colon separated list of other pex files to merge into the runtime environment.')
group.add_option(
'-f', '--find-links', '--repo',
metavar='PATH/URL',
action='callback',
dest='repos',
callback=process_find_links,
callback_args=(builder,),
type=str,
help='Additional repository path (directory or URL) to look for requirements.')
group.add_option(
'-i', '--index', '--index-url',
metavar='URL',
action='callback',
dest='repos',
callback=process_index_url,
callback_args=(builder,),
type=str,
help='Additional cheeseshop indices to use to satisfy requirements.')
group.add_option(
'--pre', '--no-pre',
dest='allow_prereleases',
default=None,
action='callback',
callback=process_prereleases,
callback_args=(builder,),
help='Whether to include pre-release and development versions of requirements; '
'Default: only stable versions are used, unless explicitly requested')
group.add_option(
'--disable-cache',
action='callback',
dest='cache_dir',
callback=process_disable_cache,
help='Disable caching in the pex tool entirely.')
group.add_option(
'--cache-dir',
dest='cache_dir',
default='{pex_root}/build',
help='The local cache directory to use for speeding up requirement '
'lookups. [Default: ~/.pex/build]')
group.add_option(
'--cache-ttl',
dest='cache_ttl',
type=int,
default=3600,
help='The cache TTL to use for inexact requirement specifications.')
group.add_option(
'--wheel', '--no-wheel', '--no-use-wheel',
dest='use_wheel',
default=True,
action='callback',
callback=process_precedence,
callback_args=(builder,),
help='Whether to allow wheel distributions; Default: allow wheels')
group.add_option(
'--build', '--no-build',
action='callback',
callback=process_precedence,
callback_args=(builder,),
help='Whether to allow building of distributions from source; Default: allow builds')
group.add_option(
'--manylinux', '--no-manylinux', '--no-use-manylinux',
dest='use_manylinux',
default=True,
action='callback',
callback=process_precedence,
callback_args=(builder,),
help=('Whether to allow resolution of manylinux dists for linux target '
'platforms; Default: allow manylinux'))
# Set the pex tool to fetch from PyPI by default if nothing is specified.
parser.set_default('repos', [PyPIFetcher()])
parser.add_option_group(group)
def get_fetchers(self): fetchers = [] fetchers.extend(Fetcher([url]) for url in self.repos) fetchers.extend(PyPIFetcher(url) for url in self.indexes) return fetchers
def build_pex(args, options):
interpreter = interpreter_from_options(options)
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
installer = WheelInstaller if options.use_wheel else EggInstaller
interpreter = interpreter_from_options(options)
fetchers = [Fetcher(options.repos)]
if options.pypi:
fetchers.append(PyPIFetcher())
if options.indices:
fetchers.extend(PyPIFetcher(index) for index in options.indices)
translator = translator_from_options(options)
if options.use_wheel:
precedence = (WheelPackage, EggPackage, SourcePackage)
else:
precedence = (EggPackage, SourcePackage)
with TRACER.timed('Resolving distributions'):
resolveds = requirement_resolver(
options.requirements,
fetchers=fetchers,
translator=translator,
interpreter=interpreter,
platform=options.platform,
precedence=precedence,
cache=options.cache_dir,
cache_ttl=options.cache_ttl)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
for source_dir in options.source_dirs:
try:
bdist = installer(source_dir).bdist()
except installer.Error:
die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL)
pex_builder.add_dist_location(bdist)
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point, v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
return pex_builder
