def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--entry-point', default='__main__')
parser.add_argument('--python', default=sys.executable)
parser.add_argument('output')
args = parser.parse_args(argv[1:])
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
interpreter = PythonInterpreter(
args.python, PythonInterpreter.from_binary(args.python).identity,
extras={('setuptools', '1.0'):
os.path.join(BUCK_ROOT, 'third-party/py/setuptools')})
pex_builder = PEXBuilder(
path=tmp_dir,
interpreter=interpreter,
)
# Mark this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS), so
# we'll want to revisit this.
pex_builder.info.zip_safe = True
# Set the starting point for this PEX.
pex_builder.info.entry_point = args.entry_point
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
pex_builder.add_source(dereference_symlinks(src), dst)
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Generate the PEX file.
pex_builder.build(args.output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def build_pex(args, options):
interpreter = None
if options.python:
if os.path.exists(options.python):
interpreter = PythonInterpreter.from_binary(options.python)
else:
interpreter = PythonInterpreter.from_env(options.python)
if interpreter is None:
die('Failed to find interpreter: %s' % options.python)
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
fetchers = [Fetcher(options.repos)]
if options.pypi:
fetchers.append(PyPIFetcher())
resolveds = requirement_resolver(options.requirements,
cache=options.cache_dir,
fetchers=fetchers,
interpreter=interpreter,
platform=options.platform)
if resolveds:
log('Resolved distributions:', v=options.verbosity)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
for source_dir in options.source_dirs:
try:
egg_path = EggInstaller(source_dir).bdist()
except EggInstaller.Error:
die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL)
pex_builder.add_egg(egg_path)
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point,
v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
return pex_builder
def interpreter_from_options(options):
interpreter = None
if options.python:
if os.path.exists(options.python):
interpreter = PythonInterpreter.from_binary(options.python)
else:
interpreter = PythonInterpreter.from_env(options.python)
if interpreter is None:
die('Failed to find interpreter: %s' % options.python)
else:
interpreter = PythonInterpreter.get()
return interpreter
def cmdline(self, args=()):
"""
The commandline to run this environment.
Optional arguments:
binary: The binary to run instead of the entry point in the environment
interpreter_args: Arguments to be passed to the interpreter before, e.g. '-E' or
['-m', 'pylint.lint']
args: Arguments to be passed to the application being invoked by the environment.
"""
interpreter = PythonInterpreter(sys.executable)
cmds = [interpreter.binary()]
cmds.append(self._pex.path())
cmds.extend(args)
return cmds
def build(self, targets, args, interpreter=None, conn_timeout=None, fast_tests=False):
test_targets = []
binary_targets = []
interpreter = interpreter or PythonInterpreter.get()
for target in targets:
assert target.is_python, "PythonBuilder can only build PythonTargets, given %s" % str(target)
# PythonBuilder supports PythonTests and PythonBinaries
for target in targets:
if isinstance(target, PythonTests):
test_targets.append(target)
elif isinstance(target, PythonBinary):
binary_targets.append(target)
rv = PythonTestBuilder(
test_targets,
args,
interpreter=interpreter,
conn_timeout=conn_timeout,
fast=fast_tests).run()
if rv != 0:
return rv
for binary_target in binary_targets:
rv = PythonBinaryBuilder(
binary_target,
self._run_tracker,
interpreter=interpreter,
conn_timeout=conn_timeout).run()
if rv != 0:
return rv
return 0
def __init__(self,
target,
run_tracker,
interpreter=None,
conn_timeout=None):
self.target = target
self.interpreter = interpreter or PythonInterpreter.get()
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
run_info = run_tracker.run_info
build_properties = {}
build_properties.update(
run_info.add_basic_info(run_id=None, timestamp=time.time()))
build_properties.update(run_info.add_scm_info())
pexinfo = target.pexinfo.copy()
pexinfo.build_properties = build_properties
builder = PEXBuilder(distpath,
pex_info=pexinfo,
interpreter=self.interpreter)
self.chroot = PythonChroot(targets=[target],
builder=builder,
platforms=target.platforms,
interpreter=self.interpreter,
conn_timeout=conn_timeout)
def __init__(self,
target,
root_dir,
extra_targets=None,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._extra_requirements = list(extra_requirements) if extra_requirements is not None else []
self._builder = builder or PEXBuilder(tempfile.mkdtemp(), interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
PythonSetup(self._config).scratch_dir('artifact_cache', default_name='artifacts'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def select_interpreter(self, compatibilities, allow_multiple=False):
if allow_multiple:
return compatibilities
me = PythonInterpreter.get()
if me in compatibilities:
return [me]
return [min(compatibilities)] if compatibilities else []
def __init__(self, target, root_dir, run_tracker, interpreter=None, conn_timeout=None):
self.target = target
self.interpreter = interpreter or PythonInterpreter.get()
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
run_info = run_tracker.run_info
build_properties = {}
build_properties.update(run_info.add_basic_info(run_id=None, timestamp=time.time()))
build_properties.update(run_info.add_scm_info())
pexinfo = target.pexinfo.copy()
pexinfo.build_properties = build_properties
builder = PEXBuilder(distpath, pex_info=pexinfo, interpreter=self.interpreter)
self.chroot = PythonChroot(
target,
root_dir,
builder=builder,
interpreter=self.interpreter,
conn_timeout=conn_timeout)
def select_interpreter(self, compatibilities, allow_multiple=False):
if allow_multiple:
return compatibilities
me = PythonInterpreter.get()
if me in compatibilities:
return [me]
return [min(compatibilities)] if compatibilities else []
def __init__(self, targets, args, root_dir, interpreter=None, conn_timeout=None):
self.targets = targets
self.args = args
self.root_dir = root_dir
self.interpreter = interpreter or PythonInterpreter.get()
self.successes = {}
self._conn_timeout = conn_timeout
def __init__(self,
target,
root_dir,
extra_targets=None,
builder=None,
interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._resolver = MultiResolver(self._config,
target,
conn_timeout=conn_timeout)
self._builder = builder or PEXBuilder(tempfile.mkdtemp(),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
self._config.get('python-setup', 'artifact_cache'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def build(self, targets, args, interpreter=None, conn_timeout=None):
test_targets = []
binary_targets = []
interpreter = interpreter or PythonInterpreter.get()
for target in targets:
assert target.is_python, "PythonBuilder can only build PythonTargets, given %s" % str(target)
# PythonBuilder supports PythonTests and PythonBinaries
for target in targets:
if isinstance(target, PythonTests) or isinstance(target, PythonTestSuite):
test_targets.append(target)
elif isinstance(target, PythonBinary):
binary_targets.append(target)
rv = PythonTestBuilder(
test_targets,
args,
self._root_dir,
interpreter=interpreter,
conn_timeout=conn_timeout).run()
if rv != 0:
return rv
for binary_target in binary_targets:
rv = PythonBinaryBuilder(
binary_target,
self._root_dir,
self._run_tracker,
interpreter=interpreter,
conn_timeout=conn_timeout).run()
if rv != 0:
return rv
return 0
def __init__(self,
target,
root_dir,
extra_targets=None,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._extra_requirements = list(
extra_requirements) if extra_requirements is not None else []
self._builder = builder or PEXBuilder(tempfile.mkdtemp(),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
PythonSetup(self._config).scratch_dir('artifact_cache',
default_name='artifacts'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def from_file(filename, interpreter=PythonInterpreter.get()):
if filename.lower().endswith(".egg"):
return PythonDependency.from_eggs(filename, interpreter=interpreter)
else:
for suffix in VALID_SOURCE_EXTENSIONS:
if filename.lower().endswith(suffix):
return PythonDependency.from_source(filename, interpreter=interpreter)
raise PythonDependency.RequirementError("Unrecognized Python dependency file format: %s!" % filename)
def test_iter_ordering():
pi = PythonInterpreter.get()
tgz = SourcePackage('psutil-0.6.1.tar.gz')
egg = EggPackage('psutil-0.6.1-py%s-%s.egg' % (pi.python, get_build_platform()))
req = Requirement.parse('psutil')
assert list(FakeObtainer([tgz, egg]).iter(req)) == [egg, tgz]
assert list(FakeObtainer([egg, tgz]).iter(req)) == [egg, tgz]
def interpreter_from_path(self, path):
interpreter_dir = os.path.basename(path)
identity = PythonIdentity.from_path(interpreter_dir)
try:
executable = os.readlink(os.path.join(path, 'python'))
except OSError:
return None
interpreter = PythonInterpreter(executable, identity)
return resolve(self._config, interpreter, logger=self._logger)
def install_requirement(req,
path=None,
extra_site_dirs=[],
index='http://pypi.python.org/simple',
repositories=['http://pypi.python.org/simple'],
interpreter=PythonInterpreter.get()):
"""
Install the requirement "req" to path "path" with extra_site_dirs put
onto the PYTHONPATH. Returns the set of newly added Distributions
(of type pkg_resource.Distribution.)
"req" can either be a pkg_resources.Requirement object (e.g. created by
pkg_resources.Requirement.parse("MySQL-python==1.2.2")) or an installable
package (e.g. a tar.gz source distribution, a source or binary .egg)
"path" is the into which we install the requirements. if path is None,
we'll create one for you.
"""
# TODO(wickman) Consider importing the easy_install Command class directly and
# manipulating it with initialize/finalize options + run.
if not isinstance(req, pkg_resources.Requirement):
if not os.path.exists(req):
try:
req = pkg_resources.Requirement.parse(req)
except:
raise TypeError(
"req should either be an installable file, a pkg_resources.Requirement "
"or a valid requirement string. got %s" % req)
if path is None:
path = tempfile.mkdtemp()
if not os.path.exists(path):
safe_mkdir(path)
easy_install_args = [
'--install-dir=%s' % path,
'--site-dirs=%s' % ','.join([path] + extra_site_dirs),
'--always-copy',
'--multi-version',
'--exclude-scripts',
'-i', index]
for repo in reversed(repositories):
easy_install_args.extend(['-f', repo])
easy_install_args.append(str(req))
distributions_backup = set(pkg_resources.find_distributions(path))
rc = ReqBuilder.run_easy_install([path] + extra_site_dirs + sys.path,
easy_install_args, interpreter)
distributions = set(pkg_resources.find_distributions(path))
new_distributions = distributions - distributions_backup
return new_distributions if rc else set()
def execute(self, _):
setup_paths = self.context.options.python_setup_paths or os.getenv('PATH').split(':')
self.context.log.debug('Finding interpreters in %s' % setup_paths)
interpreters = PythonInterpreter.all(setup_paths)
self.context.log.debug('Found %d interpreters' % len(interpreters))
setup_virtualenv_py(self.context)
for interpreter in interpreters:
self.context.log.debug('Preparing %s' % interpreter)
install_virtualenv(self.context, interpreter)
def __init__(self, targets, args, interpreter=None, conn_timeout=None, fast=False):
self.targets = targets
self.args = args
self.interpreter = interpreter or PythonInterpreter.get()
self._conn_timeout = conn_timeout
# If fast is true, we run all the tests in a single chroot. This is MUCH faster than
# creating a chroot for each test target. However running each test separately is more
# correct, as the isolation verifies that its dependencies are correctly declared.
self._fast = fast
def __init__(self, targets, args, interpreter=None, conn_timeout=None, fast=False):
self.targets = targets
self.args = args
self.interpreter = interpreter or PythonInterpreter.get()
self._conn_timeout = conn_timeout
# If fast is true, we run all the tests in a single chroot. This is MUCH faster than
# creating a chroot for each test target. However running each test separately is more
# correct, as the isolation verifies that its dependencies are correctly declared.
self._fast = fast
def test_iter_ordering():
pi = PythonInterpreter.get()
tgz = SourcePackage('psutil-0.6.1.tar.gz')
egg = EggPackage('psutil-0.6.1-py%s-%s.egg' % (pi.python, get_build_platform()))
whl = WheelPackage('psutil-0.6.1-cp%s-none-%s.whl' % (
pi.python.replace('.', ''),
get_build_platform().replace('-', '_').replace('.', '_').lower()))
req = Requirement.parse('psutil')
assert list(FakeObtainer([tgz, egg, whl]).iter(req)) == [whl, egg, tgz]
assert list(FakeObtainer([egg, tgz, whl]).iter(req)) == [whl, egg, tgz]
def _setup_paths(self, paths, filters):
for interpreter in self._matching(PythonInterpreter.all(paths), filters):
identity_str = str(interpreter.identity)
path = os.path.join(self._path, identity_str)
pi = self._interpreter_from_path(path, filters)
if pi is None:
self._setup_interpreter(interpreter)
pi = self._interpreter_from_path(path, filters)
if pi is None:
continue
self._interpreters.add(pi)
def setup_paths(self, paths):
for interpreter in PythonInterpreter.all(paths):
identity_str = str(interpreter.identity)
path = os.path.join(self._path, identity_str)
pi = self.interpreter_from_path(path)
if pi is None:
self.setup_interpreter(interpreter)
pi = self.interpreter_from_path(path)
if pi is None:
continue
self._interpreters.add(pi)
def setup_paths(self, paths):
for interpreter in PythonInterpreter.all(paths):
identity_str = str(interpreter.identity)
path = os.path.join(self._path, identity_str)
pi = self.interpreter_from_path(path)
if pi is None:
self.setup_interpreter(interpreter)
pi = self.interpreter_from_path(path)
if pi is None:
continue
self._interpreters.add(pi)
def resolve_multi(config,
requirements,
interpreter=None,
platforms=None,
conn_timeout=None,
ttl=3600):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param config: Pants :class:`Config` object.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param interpreter: :class:`PythonInterpreter` for which requirements should be resolved.
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param conn_timeout: Optional connection timeout for any remote fetching.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
"""
distributions = dict()
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError(
'Expected interpreter to be a PythonInterpreter, got %s' %
type(interpreter))
install_cache = PythonSetup(config).scratch_dir('install_cache',
default_name='eggs')
platforms = get_platforms(
platforms or config.getlist('python-setup', 'platforms', ['current']))
for platform in platforms:
translator = Translator.default(install_cache=install_cache,
interpreter=interpreter,
platform=platform,
conn_timeout=conn_timeout)
obtainer = PantsObtainer(
install_cache=install_cache,
crawler=crawler_from_config(config, conn_timeout=conn_timeout),
fetchers=fetchers_from_config(config) or [PyPIFetcher()],
translators=translator)
distributions[platform] = resolve(requirements=requirements,
obtainer=obtainer,
interpreter=interpreter,
platform=platform)
return distributions
def interpreter_from_path(cls, path):
interpreter_dir = os.path.basename(path)
identity = PythonIdentity.from_path(interpreter_dir)
try:
executable = os.readlink(os.path.join(path, 'python'))
except OSError:
return None
try:
distribute_path = os.readlink(os.path.join(path, 'distribute'))
except OSError:
distribute_path = None
return PythonInterpreter(executable, identity, distribute_path)
def execute(self, _):
setup_paths = self.context.options.python_setup_paths or os.getenv('PATH').split(':')
self.context.log.debug('Finding interpreters in %s' % setup_paths)
interpreters = PythonInterpreter.all(setup_paths)
self.context.log.debug('Found %d interpreters' % len(interpreters))
setup_virtualenv_py(self.context)
for interpreter in interpreters:
self.context.log.debug('Preparing %s' % interpreter)
try:
install_virtualenv(self.context, interpreter)
except TaskError as e:
print('Failed to install %s, continuing anyway.' % interpreter)
def test_iter_ordering():
pi = PythonInterpreter.get()
tgz = SourcePackage('psutil-0.6.1.tar.gz')
egg = EggPackage('psutil-0.6.1-py%s-%s.egg' %
(pi.python, get_build_platform()))
whl = WheelPackage(
'psutil-0.6.1-cp%s-none-%s.whl' %
(pi.python.replace('.', ''), get_build_platform().replace(
'-', '_').replace('.', '_').lower()))
req = Requirement.parse('psutil')
assert list(FakeObtainer([tgz, egg, whl]).iter(req)) == [whl, egg, tgz]
assert list(FakeObtainer([egg, tgz, whl]).iter(req)) == [whl, egg, tgz]
def setup_paths(self, paths):
for interpreter in PythonInterpreter.all(paths):
identity_str = str(interpreter.identity)
path = os.path.join(self._path, identity_str)
pi = self.interpreter_from_path(path)
if pi is None or pi.distribute is None:
self._logger('Found interpreter %s: %s (%s)' %
(interpreter.binary, str(interpreter.identity),
'uncached' if pi is None else 'incomplete'))
self.setup_interpreter(interpreter)
pi = self.interpreter_from_path(path)
if pi is None or pi.distribute is None:
continue
self._interpreters.add(pi)
def resolve_multi(config,
requirements,
interpreter=None,
platforms=None,
conn_timeout=None,
ttl=3600):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param config: Pants :class:`Config` object.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param interpreter: :class:`PythonInterpreter` for which requirements should be resolved.
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param conn_timeout: Optional connection timeout for any remote fetching.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
"""
distributions = dict()
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError('Expected interpreter to be a PythonInterpreter, got %s' % type(interpreter))
install_cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs')
platforms = get_platforms(platforms or config.getlist('python-setup', 'platforms', ['current']))
for platform in platforms:
translator = Translator.default(
install_cache=install_cache,
interpreter=interpreter,
platform=platform,
conn_timeout=conn_timeout)
obtainer = PantsObtainer(
install_cache=install_cache,
crawler=crawler_from_config(config, conn_timeout=conn_timeout),
fetchers=fetchers_from_config(config) or [PyPIFetcher()],
translators=translator)
distributions[platform] = resolve(requirements=requirements,
obtainer=obtainer,
interpreter=interpreter,
platform=platform)
return distributions
def setup_paths(self, paths):
for interpreter in PythonInterpreter.all(paths):
identity_str = str(interpreter.identity)
path = os.path.join(self._path, identity_str)
pi = self.interpreter_from_path(path)
if pi is None or pi.distribute is None:
self._logger('Found interpreter %s: %s (%s)' % (
interpreter.binary,
str(interpreter.identity),
'uncached' if pi is None else 'incomplete'))
self.setup_interpreter(interpreter)
pi = self.interpreter_from_path(path)
if pi is None or pi.distribute is None:
continue
self._interpreters.add(pi)
def build(self, targets, args, interpreter=None, conn_timeout=None):
test_targets = []
binary_targets = []
interpreter = interpreter or PythonInterpreter.get()
for target in targets:
assert target.is_python, "PythonBuilder can only build PythonTargets, given %s" % str(
target)
if 'pylint' in args:
real_args = list(args)
real_args.remove('pylint')
for target in targets:
try:
PythonLintBuilder([target],
real_args,
self._root_dir,
conn_timeout=conn_timeout).run()
except Exception as e:
print('Failed to run lint for %s: %s' % (target, e))
return 0
# PythonBuilder supports PythonTests and PythonBinaries
for target in targets:
if isinstance(target, PythonTests) or isinstance(
target, PythonTestSuite):
test_targets.append(target)
elif isinstance(target, PythonBinary):
binary_targets.append(target)
rv = PythonTestBuilder(test_targets,
args,
self._root_dir,
interpreter=interpreter,
conn_timeout=conn_timeout).run()
if rv != 0:
return rv
for binary_target in binary_targets:
rv = PythonBinaryBuilder(binary_target,
self._root_dir,
self._run_tracker,
interpreter=interpreter,
conn_timeout=conn_timeout).run()
if rv != 0:
return rv
return 0
def __init__(self, target, root_dir, extra_targets=None, builder=None, interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(extra_targets) if extra_targets is not None else []
self._resolver = MultiResolver(self._config, target, conn_timeout=conn_timeout)
self._builder = builder or PEXBuilder(tempfile.mkdtemp(), interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(self._config.get('python-setup', 'artifact_cache'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def build(self, targets, args, interpreter=None, conn_timeout=None):
test_targets = []
binary_targets = []
interpreter = interpreter or PythonInterpreter.get()
for target in targets:
assert target.is_python, "PythonBuilder can only build PythonTargets, given %s" % str(target)
if 'pylint' in args:
real_args = list(args)
real_args.remove('pylint')
for target in targets:
try:
PythonLintBuilder([target], real_args, self._root_dir, conn_timeout=conn_timeout).run()
except Exception as e:
print('Failed to run lint for %s: %s' % (target, e))
return 0
# PythonBuilder supports PythonTests and PythonBinaries
for target in targets:
if isinstance(target, PythonTests) or isinstance(target, PythonTestSuite):
test_targets.append(target)
elif isinstance(target, PythonBinary):
binary_targets.append(target)
rv = PythonTestBuilder(
test_targets,
args,
self._root_dir,
interpreter=interpreter,
conn_timeout=conn_timeout).run()
if rv != 0:
return rv
for binary_target in binary_targets:
rv = PythonBinaryBuilder(
binary_target,
self._root_dir,
self._run_tracker,
interpreter=interpreter,
conn_timeout=conn_timeout).run()
if rv != 0:
return rv
return 0
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True)
parser.add_option('--python', default=sys.executable)
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
interpreter = PythonInterpreter(
options.python,
PythonInterpreter.from_binary(options.python).identity,
extras={})
pex_builder = PEXBuilder(
path=tmp_dir,
interpreter=interpreter,
)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources`.
copy_package(pex_builder, 'pkg_resources', prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# Add resources listed in the manifest.
for dst, src in manifest['nativeLibraries'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def resolve_multi(config,
requirements,
interpreter=None,
platforms=None,
conn_timeout=None,
ttl=3600):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param config: Pants :class:`Config` object.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param interpreter: :class:`PythonInterpreter` for which requirements should be resolved.
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param conn_timeout: Optional connection timeout for any remote fetching.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
"""
now = time.time()
distributions = {}
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError('Expected interpreter to be a PythonInterpreter, got %s' % type(interpreter))
install_cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs')
platforms = get_platforms(platforms or config.getlist('python-setup', 'platforms', ['current']))
crawler = crawler_from_config(config, conn_timeout=conn_timeout)
fetchers = fetchers_from_config(config)
for platform in platforms:
env = PantsEnvironment(search_path=[], platform=platform, python=interpreter.python)
working_set = WorkingSet(entries=[])
shared_options = dict(install_cache=install_cache, platform=platform)
egg_translator = EggTranslator(python=interpreter.python, **shared_options)
egg_obtainer = Obtainer(crawler, [Fetcher([install_cache])], egg_translator)
def installer(req):
# Attempt to obtain the egg from the local cache. If it's an exact match, we can use it.
# If it's not an exact match, then if it's been resolved sufficiently recently, we still
# use it.
dist = egg_obtainer.obtain(req)
if dist and (requirement_is_exact(req) or now - os.path.getmtime(dist.location) < ttl):
return dist
# Failed, so follow through to "remote" resolution
source_translator = SourceTranslator(
interpreter=interpreter,
use_2to3=getattr(req, 'use_2to3', False),
**shared_options)
translator = ChainedTranslator(egg_translator, source_translator)
obtainer = Obtainer(
crawler,
[Fetcher([req.repository])] if getattr(req, 'repository', None) else fetchers,
translator)
dist = obtainer.obtain(req)
if dist:
try:
touch(dist.location)
except OSError:
pass
return dist
distributions[platform] = working_set.resolve(requirements, env=env, installer=installer)
return distributions
def from_req(requirement, interpreter=PythonInterpreter.get(), **kw):
dists = ReqBuilder.install_requirement(requirement, interpreter=interpreter, **kw)
return PythonDependency.from_distributions(*list(dists))
def from_source(filename, interpreter=PythonInterpreter.get(), **kw):
if not os.path.exists(filename):
raise PythonDependency.NotFoundError("Could not find PythonDependency target %s!" % filename)
dists = ReqBuilder.install_requirement(filename, interpreter=interpreter, **kw)
return PythonDependency.from_distributions(*list(dists))
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True)
parser.add_option('--python', default=sys.executable)
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
interpreter = PythonInterpreter(
options.python,
PythonInterpreter.from_binary(options.python).identity,
extras={})
pex_builder = PEXBuilder(
path=tmp_dir,
interpreter=interpreter,
)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources`.
pex_builder.add_source(
dereference_symlinks(pkg_resources_py),
os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py'))
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def resolve_multi(config,
requirements,
interpreter=None,
platforms=None,
conn_timeout=None,
ttl=3600):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param config: Pants :class:`Config` object.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param interpreter: :class:`PythonInterpreter` for which requirements should be resolved.
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param conn_timeout: Optional connection timeout for any remote fetching.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
"""
now = time.time()
distributions = {}
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError('Expected interpreter to be a PythonInterpreter, got %s' % type(interpreter))
install_cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs')
platforms = get_platforms(platforms or config.getlist('python-setup', 'platforms', ['current']))
crawler = crawler_from_config(config, conn_timeout=conn_timeout)
fetchers = fetchers_from_config(config)
for platform in platforms:
env = PantsEnvironment(interpreter, platform=platform)
working_set = WorkingSet(entries=[])
shared_options = dict(install_cache=install_cache, platform=platform)
egg_translator = EggTranslator(interpreter=interpreter, **shared_options)
egg_obtainer = Obtainer(crawler, [Fetcher([install_cache])], egg_translator)
def installer(req):
# Attempt to obtain the egg from the local cache. If it's an exact match, we can use it.
# If it's not an exact match, then if it's been resolved sufficiently recently, we still
# use it.
dist = egg_obtainer.obtain(req)
if dist and (requirement_is_exact(req) or now - os.path.getmtime(dist.location) < ttl):
return dist
# Failed, so follow through to "remote" resolution
source_translator = SourceTranslator(
interpreter=interpreter,
use_2to3=getattr(req, 'use_2to3', False),
**shared_options)
translator = ChainedTranslator(egg_translator, source_translator)
obtainer = Obtainer(
crawler,
[Fetcher([req.repository])] if getattr(req, 'repository', None) else fetchers,
translator)
dist = obtainer.obtain(req)
if dist:
try:
touch(dist.location)
except OSError:
pass
return dist
distributions[platform] = working_set.resolve(requirements, env=env, installer=installer)
return distributions
