def yield_pex_builder(zip_safe=True):
with nested(temporary_dir(),
make_distribution('p1', zipped=True,
zip_safe=zip_safe)) as (td, p1):
pb = PEXBuilder(path=td)
pb.add_egg(p1.location)
yield pb
def _run_python_test(self, target):
po = None
rv = PythonTestResult.exception()
try:
builder = PEXBuilder()
builder.info().entry_point = 'pytest'
builder.info().ignore_errors = target._soft_dependencies
chroot = PythonChroot(target, self.root_dir, extra_targets=self.generate_test_targets(),
builder=builder)
builder = chroot.dump()
builder.freeze()
test_args = PythonTestBuilder.generate_junit_args(target)
test_args.extend(self.args)
sources = [os.path.join(target.target_base, source) for source in target.sources]
po = PEX(builder.path()).run(args=test_args + sources, blocking=False, setsid=True)
rv = PythonTestBuilder.wait_on(po, timeout=target.timeout)
except Exception as e:
import traceback
print('Failed to run test!', file=sys.stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if po and po.returncode != 0:
try:
os.killpg(po.pid, signal.SIGTERM)
except OSError as e:
if e.errno == errno.EPERM:
print("Unable to kill process group: %d" % po.pid)
elif e.errno != errno.ESRCH:
rv = PythonTestResult.exception()
self.successes[target._create_id()] = rv
return rv
def write_pex(td, exe_contents):
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def execute(self):
if self.options.pex and self.options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.options.entry_point and self.options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.options.verbose:
print('Build operating on target: %s %s' % (self.target,
'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else ''))
builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
pex_info=self.target.pexinfo if isinstance(self.target, PythonBinary) else None)
if self.options.entry_point:
builder.set_entry_point(self.options.entry_point)
if self.options.ipython:
if not self.config.has_section('python-ipython'):
self.error('No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
if builder.info.entry_point is None:
self.error('Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython', 'requirements', default=[])
for requirement in requirements:
self.extra_requirements.append(PythonRequirement(requirement))
executor = PythonChroot(
self.target,
self.root_dir,
builder=builder,
interpreter=self.interpreter,
extra_targets=self.extra_targets,
extra_requirements=self.extra_requirements,
conn_timeout=self.options.conn_timeout)
executor.dump()
if self.options.pex:
pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name)
builder.build(pex_name)
print('Wrote %s' % pex_name)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('--entry-point', default='__main__')
parser.add_argument('--python', default=sys.executable)
parser.add_argument('output')
args = parser.parse_args(argv[1:])
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
interpreter = PythonInterpreter(
args.python, PythonInterpreter.from_binary(args.python).identity,
extras={('setuptools', '1.0'):
os.path.join(BUCK_ROOT, 'third-party/py/setuptools')})
pex_builder = PEXBuilder(
path=tmp_dir,
interpreter=interpreter,
)
# Mark this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS), so
# we'll want to revisit this.
pex_builder.info.zip_safe = True
# Set the starting point for this PEX.
pex_builder.info.entry_point = args.entry_point
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
pex_builder.add_source(dereference_symlinks(src), dst)
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Generate the PEX file.
pex_builder.build(args.output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def _run_python_test(self, target):
po = None
rv = PythonTestResult.exception()
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = target.entry_point
builder.info.ignore_errors = target._soft_dependencies
chroot = PythonChroot(
target,
self.root_dir,
extra_requirements=self.generate_test_targets(),
builder=builder,
platforms=('current',),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = PythonTestBuilder.generate_junit_args(target)
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(target, builder.chroot())
test_args.extend(args)
sources = target.sources_relative_to_buildroot()
po = PEX(builder.path(), interpreter=self.interpreter).run(
args=test_args + sources, blocking=False, setsid=True)
# TODO(wickman) If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestBuilder.wait_on(po, timeout=target.timeout)
except Exception as e:
import traceback
print('Failed to run test!', file=sys.stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
if po and po.returncode != 0:
try:
os.killpg(po.pid, signal.SIGTERM)
except OSError as e:
if e.errno == errno.EPERM:
print("Unable to kill process group: %d" % po.pid)
elif e.errno != errno.ESRCH:
rv = PythonTestResult.exception()
self.successes[target.id] = rv
return rv
def __init__(self,
target,
run_tracker,
interpreter=None,
conn_timeout=None):
self.target = target
self.interpreter = interpreter or PythonInterpreter.get()
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
run_info = run_tracker.run_info
build_properties = {}
build_properties.update(
run_info.add_basic_info(run_id=None, timestamp=time.time()))
build_properties.update(run_info.add_scm_info())
pexinfo = target.pexinfo.copy()
pexinfo.build_properties = build_properties
builder = PEXBuilder(distpath,
pex_info=pexinfo,
interpreter=self.interpreter)
self.chroot = PythonChroot(targets=[target],
builder=builder,
platforms=target.platforms,
interpreter=self.interpreter,
conn_timeout=conn_timeout)
def __init__(self,
target,
root_dir,
extra_targets=None,
builder=None,
interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._resolver = MultiResolver(self._config,
target,
conn_timeout=conn_timeout)
self._builder = builder or PEXBuilder(tempfile.mkdtemp(),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
self._config.get('python-setup', 'artifact_cache'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def __init__(self,
target,
root_dir,
extra_targets=None,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._extra_requirements = list(
extra_requirements) if extra_requirements is not None else []
self._builder = builder or PEXBuilder(tempfile.mkdtemp(),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
PythonSetup(self._config).scratch_dir('artifact_cache',
default_name='artifacts'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def execute(self):
if self.options.pex and self.options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.options.entry_point and self.options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.options.verbose:
print('Build operating on target: %s %s' % (self.target,
'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else ''))
builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
pex_info=self.target.pexinfo if isinstance(self.target, PythonBinary) else None)
if self.options.entry_point:
builder.set_entry_point(self.options.entry_point)
if self.options.ipython:
if not self.config.has_section('python-ipython'):
self.error('No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
if builder.info.entry_point is None:
self.error('Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython', 'requirements', default=[])
with ParseContext.temp():
for requirement in requirements:
self.extra_targets.append(PythonRequirement(requirement))
executor = PythonChroot(
self.target,
self.root_dir,
builder=builder,
interpreter=self.interpreter,
extra_targets=self.extra_targets,
conn_timeout=self.options.conn_timeout)
executor.dump()
if self.options.pex:
pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name)
builder.build(pex_name)
print('Wrote %s' % pex_name)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def build_pex(args, options):
interpreter = None
if options.python:
if os.path.exists(options.python):
interpreter = PythonInterpreter.from_binary(options.python)
else:
interpreter = PythonInterpreter.from_env(options.python)
if interpreter is None:
die('Failed to find interpreter: %s' % options.python)
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
fetchers = [Fetcher(options.repos)]
if options.pypi:
fetchers.append(PyPIFetcher())
resolveds = requirement_resolver(options.requirements,
cache=options.cache_dir,
fetchers=fetchers,
interpreter=interpreter,
platform=options.platform)
if resolveds:
log('Resolved distributions:', v=options.verbosity)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
for source_dir in options.source_dirs:
try:
egg_path = EggInstaller(source_dir).bdist()
except EggInstaller.Error:
die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL)
pex_builder.add_egg(egg_path)
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point,
v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
return pex_builder
class PythonBinaryBuilder(object):
class NotABinaryTargetException(Exception):
pass
def __init__(self, target, args, root_dir):
self.target = target
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
self.builder = PEXBuilder(distpath)
# configure builder PexInfo options
for repo in target._repositories:
self.builder.info().add_repository(repo)
for index in target._indices:
self.builder.info().add_index(index)
self.builder.info().allow_pypi = target._allow_pypi
self.builder.info().zip_safe = target._zip_safe
self.builder.info().inherit_path = target._inherit_path
self.builder.info().entry_point = target._entry_point
self.builder.info().ignore_errors = target._ignore_errors
self.chroot = PythonChroot(target, root_dir, builder=self.builder)
def run(self):
print('Building PythonBinary %s:' % self.target)
env = self.chroot.dump()
filename = os.path.join(self.distdir, '%s.pex' % self.target.name)
env.build(filename)
print('Wrote %s' % filename)
return 0
class PythonBinaryBuilder(object):
class NotABinaryTargetException(Exception): pass
def __init__(self, target, args, root_dir, conn_timeout=None):
self.target = target
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
self.builder = PEXBuilder(distpath)
# configure builder PexInfo options
for repo in target._repositories:
self.builder.info().add_repository(repo)
for index in target._indices:
self.builder.info().add_index(index)
self.builder.info().allow_pypi = target._allow_pypi
self.builder.info().zip_safe = target._zip_safe
self.builder.info().inherit_path = target._inherit_path
self.builder.info().entry_point = target._entry_point
self.builder.info().ignore_errors = target._ignore_errors
self.chroot = PythonChroot(target, root_dir, builder=self.builder, conn_timeout=conn_timeout)
def run(self):
print('Building PythonBinary %s:' % self.target)
env = self.chroot.dump()
filename = os.path.join(self.distdir, '%s.pex' % self.target.name)
env.build(filename)
print('Wrote %s' % filename)
return 0
def write_pex(td, exe_contents, dists=None):
dists = dists or []
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td)
for dist in dists:
pb.add_egg(dist.location)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def create_binary(self, binary):
if binary.compatibility:
interpreter = self.select_interpreter(binary.compatibility)
else:
interpreter = self.interpreter
distpath = tempfile.mktemp(dir=self._distdir, prefix=binary.name)
run_info = self.context.run_tracker.run_info
build_properties = {}
build_properties.update(
run_info.add_basic_info(run_id=None, timestamp=time.time()))
build_properties.update(run_info.add_scm_info())
pexinfo = binary.pexinfo.copy()
pexinfo.build_properties = build_properties
builder = PEXBuilder(distpath,
pex_info=pexinfo,
interpreter=interpreter)
chroot = PythonChroot(targets=[binary],
builder=builder,
platforms=binary.platforms,
interpreter=interpreter,
conn_timeout=self.conn_timeout)
pex_path = os.path.join(self._distdir, '%s.pex' % binary.name)
builder = chroot.dump()
try:
builder.build(pex_path)
finally:
builder.chroot().delete()
def __init__(self, target, args, root_dir):
self.target = target
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
self.builder = PEXBuilder(distpath)
# configure builder PexInfo options
for repo in target._repositories:
self.builder.info().add_repository(repo)
for index in target._indices:
self.builder.info().add_index(index)
self.builder.info().allow_pypi = target._allow_pypi
self.builder.info().zip_safe = target._zip_safe
self.builder.info().inherit_path = target._inherit_path
self.builder.info().entry_point = target._entry_point
self.builder.info().ignore_errors = target._ignore_errors
self.chroot = PythonChroot(target, root_dir, builder=self.builder)
def _run_python_tests(self, targets, stdout, stderr):
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = []
test_args.extend(PythonTestBuilder.generate_junit_args(targets))
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(targets)
test_args.extend(args)
sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets]))
pex = PEX(builder.path(), interpreter=self.interpreter)
rc = pex.run(args=test_args + sources, blocking=True, setsid=True,
stdout=stdout, stderr=stderr)
# TODO(wickman): If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestResult.rc(rc)
except Exception:
import traceback
print('Failed to run test!', file=stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
return rv
def _run_python_tests(self, targets, stdout, stderr):
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = 'pytest'
chroot = PythonChroot(
targets=targets,
extra_requirements=self._TESTING_TARGETS,
builder=builder,
platforms=('current',),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = []
test_args.extend(PythonTestBuilder.generate_junit_args(targets))
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(targets)
test_args.extend(args)
sources = list(itertools.chain(*[t.sources_relative_to_buildroot() for t in targets]))
pex = PEX(builder.path(), interpreter=self.interpreter)
rc = pex.run(args=test_args + sources, blocking=True, setsid=True,
stdout=stdout, stderr=stderr)
# TODO(wickman): If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestResult.rc(rc)
except Exception:
import traceback
print('Failed to run test!', file=stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
return rv
def write_pex(td, exe_contents):
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def write_pex(td, exe_contents, dists=None):
dists = dists or []
with open(os.path.join(td, "exe.py"), "w") as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td)
for dist in dists:
pb.add_egg(dist.location)
pb.set_executable(os.path.join(td, "exe.py"))
pb.freeze()
return pb
def build_pex(args, options):
interpreter = interpreter_from_options(options)
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
installer = WheelInstaller if options.use_wheel else EggInstaller
resolveds = requirement_resolver(
options.requirements,
obtainer=build_obtainer(options),
interpreter=interpreter,
platform=options.platform)
if resolveds:
log('Resolved distributions:', v=options.verbosity)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
for source_dir in options.source_dirs:
try:
bdist = installer(source_dir).bdist()
except installer.Error:
die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL)
pex_builder.add_dist_location(bdist)
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point, v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
return pex_builder
def _run_python_test(self, target):
po = None
rv = PythonTestResult.exception()
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = target.entry_point
builder.info.ignore_errors = target._soft_dependencies
chroot = PythonChroot(target,
self.root_dir,
extra_targets=self.generate_test_targets(),
builder=builder,
platforms=('current', ),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = PythonTestBuilder.generate_junit_args(target)
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(target, builder.chroot())
test_args.extend(args)
sources = [
os.path.join(target.target_base, source)
for source in target.sources
]
po = PEX(builder.path(), interpreter=self.interpreter).run(
args=test_args + sources, blocking=False, setsid=True)
# TODO(wickman) If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestBuilder.wait_on(po, timeout=target.timeout)
except Exception as e:
import traceback
print('Failed to run test!', file=sys.stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
if po and po.returncode != 0:
try:
os.killpg(po.pid, signal.SIGTERM)
except OSError as e:
if e.errno == errno.EPERM:
print("Unable to kill process group: %d" % po.pid)
elif e.errno != errno.ESRCH:
rv = PythonTestResult.exception()
self.successes[target._create_id()] = rv
return rv
def _run_python_test(self, target):
po = None
rv = PythonTestResult.exception()
coverage_rc = None
coverage_enabled = "PANTS_PY_COVERAGE" in os.environ
try:
builder = PEXBuilder()
builder.info().entry_point = "pytest"
builder.info().ignore_errors = target._soft_dependencies
chroot = PythonChroot(
target,
self.root_dir,
extra_targets=self.generate_test_targets(),
builder=builder,
conn_timeout=self._conn_timeout,
)
builder = chroot.dump()
builder.freeze()
test_args = PythonTestBuilder.generate_junit_args(target)
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(target, builder.chroot())
test_args.extend(args)
sources = [os.path.join(target.target_base, source) for source in target.sources]
po = PEX(builder.path()).run(args=test_args + sources, blocking=False, setsid=True)
# TODO(wickman) If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestBuilder.wait_on(po, timeout=target.timeout)
except KeyboardInterrupt:
print("Test interrupted by user")
rv = PythonTestResult.cancelled()
except Exception as e:
import traceback
print("Failed to run test!", file=sys.stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
if po and po.returncode != 0:
try:
os.killpg(po.pid, signal.SIGTERM)
except OSError as e:
if e.errno == errno.EPERM:
print("Unable to kill process group: %d" % po.pid)
elif e.errno != errno.ESRCH:
rv = PythonTestResult.exception()
self.successes[target._create_id()] = rv
return rv
def __init__(self, target, root_dir, extra_targets=None, builder=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._cache = BuildCache(
os.path.join(self._config.get('python-setup', 'artifact_cache'),
'%s' % PythonIdentity.get()))
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._resolver = PythonResolver([self._target] + self._extra_targets)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
self._platforms = (Platform.current(), )
self._pythons = (sys.version[:3], )
# TODO(wickman) Should this be in the binary builder?
if isinstance(self._target, PythonBinary):
self._platforms = self._target._platforms
self._pythons = self._target._interpreters
def __init__(self, target, args, root_dir, conn_timeout=None):
self.target = target
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
self.builder = PEXBuilder(distpath)
# configure builder PexInfo options
for repo in target._repositories:
self.builder.info().add_repository(repo)
for index in target._indices:
self.builder.info().add_index(index)
self.builder.info().allow_pypi = target._allow_pypi
self.builder.info().zip_safe = target._zip_safe
self.builder.info().inherit_path = target._inherit_path
self.builder.info().entry_point = target._entry_point
self.builder.info().ignore_errors = target._ignore_errors
self.chroot = PythonChroot(target, root_dir, builder=self.builder, conn_timeout=conn_timeout)
def __init__(self,
target,
root_dir,
extra_targets=None,
builder=None,
conn_timeout=None):
self._config = Config.load()
self._target = target
self._root = root_dir
self._key_generator = CacheKeyGenerator()
self._extra_targets = list(
extra_targets) if extra_targets is not None else []
self._resolver = MultiResolver.from_target(self._config,
target,
conn_timeout=conn_timeout)
self._builder = builder or PEXBuilder(tempfile.mkdtemp())
artifact_cache_root = os.path.join(
self._config.get('python-setup', 'artifact_cache'),
'%s' % PythonIdentity.get())
self._artifact_cache = FileBasedArtifactCache(
None, self._root, artifact_cache_root,
self._builder.add_dependency_file)
def _run_python_test(self, target):
po = None
rv = PythonTestResult.exception()
try:
builder = PEXBuilder()
builder.info().entry_point = 'pytest'
builder.info().ignore_errors = target._soft_dependencies
chroot = PythonChroot(target,
self.root_dir,
extra_targets=self.generate_test_targets(),
builder=builder)
builder = chroot.dump()
builder.freeze()
test_args = PythonTestBuilder.generate_junit_args(target)
test_args.extend(self.args)
sources = [
os.path.join(target.target_base, source)
for source in target.sources
]
po = PEX(builder.path()).run(args=test_args + sources,
blocking=False,
setsid=True)
rv = PythonTestBuilder.wait_on(po, timeout=target.timeout)
except Exception as e:
import traceback
print('Failed to run test!', file=sys.stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if po and po.returncode != 0:
try:
os.killpg(po.pid, signal.SIGTERM)
except OSError as e:
if e.errno == errno.EPERM:
print("Unable to kill process group: %d" % po.pid)
elif e.errno != errno.ESRCH:
rv = PythonTestResult.exception()
self.successes[target._create_id()] = rv
return rv
def build_pex(args, options):
pex_builder = PEXBuilder(path=safe_mkdtemp())
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
fetchers = [Fetcher(options.repos)]
if options.pypi:
fetchers.append(PyPIFetcher())
resolver = Resolver(cache=options.cache_dir, fetchers=fetchers, install_cache=options.cache_dir)
resolveds = resolver.resolve(options.requirements)
if resolveds:
log('Resolved distributions:', v=options.verbosity)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
for source_dir in options.source_dirs:
dist = Installer(source_dir).distribution()
if not dist:
die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL)
egg_path = Distiller(dist).distill()
if not egg_path:
die('Failed to distill %s into egg' % dist, CANNOT_DISTILL)
pex_builder.add_egg(egg_path)
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point, v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
else:
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), args), v=options.verbosity)
pex = PEX(pex_builder.path())
return pex.run(args=list(args))
return 0
def yield_pex_builder(zip_safe=True):
with nested(temporary_dir(), make_distribution('p1', zipped=True, zip_safe=zip_safe)) as (td, p1):
pb = PEXBuilder(path=td)
pb.add_egg(p1.location)
yield pb
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True)
parser.add_option('--python', default=sys.executable)
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
interpreter = PythonInterpreter(
options.python,
PythonInterpreter.from_binary(options.python).identity,
extras={})
pex_builder = PEXBuilder(
path=tmp_dir,
interpreter=interpreter,
)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources`.
copy_package(pex_builder, 'pkg_resources', prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# Add resources listed in the manifest.
for dst, src in manifest['nativeLibraries'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True)
parser.add_option('--python', default=sys.executable)
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
interpreter = PythonInterpreter(
options.python,
PythonInterpreter.from_binary(options.python).identity,
extras={})
pex_builder = PEXBuilder(
path=tmp_dir,
interpreter=interpreter,
)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources`.
pex_builder.add_source(
dereference_symlinks(pkg_resources_py),
os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py'))
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def temporary_pex_builder(self, interpreter=None, pex_info=None, parent_dir=None): """Yields a PEXBuilder and cleans up its chroot when it goes out of context.""" path = tempfile.mkdtemp(dir=parent_dir) builder = PEXBuilder(path=path, interpreter=interpreter, pex_info=pex_info) yield builder builder.chroot().delete()
