def _run_python_test(self, target):
po = None
rv = PythonTestResult.exception()
try:
builder = PEXBuilder()
builder.info().entry_point = 'pytest'
builder.info().ignore_errors = target._soft_dependencies
chroot = PythonChroot(target, self.root_dir, extra_targets=self.generate_test_targets(),
builder=builder)
builder = chroot.dump()
builder.freeze()
test_args = PythonTestBuilder.generate_junit_args(target)
test_args.extend(self.args)
sources = [os.path.join(target.target_base, source) for source in target.sources]
po = PEX(builder.path()).run(args=test_args + sources, blocking=False, setsid=True)
rv = PythonTestBuilder.wait_on(po, timeout=target.timeout)
except Exception as e:
import traceback
print('Failed to run test!', file=sys.stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if po and po.returncode != 0:
try:
os.killpg(po.pid, signal.SIGTERM)
except OSError as e:
if e.errno == errno.EPERM:
print("Unable to kill process group: %d" % po.pid)
elif e.errno != errno.ESRCH:
rv = PythonTestResult.exception()
self.successes[target._create_id()] = rv
return rv
def __init__(self, target, args, root_dir):
self.target = target
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
self.chroot = PythonChroot(target, root_dir)
self.distdir = os.path.join(root_dir, 'dist')
def execute(self):
if self.options.pex and self.options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.options.entry_point and self.options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.options.verbose:
print('Build operating on target: %s %s' % (self.target,
'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else ''))
builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
pex_info=self.target.pexinfo if isinstance(self.target, PythonBinary) else None)
if self.options.entry_point:
builder.set_entry_point(self.options.entry_point)
if self.options.ipython:
if not self.config.has_section('python-ipython'):
self.error('No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
if builder.info.entry_point is None:
self.error('Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython', 'requirements', default=[])
with ParseContext.temp():
for requirement in requirements:
self.extra_targets.append(PythonRequirement(requirement))
executor = PythonChroot(
self.target,
self.root_dir,
builder=builder,
interpreter=self.interpreter,
extra_targets=self.extra_targets,
conn_timeout=self.options.conn_timeout)
executor.dump()
if self.options.pex:
pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name)
builder.build(pex_name)
print('Wrote %s' % pex_name)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def execute(self):
if self.options.pex and self.options.ipython:
self.error('Cannot specify both --pex and --ipython!')
if self.options.entry_point and self.options.ipython:
self.error('Cannot specify both --entry_point and --ipython!')
if self.options.verbose:
print('Build operating on target: %s %s' % (self.target,
'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else ''))
builder = PEXBuilder(tempfile.mkdtemp(), interpreter=self.interpreter,
pex_info=self.target.pexinfo if isinstance(self.target, PythonBinary) else None)
if self.options.entry_point:
builder.set_entry_point(self.options.entry_point)
if self.options.ipython:
if not self.config.has_section('python-ipython'):
self.error('No python-ipython sections defined in your pants.ini!')
builder.info.entry_point = self.config.get('python-ipython', 'entry_point')
if builder.info.entry_point is None:
self.error('Must specify entry_point for IPython in the python-ipython section '
'of your pants.ini!')
requirements = self.config.getlist('python-ipython', 'requirements', default=[])
with ParseContext.temp():
for requirement in requirements:
self.extra_targets.append(PythonRequirement(requirement))
executor = PythonChroot(
self.target,
self.root_dir,
builder=builder,
interpreter=self.interpreter,
extra_targets=self.extra_targets,
conn_timeout=self.options.conn_timeout)
executor.dump()
if self.options.pex:
pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name)
builder.build(pex_name)
print('Wrote %s' % pex_name)
return 0
else:
builder.freeze()
pex = PEX(builder.path(), interpreter=self.interpreter)
po = pex.run(args=list(self.args), blocking=False)
try:
return po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
def _run_python_test(self, target):
chroot = PythonChroot(target, self.root_dir)
chroot.dump()
launcher = Launcher(chroot.path())
extra_deps = PythonTestBuilder.get_pytest_eggs(self.root_dir)
test_args = ['-m', 'pytest']
test_args.extend(PythonTestBuilder.generate_junit_args(target))
test_args.extend(self.args) # Pass any extra args in to pytest.
sources = [os.path.join(target.target_base, source) for source in target.sources]
return launcher.run(interpreter_args=test_args,
args=list(sources),
extra_deps=extra_deps)
def _run_python_test(self, target):
po = None
rv = PythonTestResult.exception()
coverage_rc = None
coverage_enabled = "PANTS_PY_COVERAGE" in os.environ
try:
builder = PEXBuilder()
builder.info().entry_point = "pytest"
builder.info().ignore_errors = target._soft_dependencies
chroot = PythonChroot(
target,
self.root_dir,
extra_targets=self.generate_test_targets(),
builder=builder,
conn_timeout=self._conn_timeout,
)
builder = chroot.dump()
builder.freeze()
test_args = PythonTestBuilder.generate_junit_args(target)
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(target, builder.chroot())
test_args.extend(args)
sources = [os.path.join(target.target_base, source) for source in target.sources]
po = PEX(builder.path()).run(args=test_args + sources, blocking=False, setsid=True)
# TODO(wickman) If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestBuilder.wait_on(po, timeout=target.timeout)
except KeyboardInterrupt:
print("Test interrupted by user")
rv = PythonTestResult.cancelled()
except Exception as e:
import traceback
print("Failed to run test!", file=sys.stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
if po and po.returncode != 0:
try:
os.killpg(po.pid, signal.SIGTERM)
except OSError as e:
if e.errno == errno.EPERM:
print("Unable to kill process group: %d" % po.pid)
elif e.errno != errno.ESRCH:
rv = PythonTestResult.exception()
self.successes[target._create_id()] = rv
return rv
def _run_python_test(self, target):
po = None
rv = PythonTestResult.exception()
coverage_rc = None
coverage_enabled = 'PANTS_PY_COVERAGE' in os.environ
try:
builder = PEXBuilder(interpreter=self.interpreter)
builder.info.entry_point = target.entry_point
builder.info.ignore_errors = target._soft_dependencies
chroot = PythonChroot(target,
self.root_dir,
extra_targets=self.generate_test_targets(),
builder=builder,
platforms=('current', ),
interpreter=self.interpreter,
conn_timeout=self._conn_timeout)
builder = chroot.dump()
builder.freeze()
test_args = PythonTestBuilder.generate_junit_args(target)
test_args.extend(self.args)
if coverage_enabled:
coverage_rc, args = self.cov_setup(target, builder.chroot())
test_args.extend(args)
sources = [
os.path.join(target.target_base, source)
for source in target.sources
]
po = PEX(builder.path(), interpreter=self.interpreter).run(
args=test_args + sources, blocking=False, setsid=True)
# TODO(wickman) If coverage is enabled, write an intermediate .html that points to
# each of the coverage reports generated and webbrowser.open to that page.
rv = PythonTestBuilder.wait_on(po, timeout=target.timeout)
except Exception as e:
import traceback
print('Failed to run test!', file=sys.stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if coverage_rc:
os.unlink(coverage_rc)
if po and po.returncode != 0:
try:
os.killpg(po.pid, signal.SIGTERM)
except OSError as e:
if e.errno == errno.EPERM:
print("Unable to kill process group: %d" % po.pid)
elif e.errno != errno.ESRCH:
rv = PythonTestResult.exception()
self.successes[target._create_id()] = rv
return rv
def _run_lint(self, target, args):
chroot = PythonChroot(target, self.root_dir, extra_targets=[
Target.get(Address.parse(self.root_dir, '3rdparty/python:pylint'))])
builder = chroot.dump()
builder.info().entry_point = 'pylint.lint'
builder.freeze()
interpreter_args = [
'--rcfile=%s' % os.path.join(self.root_dir, 'build-support', 'pylint', 'pylint.rc')]
interpreter_args.extend(args or [])
sources = OrderedSet([])
target.walk(lambda trg: sources.update(
trg.sources if hasattr(trg, 'sources') and trg.sources is not None else []))
pex = PEX(builder.path())
pex.run(args=interpreter_args + list(sources), with_chroot=True)
def _run_python_test(self, target):
chroot = PythonChroot(target, self.root_dir)
chroot.dump()
launcher = Launcher(chroot.path())
extra_deps = PythonTestBuilder.get_pytest_eggs(self.root_dir)
test_args = ['-m', 'pytest']
test_args.extend(PythonTestBuilder.generate_junit_args(target))
test_args.extend(self.args) # Pass any extra args in to pytest.
sources = [
os.path.join(target.target_base, source)
for source in target.sources
]
return launcher.run(interpreter_args=test_args,
args=list(sources),
extra_deps=extra_deps)
def execute(self):
if self.options.verbose:
print("Build operating on target: %s %s" %
(self.target,
'Extra targets: %s' % ' '.join(map(str, self.extra_targets))
if self.extra_targets else ''))
if self.options.resolve:
executor = PythonResolver([self.target] + self.extra_targets)
executor.dump()
return 0
executor = PythonChroot(self.target,
self.root_dir,
extra_targets=self.extra_targets,
conn_timeout=self.options.conn_timeout)
builder = executor.dump()
if self.options.pex:
pex_name = os.path.join(self.root_dir, 'dist',
'%s.pex' % self.target.name)
builder.build(pex_name)
print('Wrote %s' % pex_name)
return 0
else:
builder.freeze()
pex = PEX(builder.path())
po = pex.run(args=list(self.args), blocking=False)
try:
po.wait()
except KeyboardInterrupt:
po.send_signal(signal.SIGINT)
raise
class PythonBinaryBuilder(object):
class NotABinaryTargetException(Exception): pass
def __init__(self, target, args, root_dir, conn_timeout=None):
self.target = target
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
self.builder = PEXBuilder(distpath)
# configure builder PexInfo options
for repo in target._repositories:
self.builder.info().add_repository(repo)
for index in target._indices:
self.builder.info().add_index(index)
self.builder.info().allow_pypi = target._allow_pypi
self.builder.info().zip_safe = target._zip_safe
self.builder.info().inherit_path = target._inherit_path
self.builder.info().entry_point = target._entry_point
self.builder.info().ignore_errors = target._ignore_errors
self.chroot = PythonChroot(target, root_dir, builder=self.builder, conn_timeout=conn_timeout)
def run(self):
print('Building PythonBinary %s:' % self.target)
env = self.chroot.dump()
filename = os.path.join(self.distdir, '%s.pex' % self.target.name)
env.build(filename)
print('Wrote %s' % filename)
return 0
class PythonBinaryBuilder(object):
class NotABinaryTargetException(Exception):
pass
def __init__(self, target, args, root_dir):
self.target = target
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
self.builder = PEXBuilder(distpath)
# configure builder PexInfo options
for repo in target._repositories:
self.builder.info().add_repository(repo)
for index in target._indices:
self.builder.info().add_index(index)
self.builder.info().allow_pypi = target._allow_pypi
self.builder.info().zip_safe = target._zip_safe
self.builder.info().inherit_path = target._inherit_path
self.builder.info().entry_point = target._entry_point
self.builder.info().ignore_errors = target._ignore_errors
self.chroot = PythonChroot(target, root_dir, builder=self.builder)
def run(self):
print('Building PythonBinary %s:' % self.target)
env = self.chroot.dump()
filename = os.path.join(self.distdir, '%s.pex' % self.target.name)
env.build(filename)
print('Wrote %s' % filename)
return 0
class PythonBinaryBuilder(object):
class NotABinaryTargetException(Exception): pass
def __init__(self, target, args, root_dir):
self.target = target
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
self.chroot = PythonChroot(target, root_dir)
self.distdir = os.path.join(root_dir, 'dist')
def _generate_zip(self):
chroot = self.chroot.dump()
zp_path = os.path.join(self.distdir, '%s.zip' % self.target.name)
zippy = zipfile.ZipFile(zp_path, 'w', compression = zipfile.ZIP_DEFLATED)
sorted_digest = list(chroot.files())
sorted_digest.sort()
for pth in sorted_digest:
zippy.write(os.path.join(chroot.path(), pth), pth)
zippy.close()
return zp_path
def run(self):
print 'Building PythonBinary %s:' % self.target
zp_path = self._generate_zip()
print 'generated zip binary in: %s' % zp_path
pexer = PexCreator(zp_path, self.target.name)
pex = pexer.build(os.path.join(self.distdir, '%s.pex' % self.target.name))
print 'generated pex binary in: %s' % pex
return 0
def _run_lint(self, target, args):
chroot = PythonChroot(target, self.root_dir)
launcher = PythonLauncher(chroot.dump().path())
interpreter_args = ['-m', 'pylint.lint',
'--rcfile=%s' % os.path.join(self.root_dir, 'build-support', 'pylint', 'pylint.rc')]
if args:
interpreter_args.extend(args)
sources = OrderedSet([])
if not isinstance(target, PythonEgg):
target.walk(lambda trg: sources.update(
trg.sources if hasattr(trg, 'sources') and trg.sources is not None else []))
launcher.run(
interpreter_args=interpreter_args,
args=list(sources),
extra_deps=sys.path, # TODO(wickman) Extract only the pylint dependencies from sys.path
with_chroot=True)
def _run_lint(self, target, args):
lint_target = Target.get(Address.parse(self.root_dir, self._opts['lint_target']))
assert lint_target, 'Could not find target %r' % self._opts['lint_target']
chroot = PythonChroot(target, self.root_dir, extra_targets=[lint_target],
conn_timeout=self._conn_timeout)
chroot.builder.info().ignore_errors = True
builder = chroot.dump()
builder.info().entry_point = self._opts['entry_point']
builder.info().run_name = 'main'
builder.freeze()
interpreter_args = self._opts['interpreter_args']
interpreter_args.extend(args or [])
sources = OrderedSet([])
target.walk(lambda trg: sources.update(
trg.sources if hasattr(trg, 'sources') and trg.sources is not None else []))
pex = PEX(builder.path())
pex.run(args=interpreter_args + list(sources), with_chroot=True)
def _run_lint(self, target, args):
chroot = PythonChroot(target, self.root_dir)
chroot.dump()
launcher = Launcher(chroot.path())
interpreter_args = [
'-m', 'pylint.lint',
'--rcfile=%s' %
os.path.join(self.root_dir, 'build-support', 'pylint', 'pylint.rc')
]
if args:
interpreter_args.extend(args)
sources = OrderedSet([])
if not isinstance(target, PythonEgg):
target.walk(lambda trg: sources.update(trg.sources),
lambda trg: not isinstance(trg, PythonEgg))
launcher.run(interpreter_args=interpreter_args,
args=list(sources),
with_chroot=True)
def _run_lint(self, target, args):
chroot = PythonChroot(
target,
self.root_dir,
extra_targets=[Target.get(Address.parse(self.root_dir, "3rdparty/python:pylint"))],
conn_timeout=self._conn_timeout,
)
chroot.builder.info().ignore_errors = True
builder = chroot.dump()
builder.info().entry_point = "pylint.lint"
builder.freeze()
interpreter_args = ["--rcfile=%s" % os.path.join(self.root_dir, "build-support", "pylint", "pylint.rc")]
interpreter_args.extend(args or [])
sources = OrderedSet([])
target.walk(
lambda trg: sources.update(trg.sources if hasattr(trg, "sources") and trg.sources is not None else [])
)
pex = PEX(builder.path())
pex.run(args=interpreter_args + list(sources), with_chroot=True)
def _run_python_test(self, target):
po = None
rv = PythonTestResult.exception()
try:
builder = PEXBuilder()
builder.info().entry_point = 'pytest'
builder.info().ignore_errors = target._soft_dependencies
chroot = PythonChroot(target,
self.root_dir,
extra_targets=self.generate_test_targets(),
builder=builder)
builder = chroot.dump()
builder.freeze()
test_args = PythonTestBuilder.generate_junit_args(target)
test_args.extend(self.args)
sources = [
os.path.join(target.target_base, source)
for source in target.sources
]
po = PEX(builder.path()).run(args=test_args + sources,
blocking=False,
setsid=True)
rv = PythonTestBuilder.wait_on(po, timeout=target.timeout)
except Exception as e:
import traceback
print('Failed to run test!', file=sys.stderr)
traceback.print_exc()
rv = PythonTestResult.exception()
finally:
if po and po.returncode != 0:
try:
os.killpg(po.pid, signal.SIGTERM)
except OSError as e:
if e.errno == errno.EPERM:
print("Unable to kill process group: %d" % po.pid)
elif e.errno != errno.ESRCH:
rv = PythonTestResult.exception()
self.successes[target._create_id()] = rv
return rv
def _run_lint(self, target, args):
chroot = PythonChroot(target,
self.root_dir,
extra_targets=[
Target.get(
Address.parse(self.root_dir,
'3rdparty/python:pylint'))
])
builder = chroot.dump()
builder.info().entry_point = 'pylint.lint'
builder.freeze()
interpreter_args = [
'--rcfile=%s' %
os.path.join(self.root_dir, 'build-support', 'pylint', 'pylint.rc')
]
interpreter_args.extend(args or [])
sources = OrderedSet([])
target.walk(lambda trg: sources.update(trg.sources if hasattr(
trg, 'sources') and trg.sources is not None else []))
pex = PEX(builder.path())
pex.run(args=interpreter_args + list(sources), with_chroot=True)
def __init__(self, target, args, root_dir):
self.target = target
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
self.builder = PEXBuilder(distpath)
# configure builder PexInfo options
for repo in target._repositories:
self.builder.info().add_repository(repo)
for index in target._indices:
self.builder.info().add_index(index)
self.builder.info().allow_pypi = target._allow_pypi
self.builder.info().zip_safe = target._zip_safe
self.builder.info().inherit_path = target._inherit_path
self.builder.info().entry_point = target._entry_point
self.builder.info().ignore_errors = target._ignore_errors
self.chroot = PythonChroot(target, root_dir, builder=self.builder)
def _run_python_test(self, target):
chroot = PythonChroot(target, self.root_dir)
chroot.add_req(pkg_resources.Requirement.parse('pytest'))
chroot.add_req(pkg_resources.Requirement.parse('unittest2'))
launcher = PythonLauncher(chroot.dump().path())
test_args = ['-m', 'pytest']
test_args.extend(PythonTestBuilder.generate_junit_args(target))
test_args.extend(self.args) # Pass any extra args in to pytest.
sources = [os.path.join(target.target_base, source) for source in target.sources]
return launcher.run(interpreter_args=test_args,
args=list(sources),
kill_orphans=True,
)
def execute(self):
print("Build operating on target: %s %s" % (self.target,
'Extra targets: %s' % ' '.join(map(str, self.extra_targets)) if self.extra_targets else ''))
if self.options.resolve:
executor = PythonResolver(self.target, self.root_dir, extra_targets=self.extra_targets)
executor.dump()
return 0
executor = PythonChroot(self.target, self.root_dir, extra_targets=self.extra_targets)
builder = executor.dump()
if self.options.pex:
pex_name = os.path.join(self.root_dir, 'dist', '%s.pex' % self.target.name)
builder.build(pex_name)
print('Wrote %s' % pex_name)
return 0
else:
builder.freeze()
pex = PEX(builder.path())
return pex.run(args=list(self.args))
class PythonBinaryBuilder(object):
class NotABinaryTargetException(Exception):
pass
def __init__(self, target, args, root_dir):
self.target = target
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException("Target %s is not a PythonBinary!" % target)
self.chroot = PythonChroot(target, root_dir)
self.distdir = os.path.join(root_dir, "dist")
def _generate(self):
env = self.chroot.dump()
pex = PexBuilder(env)
pex_name = os.path.join(self.distdir, "%s.pex" % self.target.name)
pex.write(pex_name)
print("Wrote %s" % pex_name)
def run(self):
print("Building PythonBinary %s:" % self.target)
self._generate()
return 0
def __init__(self, target, args, root_dir, conn_timeout=None):
self.target = target
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
self.builder = PEXBuilder(distpath)
# configure builder PexInfo options
for repo in target._repositories:
self.builder.info().add_repository(repo)
for index in target._indices:
self.builder.info().add_index(index)
self.builder.info().allow_pypi = target._allow_pypi
self.builder.info().zip_safe = target._zip_safe
self.builder.info().inherit_path = target._inherit_path
self.builder.info().entry_point = target._entry_point
self.builder.info().ignore_errors = target._ignore_errors
self.chroot = PythonChroot(target, root_dir, builder=self.builder, conn_timeout=conn_timeout)