def create_scaladoc_command(self, classpath, gendir, *targets):
sources = []
for target in targets:
sources.extend(target.sources_relative_to_buildroot())
# TODO(Tejal Desai): pantsbuild/pants/65: Remove java_sources attribute for ScalaLibrary
for java_target in target.java_sources:
sources.extend(java_target.sources_relative_to_buildroot())
if not sources:
return None
scala_platform = ScalaPlatform.global_instance()
tool_classpath = scala_platform.compiler_classpath(self.context.products)
args = ['-usejavacp',
'-classpath', ':'.join(classpath),
'-d', gendir]
args.extend(self.args)
args.extend(sources)
java_executor = SubprocessExecutor()
runner = java_executor.runner(jvm_options=self.jvm_options,
classpath=tool_classpath,
main='scala.tools.nsc.ScalaDoc',
args=args)
return runner.command
def create_scaladoc_command(self, classpath, gendir, *targets):
sources = []
for target in targets:
sources.extend(target.sources_relative_to_buildroot())
# TODO(Tejal Desai): pantsbuild/pants/65: Remove java_sources attribute for ScalaLibrary
# A '.scala' owning target may not have java_sources, eg: junit_tests
if hasattr(target, "java_sources"):
for java_target in target.java_sources:
sources.extend(java_target.sources_relative_to_buildroot())
if not sources:
return None
scala_platform = ScalaPlatform.global_instance()
tool_classpath = [
cp_entry.path
for cp_entry in scala_platform.compiler_classpath_entries(self.context.products)
]
args = ["-usejavacp", "-classpath", ":".join(classpath), "-d", gendir]
args.extend(self.args)
args.extend(sources)
java_executor = SubprocessExecutor(self.preferred_jvm_distribution_for_targets(targets))
runner = java_executor.runner(
jvm_options=self.jvm_options,
classpath=tool_classpath,
main="scala.tools.nsc.ScalaDoc",
args=args,
)
return runner.command
def create_scaladoc_command(self, classpath, gendir, *targets):
sources = []
for target in targets:
sources.extend(target.sources_relative_to_buildroot())
# TODO(Tejal Desai): pantsbuild/pants/65: Remove java_sources attribute for ScalaLibrary
# A '.scala' owning target may not have java_sources, eg: junit_tests
if hasattr(target, 'java_sources'):
for java_target in target.java_sources:
sources.extend(java_target.sources_relative_to_buildroot())
if not sources:
return None
scala_platform = ScalaPlatform.global_instance()
tool_classpath = scala_platform.compiler_classpath(
self.context.products)
args = ['-usejavacp', '-classpath', ':'.join(classpath), '-d', gendir]
args.extend(self.args)
args.extend(sources)
java_executor = SubprocessExecutor(DistributionLocator.cached())
runner = java_executor.runner(jvm_options=self.jvm_options,
classpath=tool_classpath,
main='scala.tools.nsc.ScalaDoc',
args=args)
return runner.command
def create_scaladoc_command(self, classpath, gendir, *targets):
sources = []
for target in targets:
sources.extend(target.sources_relative_to_buildroot())
# TODO(Tejal Desai): pantsbuild/pants/65: Remove java_sources attribute for ScalaLibrary
# A '.scala' owning target may not have java_sources, eg: junit_tests
if hasattr(target, 'java_sources'):
for java_target in target.java_sources:
sources.extend(java_target.sources_relative_to_buildroot())
if not sources:
return None
scala_platform = ScalaPlatform.global_instance()
tool_classpath = [cp_entry.path for cp_entry in scala_platform.compiler_classpath_entries(
self.context.products, self.context._scheduler)]
args = ['-usejavacp',
'-classpath', ':'.join(classpath),
'-d', gendir]
args.extend(self.args)
args.extend(sources)
java_executor = SubprocessExecutor(DistributionLocator.cached())
runner = java_executor.runner(jvm_options=self.jvm_options,
classpath=tool_classpath,
main='scala.tools.nsc.ScalaDoc',
args=args)
return runner.command
def execute_tool(self, classpath, main, args=None):
init_subsystem(DistributionLocator)
executor = SubprocessExecutor(DistributionLocator.cached())
process = executor.spawn(classpath, main, args=args,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = process.communicate()
self.assertEqual(0, process.returncode)
self.assertEqual('', err.strip().decode())
yield out.decode()
def execute_tool(self, classpath, main, args=None):
init_subsystem(DistributionLocator)
executor = SubprocessExecutor(DistributionLocator.cached())
process = executor.spawn(classpath, main, args=args,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = process.communicate()
self.assertEqual(0, process.returncode)
self.assertEqual('', err.strip())
yield out
def assert_run_ant_version(classpath):
with subsystem_instance(DistributionLocator):
executor = SubprocessExecutor(DistributionLocator.cached())
process = executor.spawn(classpath, 'org.apache.tools.ant.Main', args=['-version'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = process.communicate()
self.assertEqual(0, process.returncode)
self.assertTrue(out.strip().startswith('Apache Ant(TM) version 1.9.4'))
self.assertEqual('', err.strip())
def do_test_jre_env_var(self, env_var, env_value, scrubbed=True):
with self.jre(env_var=env_var) as jre:
executor = SubprocessExecutor(Distribution(bin_path=jre))
with environment_as(**{env_var: env_value}):
self.assertEqual(env_value, os.getenv(env_var))
process = executor.spawn(classpath=['dummy/classpath'],
main='dummy.main',
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_, stderr = process.communicate()
self.assertEqual(0, process.returncode)
self.assertEqual('' if scrubbed else env_value, stderr.strip())
def create_javadoc_command(self, classpath, gendir, *targets):
sources = []
for target in targets:
sources.extend(target.sources_relative_to_buildroot())
if not sources:
return None
# Without a JDK/tools.jar we have no javadoc tool and cannot proceed, so check/acquire early.
jdk = DistributionLocator.cached(jdk=True)
tool_classpath = jdk.find_libs(['tools.jar'])
args = ['-quiet',
'-encoding', 'UTF-8',
'-notimestamp',
'-use',
'-Xmaxerrs', '10000', # the default is 100
'-Xmaxwarns', '10000', # the default is 100
'-d', gendir]
# Always provide external linking for java API
offlinelinks = {'http://download.oracle.com/javase/8/docs/api/'}
def link(target):
for jar in target.jar_dependencies:
if jar.apidocs:
offlinelinks.add(jar.apidocs)
for target in targets:
target.walk(link, lambda t: isinstance(t, (JvmTarget, JarLibrary)))
for link in offlinelinks:
args.extend(['-linkoffline', link, link])
args.extend(self.args)
javadoc_classpath_file = os.path.join(self.workdir, '{}.classpath'.format(os.path.basename(gendir)))
with open(javadoc_classpath_file, 'w') as f:
f.write('-classpath ')
f.write(':'.join(classpath))
args.extend(['@{}'.format(javadoc_classpath_file)])
javadoc_sources_file = os.path.join(self.workdir, '{}.source.files'.format(os.path.basename(gendir)))
with open(javadoc_sources_file, 'w') as f:
f.write('\n'.join(sources))
args.extend(['@{}'.format(javadoc_sources_file)])
java_executor = SubprocessExecutor(jdk)
runner = java_executor.runner(jvm_options=self.jvm_options,
classpath=tool_classpath,
main='com.sun.tools.javadoc.Main',
args=args)
return runner.command
def post_fork_child(self, fingerprint, jvm_options, classpath, stdout, stderr):
"""Post-fork() child callback for ProcessManager.daemon_spawn()."""
java = SubprocessExecutor(self._distribution)
subproc = java.spawn(classpath=classpath,
main='com.martiansoftware.nailgun.NGServer',
jvm_options=jvm_options,
args=[':0'],
stdin=safe_open('/dev/null', 'r'),
stdout=safe_open(self._ng_stdout, 'w'),
stderr=safe_open(self._ng_stderr, 'w'),
close_fds=True)
self.write_pid(subproc.pid)
def post_fork_child(self, fingerprint, jvm_options, classpath, stdout, stderr):
"""Post-fork() child callback for ProcessManager.daemon_spawn()."""
java = SubprocessExecutor(self._distribution)
subproc = java.spawn(classpath=classpath,
main='com.martiansoftware.nailgun.NGServer',
jvm_options=jvm_options,
args=[':0'],
stdin=safe_open('/dev/null', 'r'),
stdout=safe_open(self._ng_stdout, 'w'),
stderr=safe_open(self._ng_stderr, 'w'),
close_fds=True)
self.write_pid(subproc.pid)
def runner(self, jvm_options=None, args=None, executor=None):
"""Creates an ivy commandline client runner for the given args."""
args = args or []
jvm_options = jvm_options or []
executor = executor or SubprocessExecutor()
if not isinstance(executor, Executor):
raise ValueError(
'The executor argument must be an Executor instance, given {} of type {}'
.format(executor, type(executor)))
if self._ivy_cache_dir and '-cache' not in args:
# TODO(John Sirois): Currently this is a magic property to support hand-crafted <caches/> in
# ivysettings.xml. Ideally we'd support either simple -caches or these hand-crafted cases
# instead of just hand-crafted. Clean this up by taking over ivysettings.xml and generating
# it from BUILD constructs.
jvm_options += ['-Divy.cache.dir={}'.format(self._ivy_cache_dir)]
if self._ivy_settings and '-settings' not in args:
args = ['-settings', self._ivy_settings] + args
jvm_options += self._extra_jvm_options
return executor.runner(classpath=self._classpath,
main='org.apache.ivy.Main',
jvm_options=jvm_options,
args=args)
def setUp(self):
self.jarjar = '/not/really/jarjar.jar'
with subsystem_instance(DistributionLocator):
executor = SubprocessExecutor(DistributionLocator.cached())
self.shader = Shader(jarjar_classpath=[self.jarjar],
executor=executor)
self.output_jar = '/not/really/shaded.jar'
def resolve_jars(self, targets):
# TODO: Why is this computed directly here instead of taking from the actual product
# computed by the {Ivy,Coursier}Resolve task?
executor = SubprocessExecutor(DistributionLocator.cached())
confs = []
if self.get_options().libraries:
confs.append("default")
if self.get_options().libraries_sources:
confs.append("sources")
if self.get_options().libraries_javadocs:
confs.append("javadoc")
compile_classpath = None
if confs:
compile_classpath = ClasspathProducts(
self.get_options().pants_workdir)
CoursierMixin.resolve(
self,
targets,
compile_classpath,
sources=self.get_options().libraries_sources,
javadoc=self.get_options().libraries_javadocs,
executor=executor,
)
return compile_classpath
def resolve_jars(self, targets):
executor = SubprocessExecutor(DistributionLocator.cached())
confs = []
if self.get_options().libraries:
confs.append('default')
if self.get_options().libraries_sources:
confs.append('sources')
if self.get_options().libraries_javadocs:
confs.append('javadoc')
compile_classpath = None
if confs:
compile_classpath = ClasspathProducts(
self.get_options().pants_workdir)
if JvmResolveSubsystem.global_instance().get_options(
).resolver == 'ivy':
IvyTaskMixin.resolve(self,
executor=executor,
targets=targets,
classpath_products=compile_classpath,
confs=confs)
else:
CoursierMixin.resolve(
self,
targets,
compile_classpath,
sources=self.get_options().libraries_sources,
javadoc=self.get_options().libraries_javadocs)
return compile_classpath
def __init__(self,
classpath,
java_executor=None,
ivy_settings=None,
ivy_cache_dir=None):
"""Configures an ivy wrapper for the ivy distribution at the given classpath."""
self._classpath = maybe_list(classpath)
self._java = java_executor or SubprocessExecutor()
if not isinstance(self._java, Executor):
raise ValueError(
'java_executor must be an Executor instance, given %s of type %s'
% (self._java, type(self._java)))
self._ivy_settings = ivy_settings
if self._ivy_settings and not isinstance(self._ivy_settings,
Compatibility.string):
raise ValueError(
'ivy_settings must be a string, given %s of type %s' %
(self._ivy_settings, type(self._ivy_settings)))
self._ivy_cache_dir = ivy_cache_dir
if self._ivy_cache_dir and not isinstance(self._ivy_cache_dir,
Compatibility.string):
raise ValueError(
'ivy_cache_dir must be a string, given %s of type %s' %
(self._ivy_cache_dir, type(self._ivy_cache_dir)))
def execute_java(classpath, main, jvm_options=None, args=None, executor=None,
workunit_factory=None, workunit_name=None, workunit_labels=None,
cwd=None, workunit_log_config=None):
"""Executes the java program defined by the classpath and main.
If `workunit_factory` is supplied, does so in the context of a workunit.
:param list classpath: the classpath for the java program
:param string main: the fully qualified class name of the java program's entry point
:param list jvm_options: an optional sequence of options for the underlying jvm
:param list args: an optional sequence of args to pass to the java program
:param executor: an optional java executor to use to launch the program; defaults to a subprocess
spawn of the default java distribution
:param workunit_factory: an optional callable that can produce a workunit context
:param string workunit_name: an optional name for the work unit; defaults to the main
:param list workunit_labels: an optional sequence of labels for the work unit
:param string cwd: optionally set the working directory
:param WorkUnit.LogConfig workunit_log_config: an optional tuple of options affecting reporting
Returns the exit code of the java program.
Raises `pants.java.Executor.Error` if there was a problem launching java itself.
"""
executor = executor or SubprocessExecutor()
if not isinstance(executor, Executor):
raise ValueError('The executor argument must be a java Executor instance, give {} of type {}'
.format(executor, type(executor)))
runner = executor.runner(classpath, main, args=args, jvm_options=jvm_options, cwd=cwd)
workunit_name = workunit_name or main
return execute_runner(runner,
workunit_factory=workunit_factory,
workunit_name=workunit_name,
workunit_labels=workunit_labels,
cwd=cwd,
workunit_log_config=workunit_log_config)
def execute_junit_runner(self, content):
# Create the temporary base test directory
test_rel_path = 'tests/java/org/pantsbuild/foo'
test_abs_path = os.path.join(self.build_root, test_rel_path)
self.create_dir(test_rel_path)
# Generate the temporary java test source code.
test_java_file_rel_path = os.path.join(test_rel_path, 'FooTest.java')
test_java_file_abs_path = os.path.join(self.build_root, test_java_file_rel_path)
self.create_file(test_java_file_rel_path, content)
# Invoke ivy to resolve classpath for junit.
distribution = Distribution.cached(jdk=True)
executor = SubprocessExecutor(distribution=distribution)
classpath_file_abs_path = os.path.join(test_abs_path, 'junit.classpath')
with subsystem_instance(IvySubsystem) as ivy_subsystem:
ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy()
ivy.execute(args=['-cachepath', classpath_file_abs_path,
'-dependency', 'junit', 'junit-dep', '4.10'], executor=executor)
with open(classpath_file_abs_path) as fp:
classpath = fp.read()
# Now directly invoking javac to compile the test java code into java class
# so later we can inject the class into products mapping for JUnitRun to execute
# the test on.
javac = distribution.binary('javac')
subprocess.check_call(
[javac, '-d', test_abs_path, '-cp', classpath, test_java_file_abs_path])
# Create a java_tests target and a synthetic resource target.
java_tests = self.create_library(test_rel_path, 'java_tests', 'foo_test', ['FooTest.java'])
resources = self.make_target('some_resources', Resources)
# Set the context with the two targets, one java_tests target and
# one synthetic resources target.
# The synthetic resources target is to make sure we won't regress
# in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
# in that bug, the resources target must be the first one in the list.
context = self.context(target_roots=[resources, java_tests])
# Before we run the task, we need to inject the "classes_by_target" with
# the compiled test java classes that JUnitRun will know which test
# classes to execute. In a normal run, this "classes_by_target" will be
# populated by java compiling step.
class_products = context.products.get_data(
'classes_by_target', lambda: defaultdict(MultipleRootedProducts))
java_tests_products = MultipleRootedProducts()
java_tests_products.add_rel_paths(test_abs_path, ['FooTest.class'])
class_products[java_tests] = java_tests_products
# Also we need to add the FooTest.class's classpath to the compile_classpath
# products data mapping so JUnitRun will be able to add that into the final
# classpath under which the junit will be executed.
self.populate_compile_classpath(context=context, classpath=[test_abs_path])
# Finally execute the task.
self.execute(context)
def _spawn_nailgun_server(self, fingerprint, jvm_options, classpath,
stdout, stderr):
logger.debug(
'No ng server found with fingerprint {fingerprint}, spawning...'.
format(fingerprint=fingerprint))
with safe_open(self._ng_out, 'w'):
pass # truncate
pid = os.fork()
if pid != 0:
# In the parent tine - block on ng being up for connections
return self._await_nailgun_server(
stdout, stderr,
'jvm_options={jvm_options} classpath={classpath}'.format(
jvm_options=jvm_options, classpath=classpath))
os.setsid()
in_fd = open('/dev/null', 'r')
out_fd = safe_open(self._ng_out, 'w')
err_fd = safe_open(self._ng_err, 'w')
java = SubprocessExecutor(self._distribution)
jvm_options = jvm_options + [
self._PANTS_NG_ARG,
self.create_owner_arg(self._workdir),
self._create_fingerprint_arg(fingerprint)
]
process = java.spawn(classpath=classpath,
main='com.martiansoftware.nailgun.NGServer',
jvm_options=jvm_options,
args=[':0'],
stdin=in_fd,
stdout=out_fd,
stderr=err_fd,
close_fds=True)
logger.debug(
'Spawned ng server with fingerprint {fingerprint} @ {pid}'.format(
fingerprint=fingerprint, pid=process.pid))
# Prevents finally blocks and atexit handlers from being executed, unlike sys.exit(). We
# don't want to execute finally blocks because we might, e.g., clean up tempfiles that the
# parent still needs.
os._exit(0)
def setUp(self):
self.jarjar = '/not/really/jarjar.jar'
init_subsystem(DistributionLocator)
executor = SubprocessExecutor(DistributionLocator.cached())
self.shader = Shader(jarjar_classpath=[self.jarjar],
executor=executor,
binary_package_excludes=['javax'])
self.output_jar = '/not/really/shaded.jar'
def execute_java_for_targets(self, targets, *args, **kwargs):
"""Execute java for targets using the test mixin spawn and wait.
Activates timeouts and other common functionality shared among tests.
"""
distribution = self.preferred_jvm_distribution_for_targets(targets)
actual_executor = kwargs.get('executor') or SubprocessExecutor(distribution)
return self._spawn_and_wait(*args, executor=actual_executor, distribution=distribution, **kwargs)
def _bundle_and_run(self, bundle_args, classpath):
self.assert_success(self.run_pants(['clean-all']))
pants_command = list(bundle_args)
pants_command.append('testprojects/src/java/org/pantsbuild/testproject/shading:third')
self.assert_success(self.run_pants(pants_command))
main_class = 'org.pantsbuild.testproject.shading.Third'
with subsystem_instance(DistributionLocator):
executor = SubprocessExecutor(DistributionLocator.cached(minimum_version='1.7'))
p = executor.spawn(classpath, main_class, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
self.assertEqual(0, p.returncode, err)
class_names = json.loads(out.strip())
self.assertEqual({
'Gson': 'moc.elgoog.nosg.Gson',
'Third': 'org.pantsbuild.testproject.shading.Third',
'Second': 'hello.org.pantsbuild.testproject.shading.Second',
}, class_names)
def __init__(self, jarjar, executor=None):
"""Creates a `Shader` the will use the given `jarjar` jar to create shaded jars.
:param unicode jarjar: The path to the jarjar jar.
:param executor: An optional java `Executor` to use to create shaded jar files. Defaults to a
`SubprocessExecutor` that uses the default java distribution.
"""
self._jarjar = jarjar
self._executor = executor or SubprocessExecutor()
self._system_packages = None
def execute_java_for_targets(self,
targets,
executor=None,
*args,
**kwargs):
distribution = self.preferred_jvm_distribution_for_targets(targets)
self._executor = executor or SubprocessExecutor(distribution)
return distribution.execute_java(*args,
executor=self._executor,
**kwargs)
def create_javadoc_command(self, classpath, gendir, *targets):
sources = []
for target in targets:
sources.extend(target.sources_relative_to_buildroot())
if not sources:
return None
# Without a JDK/tools.jar we have no javadoc tool and cannot proceed, so check/acquire early.
jdk = DistributionLocator.cached(jdk=True)
tool_classpath = jdk.find_libs(['tools.jar'])
args = ['-quiet',
'-encoding', 'UTF-8',
'-notimestamp',
'-use',
'-classpath', ':'.join(classpath),
'-d', gendir]
# Always provide external linking for java API
offlinelinks = {'http://download.oracle.com/javase/6/docs/api/'}
def link(target):
for jar in target.jar_dependencies:
if jar.apidocs:
offlinelinks.add(jar.apidocs)
for target in targets:
target.walk(link, lambda t: t.is_jvm)
for link in offlinelinks:
args.extend(['-linkoffline', link, link])
args.extend(self.args)
args.extend(sources)
java_executor = SubprocessExecutor(jdk)
runner = java_executor.runner(jvm_options=self.jvm_options,
classpath=tool_classpath,
main='com.sun.tools.javadoc.Main',
args=args)
return runner.command
def _spawn(self, distribution, executor=None, *args, **kwargs):
"""Returns a processhandler to a process executing java.
:param Executor executor: the java subprocess executor to use. If not specified, construct
using the distribution.
:param Distribution distribution: The JDK or JRE installed.
:rtype: ProcessHandler
"""
actual_executor = executor or SubprocessExecutor(distribution)
return distribution.execute_java_async(*args, executor=actual_executor, **kwargs)
def _spawn_nailgun_server(self, fingerprint, jvm_options, classpath, stdout, stderr):
logger.debug('No ng server found with fingerprint {fingerprint}, spawning...'
.format(fingerprint=fingerprint))
with safe_open(self._ng_out, 'w'):
pass # truncate
pid = os.fork()
if pid != 0:
# In the parent tine - block on ng being up for connections
return self._await_nailgun_server(stdout, stderr,
'jvm_options={jvm_options} classpath={classpath}'
.format(jvm_options=jvm_options, classpath=classpath))
os.setsid()
in_fd = open('/dev/null', 'r')
out_fd = safe_open(self._ng_out, 'w')
err_fd = safe_open(self._ng_err, 'w')
java = SubprocessExecutor(self._distribution)
jvm_options = jvm_options + [self._PANTS_NG_ARG,
self.create_owner_arg(self._workdir),
self._create_fingerprint_arg(fingerprint)]
process = java.spawn(classpath=classpath,
main='com.martiansoftware.nailgun.NGServer',
jvm_options=jvm_options,
args=[':0'],
stdin=in_fd,
stdout=out_fd,
stderr=err_fd,
close_fds=True)
logger.debug('Spawned ng server with fingerprint {fingerprint} @ {pid}'
.format(fingerprint=fingerprint, pid=process.pid))
# Prevents finally blocks and atexit handlers from being executed, unlike sys.exit(). We
# don't want to execute finally blocks because we might, e.g., clean up tempfiles that the
# parent still needs.
os._exit(0)
def _run_tests(self, tests_to_targets):
if self._coverage:
extra_jvm_options = self._coverage.extra_jvm_options
classpath_prepend = self._coverage.classpath_prepend
classpath_append = self._coverage.classpath_append
else:
extra_jvm_options = []
classpath_prepend = ()
classpath_append = ()
tests_by_properties = self._tests_by_properties(tests_to_targets,
self._infer_workdir,
lambda target: target.test_platform)
# the below will be None if not set, and we'll default back to runtime_classpath
classpath_product = self.context.products.get_data('instrument_classpath')
result = 0
for (workdir, platform), tests in tests_by_properties.items():
for (target_jvm_options, target_tests) in self._partition_by_jvm_options(tests_to_targets,
tests):
for batch in self._partition(target_tests):
# Batches of test classes will likely exist within the same targets: dedupe them.
relevant_targets = set(map(tests_to_targets.get, batch))
complete_classpath = OrderedSet()
complete_classpath.update(classpath_prepend)
complete_classpath.update(self.tool_classpath('junit'))
complete_classpath.update(self.classpath(relevant_targets,
classpath_product=classpath_product))
complete_classpath.update(classpath_append)
distribution = self.preferred_jvm_distribution([platform])
with binary_util.safe_args(batch, self.get_options()) as batch_tests:
self.context.log.debug('CWD = {}'.format(workdir))
self.context.log.debug('platform = {}'.format(platform))
self._executor = SubprocessExecutor(distribution)
result += abs(distribution.execute_java(
executor=self._executor,
classpath=complete_classpath,
main=JUnitRun._MAIN,
jvm_options=self.jvm_options + extra_jvm_options + target_jvm_options,
args=self._args + batch_tests + [u'-xmlreport'],
workunit_factory=self.context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnitLabel.TEST],
cwd=workdir,
synthetic_jar_dir=self.workdir,
))
if result != 0 and self._fail_fast:
break
if result != 0:
failed_targets_and_tests = self._get_failed_targets(tests_to_targets)
failed_targets = sorted(failed_targets_and_tests, key=lambda target: target.address.spec)
error_message_lines = []
if self._failure_summary:
for target in failed_targets:
error_message_lines.append('\n{0}{1}'.format(' '*4, target.address.spec))
for test in sorted(failed_targets_and_tests[target]):
error_message_lines.append('{0}{1}'.format(' '*8, test))
error_message_lines.append(
'\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
.format(main=JUnitRun._MAIN, code=result, failed=len(failed_targets),
targets=pluralize(len(failed_targets), 'target'))
)
raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))
class JUnitRun(TestRunnerTaskMixin, JvmToolTaskMixin, JvmTask):
_MAIN = 'org.pantsbuild.tools.junit.ConsoleRunner'
@classmethod
def register_options(cls, register):
super(JUnitRun, cls).register_options(register)
register('--batch-size', advanced=True, type=int, default=sys.maxint,
help='Run at most this many tests in a single test process.')
register('--test', action='append',
help='Force running of just these tests. Tests can be specified using any of: '
'[classname], [classname]#[methodname], [filename] or [filename]#[methodname]')
register('--per-test-timer', action='store_true', help='Show progress and timer for each test.')
register('--default-parallel', advanced=True, action='store_true',
help='Run classes without @TestParallel or @TestSerial annotations in parallel.')
register('--parallel-threads', advanced=True, type=int, default=0,
help='Number of threads to run tests in parallel. 0 for autoset.')
register('--test-shard', advanced=True,
help='Subset of tests to run, in the form M/N, 0 <= M < N. '
'For example, 1/3 means run tests number 2, 5, 8, 11, ...')
register('--suppress-output', action='store_true', default=True,
deprecated_hint='Use --output-mode instead.',
deprecated_version='0.0.64',
help='Redirect test output to files in .pants.d/test/junit.')
register('--output-mode', choices=['ALL', 'FAILURE_ONLY', 'NONE'], default='NONE',
help='Specify what part of output should be passed to stdout. '
'In case of FAILURE_ONLY and parallel tests execution '
'output can be partial or even wrong. '
'All tests output also redirected to files in .pants.d/test/junit.')
register('--cwd', advanced=True,
help='Set the working directory. If no argument is passed, use the build root. '
'If cwd is set on a target, it will supersede this argument.')
register('--strict-jvm-version', action='store_true', default=False, advanced=True,
help='If true, will strictly require running junits with the same version of java as '
'the platform -target level. Otherwise, the platform -target level will be '
'treated as the minimum jvm to run.')
register('--failure-summary', action='store_true', default=True,
help='If true, includes a summary of which test-cases failed at the end of a failed '
'junit run.')
register('--allow-empty-sources', action='store_true', default=False, advanced=True,
help='Allows a junit_tests() target to be defined with no sources. Otherwise,'
'such a target will raise an error during the test run.')
cls.register_jvm_tool(register,
'junit',
classpath=[
JarDependency(org='org.pantsbuild', name='junit-runner', rev='0.0.13'),
],
main=JUnitRun._MAIN,
# TODO(John Sirois): Investigate how much less we can get away with.
# Clearly both tests and the runner need access to the same @Test,
# @Before, as well as other annotations, but there is also the Assert
# class and some subset of the @Rules, @Theories and @RunWith APIs.
custom_rules=[
Shader.exclude_package('junit.framework', recursive=True),
Shader.exclude_package('org.junit', recursive=True),
Shader.exclude_package('org.hamcrest', recursive=True),
Shader.exclude_package('org.pantsbuild.junit.annotations', recursive=True),
])
# TODO: Yuck, but will improve once coverage steps are in their own tasks.
for c in [Coverage, Cobertura]:
c.register_options(register, cls.register_jvm_tool)
@classmethod
def subsystem_dependencies(cls):
return super(JUnitRun, cls).subsystem_dependencies() + (DistributionLocator,)
@classmethod
def request_classes_by_source(cls, test_specs):
"""Returns true if the given test specs require the `classes_by_source` product to satisfy."""
for test_spec in test_specs:
src_spec, _ = interpret_test_spec(test_spec)
if src_spec:
return True
return False
@classmethod
def prepare(cls, options, round_manager):
super(JUnitRun, cls).prepare(options, round_manager)
# Compilation and resource preparation must have completed.
round_manager.require_data('runtime_classpath')
# If the given test specs require the classes_by_source product, request it.
if cls.request_classes_by_source(options.test or []):
round_manager.require_data('classes_by_source')
def __init__(self, *args, **kwargs):
super(JUnitRun, self).__init__(*args, **kwargs)
options = self.get_options()
self._coverage = None
if options.coverage or options.is_flagged('coverage_open'):
coverage_processor = options.coverage_processor
if coverage_processor == 'cobertura':
settings = CoberturaTaskSettings.from_task(self)
self._coverage = Cobertura(settings)
else:
raise TaskError('unknown coverage processor {0}'.format(coverage_processor))
self._tests_to_run = options.test
self._batch_size = options.batch_size
self._fail_fast = options.fail_fast
self._working_dir = options.cwd or get_buildroot()
self._strict_jvm_version = options.strict_jvm_version
self._args = copy.copy(self.args)
self._failure_summary = options.failure_summary
if (not options.suppress_output) or options.output_mode == 'ALL':
self._args.append('-output-mode=ALL')
elif options.output_mode == 'FAILURE_ONLY':
self._args.append('-output-mode=FAILURE_ONLY')
else:
self._args.append('-output-mode=NONE')
if self._fail_fast:
self._args.append('-fail-fast')
self._args.append('-outdir')
self._args.append(self.workdir)
if options.per_test_timer:
self._args.append('-per-test-timer')
if options.default_parallel:
self._args.append('-default-parallel')
self._args.append('-parallel-threads')
self._args.append(str(options.parallel_threads))
if options.test_shard:
self._args.append('-test-shard')
self._args.append(options.test_shard)
self._executor = None
def preferred_jvm_distribution_for_targets(self, targets):
return self.preferred_jvm_distribution([target.platform for target in targets
if isinstance(target, JvmTarget)])
def preferred_jvm_distribution(self, platforms):
"""Returns a jvm Distribution with a version that should work for all the platforms."""
if not platforms:
return DistributionLocator.cached()
min_version = max(platform.target_level for platform in platforms)
max_version = Revision(*(min_version.components + [9999])) if self._strict_jvm_version else None
return DistributionLocator.cached(minimum_version=min_version, maximum_version=max_version)
def execute_java_for_targets(self, targets, executor=None, *args, **kwargs):
distribution = self.preferred_jvm_distribution_for_targets(targets)
self._executor = executor or SubprocessExecutor(distribution)
return distribution.execute_java(*args, executor=self._executor, **kwargs)
def _collect_test_targets(self, targets):
"""Returns a mapping from test names to target objects for all tests that
are included in targets. If self._tests_to_run is set, return {test: None}
for these tests instead.
"""
tests_from_targets = dict(list(self._calculate_tests_from_targets(targets)))
if targets and self._tests_to_run:
# If there are some junit_test targets in the graph, find ones that match the requested
# test(s).
tests_with_targets = {}
unknown_tests = []
for test in self._get_tests_to_run():
# A test might contain #specific_method, which is not needed to find a target.
test_class_name = test.partition('#')[0]
target = tests_from_targets.get(test_class_name)
if target is None:
unknown_tests.append(test)
else:
tests_with_targets[test] = target
if len(unknown_tests) > 0:
raise TaskError("No target found for test specifier(s):\n\n '{}'\n\nPlease change " \
"specifier or bring in the proper target(s)."
.format("'\n '".join(unknown_tests)))
return tests_with_targets
else:
return tests_from_targets
def _get_failed_targets(self, tests_and_targets):
"""Return a mapping of target -> set of individual test cases that failed.
Targets with no failed tests are omitted.
Analyzes JUnit XML files to figure out which test had failed.
The individual test cases are formatted strings of the form org.foo.bar.classname#methodName.
:tests_and_targets: {test: target} mapping.
"""
def get_test_filename(test):
return os.path.join(self.workdir, 'TEST-{0}.xml'.format(test))
failed_targets = defaultdict(set)
for test, target in tests_and_targets.items():
if target is None:
self.context.log.warn('Unknown target for test %{0}'.format(test))
filename = get_test_filename(test)
if os.path.exists(filename):
try:
xml = XmlParser.from_file(filename)
str_failures = xml.get_attribute('testsuite', 'failures')
int_failures = int(str_failures)
str_errors = xml.get_attribute('testsuite', 'errors')
int_errors = int(str_errors)
if target and (int_failures or int_errors):
for testcase in xml.parsed.getElementsByTagName('testcase'):
test_failed = testcase.getElementsByTagName('failure')
test_errored = testcase.getElementsByTagName('error')
if test_failed or test_errored:
failed_targets[target].add('{testclass}#{testname}'.format(
testclass=testcase.getAttribute('classname'),
testname=testcase.getAttribute('name'),
))
except (XmlParser.XmlError, ValueError) as e:
self.context.log.error('Error parsing test result file {0}: {1}'.format(filename, e))
return dict(failed_targets)
def _run_tests(self, tests_to_targets):
if self._coverage:
extra_jvm_options = self._coverage.extra_jvm_options
classpath_prepend = self._coverage.classpath_prepend
classpath_append = self._coverage.classpath_append
else:
extra_jvm_options = []
classpath_prepend = ()
classpath_append = ()
tests_by_properties = self._tests_by_properties(tests_to_targets,
self._infer_workdir,
lambda target: target.test_platform)
# the below will be None if not set, and we'll default back to runtime_classpath
classpath_product = self.context.products.get_data('instrument_classpath')
result = 0
for (workdir, platform), tests in tests_by_properties.items():
for (target_jvm_options, target_tests) in self._partition_by_jvm_options(tests_to_targets,
tests):
for batch in self._partition(target_tests):
# Batches of test classes will likely exist within the same targets: dedupe them.
relevant_targets = set(map(tests_to_targets.get, batch))
complete_classpath = OrderedSet()
complete_classpath.update(classpath_prepend)
complete_classpath.update(self.tool_classpath('junit'))
complete_classpath.update(self.classpath(relevant_targets,
classpath_product=classpath_product))
complete_classpath.update(classpath_append)
distribution = self.preferred_jvm_distribution([platform])
with binary_util.safe_args(batch, self.get_options()) as batch_tests:
self.context.log.debug('CWD = {}'.format(workdir))
self.context.log.debug('platform = {}'.format(platform))
self._executor = SubprocessExecutor(distribution)
result += abs(distribution.execute_java(
executor=self._executor,
classpath=complete_classpath,
main=JUnitRun._MAIN,
jvm_options=self.jvm_options + extra_jvm_options + target_jvm_options,
args=self._args + batch_tests + [u'-xmlreport'],
workunit_factory=self.context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnitLabel.TEST],
cwd=workdir,
synthetic_jar_dir=self.workdir,
))
if result != 0 and self._fail_fast:
break
if result != 0:
failed_targets_and_tests = self._get_failed_targets(tests_to_targets)
failed_targets = sorted(failed_targets_and_tests, key=lambda target: target.address.spec)
error_message_lines = []
if self._failure_summary:
for target in failed_targets:
error_message_lines.append('\n{0}{1}'.format(' '*4, target.address.spec))
for test in sorted(failed_targets_and_tests[target]):
error_message_lines.append('{0}{1}'.format(' '*8, test))
error_message_lines.append(
'\njava {main} ... exited non-zero ({code}); {failed} failed {targets}.'
.format(main=JUnitRun._MAIN, code=result, failed=len(failed_targets),
targets=pluralize(len(failed_targets), 'target'))
)
raise TestFailedTaskError('\n'.join(error_message_lines), failed_targets=list(failed_targets))
def _infer_workdir(self, target):
if target.cwd is not None:
return target.cwd
return self._working_dir
def _tests_by_property(self, tests_to_targets, get_property):
properties = defaultdict(OrderedSet)
for test, target in tests_to_targets.items():
properties[get_property(target)].add(test)
return {property: list(tests) for property, tests in properties.items()}
def _tests_by_properties(self, tests_to_targets, *properties):
def combined_property(target):
return tuple(prop(target) for prop in properties)
return self._tests_by_property(tests_to_targets, combined_property)
def _partition_by_jvm_options(self, tests_to_targets, tests):
"""Partitions a list of tests by the jvm options to run them with.
:param dict tests_to_targets: A mapping from each test to its target.
:param list tests: The list of tests to run.
:returns: A list of tuples where the first element is an array of jvm options and the second
is a list of tests to run with the jvm options. Each test in tests will appear in exactly
one one tuple.
"""
jvm_options_to_tests = defaultdict(list)
for test in tests:
extra_jvm_options = tests_to_targets[test].payload.extra_jvm_options
jvm_options_to_tests[extra_jvm_options].append(test)
return [(list(jvm_options), tests) for jvm_options, tests in jvm_options_to_tests.items()]
def _partition(self, tests):
stride = min(self._batch_size, len(tests))
for i in range(0, len(tests), stride):
yield tests[i:i + stride]
def _get_tests_to_run(self):
for test_spec in self._tests_to_run:
src_spec, cls_spec = interpret_test_spec(test_spec)
if src_spec:
sourcefile, methodname = src_spec
for classname in self._classnames_from_source_file(sourcefile):
# Tack the methodname onto all classes in the source file, as we
# can't know which method the user intended.
yield classname + methodname
else:
classname, methodname = cls_spec
yield classname + methodname
def _calculate_tests_from_targets(self, targets):
"""
:param list targets: list of targets to calculate test classes for.
generates tuples (class_name, target).
"""
classpath_products = self.context.products.get_data('runtime_classpath')
for target in targets:
contents = ClasspathUtil.classpath_contents((target,), classpath_products, confs=self.confs)
for f in contents:
classname = ClasspathUtil.classname_for_rel_classfile(f)
if classname:
yield (classname, target)
def _classnames_from_source_file(self, srcfile):
relsrc = os.path.relpath(srcfile, get_buildroot())
source_products = self.context.products.get_data('classes_by_source').get(relsrc)
if not source_products:
# It's valid - if questionable - to have a source file with no classes when, for
# example, the source file has all its code commented out.
self.context.log.warn('Source file {0} generated no classes'.format(srcfile))
else:
for _, classes in source_products.rel_paths():
for cls in classes:
yield _classfile_to_classname(cls)
def _test_target_filter(self):
def target_filter(target):
return isinstance(target, junit_tests)
return target_filter
def _validate_target(self, target):
# TODO: move this check to an optional phase in goal_runner, so
# that missing sources can be detected early.
if not target.payload.sources.source_paths and not self.get_options().allow_empty_sources:
msg = 'JavaTests target must include a non-empty set of sources.'
raise TargetDefinitionException(target, msg)
def _timeout_abort_handler(self):
"""Kills the test run."""
# TODO(sameerbrenn): When we refactor the test code to be more standardized, rather than
# storing the process handle here, the test mixin class will call the start_test() fn
# on the language specific class which will return an object that can kill/monitor/etc
# the test process.
if self._executor is not None:
self._executor.kill()
def _execute(self, targets):
"""
Implements the primary junit test execution. This method is called by the TestRunnerTaskMixin,
which contains the primary Task.execute function and wraps this method in timeouts.
"""
# We only run tests within java_tests/junit_tests targets.
#
# But if coverage options are specified, we want to instrument
# and report on all the original targets, not just the test targets.
#
# We've already filtered out the non-test targets in the
# TestRunnerTaskMixin, so the mixin passes to us both the test
# targets and the unfiltered list of targets
tests_and_targets = self._collect_test_targets(self._get_test_targets())
if not tests_and_targets:
return
bootstrapped_cp = self.tool_classpath('junit')
def compute_complete_classpath():
return self.classpath(targets)
self.context.release_lock()
if self._coverage:
self._coverage.instrument(
targets, tests_and_targets.keys(), compute_complete_classpath, self.execute_java_for_targets)
def _do_report(exception=None):
if self._coverage:
self._coverage.report(
targets, tests_and_targets.keys(), self.execute_java_for_targets, tests_failed_exception=exception)
try:
self._run_tests(tests_and_targets)
_do_report(exception=None)
except TaskError as e:
_do_report(exception=e)
raise
