def execute(self, targets):
# The repl session may last a while, allow concurrent pants activity during this pants idle
# period.
tools_classpath = self._jvm_tool_bootstrapper.get_jvm_tool_classpath(self._bootstrap_key)
self.context.lock.release()
self.save_stty_options()
classpath = self.classpath(tools_classpath,
confs=self.confs,
exclusives_classpath=self.get_base_classpath_for_target(targets[0]))
print('') # Start REPL output on a new line.
try:
execute_java(classpath=classpath,
main=self.main,
jvm_options=self.jvm_args,
args=self.args,
workunit_factory=self.context.new_workunit,
workunit_name='repl',
workunit_labels=[WorkUnit.REPL, WorkUnit.JVM])
except KeyboardInterrupt:
# TODO(John Sirois): Confirm with Steve Gury that finally does not work on mac and an
# explicit catch of KeyboardInterrupt is required.
pass
self.restore_ssty_options()
def execute(self):
# The repl session may last a while, allow concurrent pants activity during this pants idle
# period.
tools_classpath = self.tool_classpath(self._bootstrap_key)
self.context.lock.release()
self.save_stty_options()
targets = self.context.targets()
classpath = self.classpath(tools_classpath,
confs=self.confs,
exclusives_classpath=self.get_base_classpath_for_target(targets[0]))
print('') # Start REPL output on a new line.
try:
# NOTE: We execute with no workunit, as capturing REPL output makes it very sluggish.
execute_java(classpath=classpath,
main=self.main,
jvm_options=self.jvm_args,
args=self.args)
except KeyboardInterrupt:
# TODO(John Sirois): Confirm with Steve Gury that finally does not work on mac and an
# explicit catch of KeyboardInterrupt is required.
pass
self.restore_ssty_options()
def execute(self):
(accept_predicate, reject_predicate) = Target.lang_discriminator('java')
targets = self.require_homogeneous_targets(accept_predicate, reject_predicate)
if targets:
tools_classpath = self.tool_classpath('scala-repl')
self.context.release_lock()
with preserve_stty_settings():
classpath = self.classpath(targets, cp=tools_classpath)
# The scala repl requires -Dscala.usejavacp=true since Scala 2.8 when launching in the way
# we do here (not passing -classpath as a program arg to scala.tools.nsc.MainGenericRunner).
jvm_options = self.jvm_options
if not any(opt.startswith('-Dscala.usejavacp=') for opt in jvm_options):
jvm_options.append('-Dscala.usejavacp=true')
print('') # Start REPL output on a new line.
try:
# NOTE: We execute with no workunit, as capturing REPL output makes it very sluggish.
execute_java(classpath=classpath,
main=self.get_options().main,
jvm_options=jvm_options,
args=self.args)
except KeyboardInterrupt:
# TODO(John Sirois): Confirm with Steve Gury that finally does not work on mac and an
# explicit catch of KeyboardInterrupt is required.
pass
def test_execute_java_argument_list_too_long_still_fail_still_too_long(self):
with self.mock_bundled_classpath_helper(argument_list_too_long=True) as mock_bundled_classpath:
mock_bundled_classpath.side_effect = fake_bundled_classpath
# very unlikely but still may fail with TOO_BIG error
self.runner.run.side_effect = [self.TOO_BIG, self.TOO_BIG]
with self.assertRaises(type(self.TOO_BIG)):
execute_java(self.TEST_CLASSPATH, self.TEST_MAIN, executor=self.executor)
def runjava(self, classpath, main, jvm_options=None, args=None, workunit_name=None,
workunit_labels=None, workunit_log_config=None):
"""Runs the java main using the given classpath and args.
If --no-use-nailgun is specified then the java main is run in a freshly spawned subprocess,
otherwise a persistent nailgun server dedicated to this Task subclass is used to speed up
amortized run times.
:API: public
"""
executor = self.create_java_executor()
# Creating synthetic jar to work around system arg length limit is not necessary
# when `NailgunExecutor` is used because args are passed through socket, therefore turning off
# creating synthetic jar if nailgun is used.
create_synthetic_jar = not self.get_options().use_nailgun
try:
return util.execute_java(classpath=classpath,
main=main,
jvm_options=jvm_options,
args=args,
executor=executor,
workunit_factory=self.context.new_workunit,
workunit_name=workunit_name,
workunit_labels=workunit_labels,
workunit_log_config=workunit_log_config,
create_synthetic_jar=create_synthetic_jar,
synthetic_jar_dir=self._executor_workdir)
except executor.Error as e:
raise TaskError(e)
def execute(self):
targets = self.context.targets()
if not any(isinstance(t, Benchmark) for t in targets):
raise TaskError('No jvm targets specified for benchmarking.')
# For rewriting JDK classes to work, the JAR file has to be listed specifically in
# the JAR manifest as something that goes in the bootclasspath.
# The MANIFEST list a jar 'allocation.jar' this is why we have to rename it
agent_tools_classpath = self.tool_classpath('benchmark-agent')
agent_jar = agent_tools_classpath[0]
allocation_jar = os.path.join(os.path.dirname(agent_jar),
"allocation.jar")
# TODO(Steve Gury): Find a solution to avoid copying the jar every run and being resilient
# to version upgrade
shutil.copyfile(agent_jar, allocation_jar)
os.environ['ALLOCATION_JAR'] = str(allocation_jar)
benchmark_tools_classpath = self.tool_classpath('benchmark-tool')
classpath = self.classpath(targets, benchmark_tools_classpath)
exit_code = execute_java(classpath=classpath,
main=self._CALIPER_MAIN,
jvm_options=self.jvm_options,
args=self.args,
workunit_factory=self.context.new_workunit,
workunit_name='caliper',
workunit_labels=[WorkUnitLabel.RUN])
if exit_code != 0:
raise TaskError('java {} ... exited non-zero ({})'.format(
self._CALIPER_MAIN, exit_code))
def runjava(self,
classpath,
main,
jvm_options=None,
args=None,
workunit_name=None,
workunit_labels=None):
"""Runs the java main using the given classpath and args.
If --no-ng-daemons is specified then the java main is run in a freshly spawned subprocess,
otherwise a persistent nailgun server dedicated to this Task subclass is used to speed up
amortized run times.
"""
executor = self.create_java_executor()
try:
return util.execute_java(
classpath=classpath,
main=main,
jvm_options=jvm_options,
args=args,
executor=executor,
workunit_factory=self.context.new_workunit,
workunit_name=workunit_name,
workunit_labels=workunit_labels)
except executor.Error as e:
raise TaskError(e)
def execute(self):
# For rewriting JDK classes to work, the JAR file has to be listed specifically in
# the JAR manifest as something that goes in the bootclasspath.
# The MANIFEST list a jar 'allocation.jar' this is why we have to rename it
agent_tools_classpath = self.tool_classpath('benchmark-agent')
agent_jar = agent_tools_classpath[0]
allocation_jar = os.path.join(os.path.dirname(agent_jar),
"allocation.jar")
# TODO(Steve Gury): Find a solution to avoid copying the jar every run and being resilient
# to version upgrade
shutil.copyfile(agent_jar, allocation_jar)
os.environ['ALLOCATION_JAR'] = str(allocation_jar)
benchmark_tools_classpath = self.tool_classpath('benchmark-tool')
classpath = self.classpath(self.context.targets(),
benchmark_tools_classpath,
confs=self.confs)
caliper_main = 'com.google.caliper.Runner'
exit_code = execute_java(classpath=classpath,
main=caliper_main,
jvm_options=self.jvm_options,
args=self.args,
workunit_factory=self.context.new_workunit,
workunit_name='caliper')
if exit_code != 0:
raise TaskError('java %s ... exited non-zero (%i)' %
(caliper_main, exit_code))
def execute(self):
targets = self.context.targets(predicate=self._is_benchmark)
if not targets:
raise TaskError('No jvm targets specified for benchmarking.')
# For rewriting JDK classes to work, the JAR file has to be listed specifically in
# the JAR manifest as something that goes in the bootclasspath.
# The MANIFEST list a jar 'allocation.jar' this is why we have to rename it
agent_tools_classpath = self.tool_classpath('benchmark-agent')
agent_jar = agent_tools_classpath[0]
allocation_jar = os.path.join(os.path.dirname(agent_jar), "allocation.jar")
# TODO(Steve Gury): Find a solution to avoid copying the jar every run and being resilient
# to version upgrade
shutil.copyfile(agent_jar, allocation_jar)
os.environ['ALLOCATION_JAR'] = str(allocation_jar)
benchmark_tools_classpath = self.tool_classpath('benchmark-tool')
# Collect a transitive classpath for the benchmark targets.
classpath = self.classpath(targets, benchmark_tools_classpath)
java_executor = SubprocessExecutor(DistributionLocator.cached())
exit_code = execute_java(classpath=classpath,
main=self._CALIPER_MAIN,
jvm_options=self.jvm_options,
args=self.args,
workunit_factory=self.context.new_workunit,
workunit_name='caliper',
workunit_labels=[WorkUnitLabel.RUN],
executor=java_executor,
create_synthetic_jar=self.synthetic_classpath)
if exit_code != 0:
raise TaskError(f'java {self._CALIPER_MAIN} ... exited non-zero ({exit_code})')
def instrument(self, targets, tests, compute_junit_classpath):
junit_classpath = compute_junit_classpath()
safe_mkdir(self._coverage_instrument_dir, clean=True)
self._emma_classpath = self._task_exports.tool_classpath('emma')
with binary_util.safe_args(self.get_coverage_patterns(targets),
self._task_exports.task_options) as patterns:
args = [
'instr',
'-out', self._coverage_metadata_file,
'-d', self._coverage_instrument_dir,
'-cp', os.pathsep.join(junit_classpath),
'-exit'
]
for pattern in patterns:
args.extend(['-filter', pattern])
main = 'emma'
result = execute_java(classpath=self._emma_classpath,
main=main,
jvm_options=self._coverage_jvm_options,
args=args,
workunit_factory=self._context.new_workunit,
workunit_name='emma-instrument')
if result != 0:
raise TaskError("java {0} ... exited non-zero ({1})"
" 'failed to instrument'".format(main, result))
def execute_codegen(self, targets):
for target in targets:
args = ['-o', self.codegen_workdir(target)]
compiler = target.compiler
if compiler == 'antlr3':
if target.package is not None:
logger.warn(
"The 'package' attribute is not supported for antlr3 and will be ignored."
)
java_main = 'org.antlr.Tool'
elif compiler == 'antlr4':
args.append('-visitor') # Generate Parse Tree Visitor As Well
# Note that this assumes that there is no package set in the antlr file itself,
# which is considered an ANTLR best practice.
args.append('-package')
if target.package is None:
args.append(self._get_sources_package(target))
else:
args.append(target.package)
java_main = 'org.antlr.v4.Tool'
antlr_classpath = self.tool_classpath(compiler)
sources = self._calculate_sources([target])
args.extend(sources)
result = util.execute_java(classpath=antlr_classpath,
main=java_main,
args=args,
workunit_name='antlr')
if result != 0:
raise TaskError('java {} ... exited non-zero ({})'.format(
java_main, result))
if compiler == 'antlr3':
for source in list(self.codegen_strategy.find_sources(target)):
self._scrub_generated_timestamp(source)
def test_execute_java_no_error_weird_path(self):
"""
:API: public
"""
with temporary_file(suffix=".jar") as temp_path:
fetcher = Fetcher(get_buildroot())
try:
# Download a jar that echoes things.
fetcher.download(
"https://maven-central.storage-download.googleapis.com/repos/central/data/io/get-coursier/echo/1.0.0/echo-1.0.0.jar",
path_or_fd=temp_path.name,
timeout_secs=2,
)
except fetcher.Error:
self.fail("fail to download echo jar")
task = self.execute(self.context([]))
executor = task.create_java_executor()
# Executing the jar as is should work.
self.assertEqual(
0,
util.execute_java(
executor=executor,
classpath=[temp_path.name],
main="coursier.echo.Echo",
args=["Hello World"],
create_synthetic_jar=True,
),
)
# Rename the jar to contain reserved characters.
new_path = os.path.join(os.path.dirname(temp_path.name),
"%%!!!===++.jar")
safe_concurrent_rename(temp_path.name, new_path)
# Executing the new path should work.
self.assertEqual(
0,
util.execute_java(
executor=executor,
classpath=[new_path],
main="coursier.echo.Echo",
args=["Hello World"],
create_synthetic_jar=True,
),
)
def execute_codegen(self, targets):
# Invoke the generator once per target. Because the wire compiler has flags that try to reduce
# the amount of code emitted, Invoking them all together will break if one target specifies a
# service_writer and another does not, or if one specifies roots and another does not.
for target in targets:
sources_by_base = self._calculate_sources([target])
if self.codegen_strategy.name() == 'isolated':
sources = OrderedSet(target.sources_relative_to_buildroot())
else:
sources = OrderedSet(
itertools.chain.from_iterable(sources_by_base.values()))
if not self.validate_sources_present(sources, [target]):
continue
relative_sources = OrderedSet()
for source in sources:
source_root = SourceRoot.find_by_path(source)
if not source_root:
source_root = SourceRoot.find(target)
relative_source = os.path.relpath(source, source_root)
relative_sources.add(relative_source)
check_duplicate_conflicting_protos(self, sources_by_base,
relative_sources,
self.context.log)
args = ['--java_out={0}'.format(self.codegen_workdir(target))]
# Add all params in payload to args
if target.payload.get_field_value('no_options'):
args.append('--no_options')
service_writer = target.payload.service_writer
if service_writer:
args.append('--service_writer={0}'.format(service_writer))
registry_class = target.payload.registry_class
if registry_class:
args.append('--registry_class={0}'.format(registry_class))
if target.payload.roots:
args.append('--roots={0}'.format(','.join(
target.payload.roots)))
if target.payload.enum_options:
args.append('--enum_options={0}'.format(','.join(
target.payload.enum_options)))
args.append('--proto_path={0}'.format(
os.path.join(get_buildroot(), SourceRoot.find(target))))
args.extend(relative_sources)
result = util.execute_java(
classpath=self.tool_classpath('wire-compiler'),
main='com.squareup.wire.WireCompiler',
args=args)
if result != 0:
raise TaskError(
'Wire compiler exited non-zero ({0})'.format(result))
def report(self, targets, tests, tests_failed_exception=None):
if tests_failed_exception:
self._context.log.warn('Test failed: {0}'.format(
str(tests_failed_exception)))
if self._coverage_force:
self._context.log.warn(
'Generating report even though tests failed')
else:
return
args = [
'report', '-in', self._coverage_metadata_file, '-in',
self._coverage_file, '-exit'
]
source_bases = set()
def collect_source_base(target):
if self.is_coverage_target(target):
source_bases.add(target.target_base)
for target in self._test_target_candidates(targets):
target.walk(collect_source_base)
for source_base in source_bases:
args.extend(['-sp', source_base])
sorting = ['-Dreport.sort', '+name,+class,+method,+block']
if self._coverage_report_console:
args.extend([
'-r', 'txt', '-Dreport.txt.out.file={0}'.format(
self._coverage_console_file)
] + sorting)
if self._coverage_report_xml:
args.extend([
'-r', 'xml',
'-Dreport.xml.out.file={0}'.format(self._coverage_xml_file)
])
if self._coverage_report_html:
args.extend([
'-r', 'html', '-Dreport.html.out.file={0}'.format(
self._coverage_html_file), '-Dreport.out.encoding=UTF-8'
] + sorting)
main = 'emma'
result = execute_java(classpath=self._emma_classpath,
main=main,
jvm_options=self._coverage_jvm_options,
args=args,
workunit_factory=self._context.new_workunit,
workunit_name='emma-report')
if result != 0:
raise TaskError(
"java {0} ... exited non-zero ({1})"
" 'failed to generate code coverage reports'".format(
main, result))
if self._coverage_report_console:
with safe_open(self._coverage_console_file) as console_report:
sys.stdout.write(console_report.read())
if self._coverage_report_html_open:
binary_util.ui_open(self._coverage_html_file)
def test_execute_java_no_synthentic_jar(self):
with self.mock_safe_classpath_helper(create_synthetic_jar=False):
self.assertEquals(
0,
execute_java(self.TEST_CLASSPATH,
self.TEST_MAIN,
executor=self.executor,
create_synthetic_jar=False))
def test_execute_java_no_error(self):
with self.mock_safe_classpath_helper():
self.assertEquals(
0,
execute_java(self.TEST_CLASSPATH,
self.TEST_MAIN,
executor=self.executor,
synthetic_jar_dir=self.SYNTHETIC_JAR_DIR))
def instrument(self, targets, tests, compute_junit_classpath):
junit_classpath = compute_junit_classpath()
cobertura_cp = self._task_exports.tool_classpath('cobertura-instrument')
aux_classpath = os.pathsep.join(relativize_paths(junit_classpath, get_buildroot()))
safe_delete(self._coverage_datafile)
classes_by_target = self._context.products.get_data('classes_by_target')
for target in targets:
if self.is_coverage_target(target):
classes_by_rootdir = classes_by_target.get(target)
if classes_by_rootdir:
for root, products in classes_by_rootdir.rel_paths():
self._rootdirs[root].update(products)
# Cobertura uses regular expressions for filters, and even then there are still problems
# with filtering. It turned out to be easier to just select which classes to instrument
# by filtering them here.
# TODO(ji): Investigate again how we can use cobertura's own filtering mechanisms.
if self._coverage_filters:
for basedir, classes in self._rootdirs.items():
updated_classes = []
for cls in classes:
does_match = False
for positive_filter in self._include_filters:
if fnmatch.fnmatchcase(_classfile_to_classname(cls), positive_filter):
does_match = True
for negative_filter in self._exclude_filters:
if fnmatch.fnmatchcase(_classfile_to_classname(cls), negative_filter):
does_match = False
if does_match:
updated_classes.append(cls)
self._rootdirs[basedir] = updated_classes
for basedir, classes in self._rootdirs.items():
if not classes:
continue # No point in running instrumentation if there is nothing to instrument!
self._nothing_to_instrument = False
args = [
'--basedir',
basedir,
'--datafile',
self._coverage_datafile,
'--auxClasspath',
aux_classpath,
]
with temporary_file_path(cleanup=False) as instrumented_classes_file:
with file(instrumented_classes_file, 'wb') as icf:
icf.write(('\n'.join(classes) + '\n').encode('utf-8'))
self._context.log.debug('instrumented classes in {0}'.format(instrumented_classes_file))
args.append('--listOfFilesToInstrument')
args.append(instrumented_classes_file)
main = 'net.sourceforge.cobertura.instrument.InstrumentMain'
result = execute_java(classpath=cobertura_cp,
main=main,
jvm_options=self._coverage_jvm_options,
args=args,
workunit_factory=self._context.new_workunit,
workunit_name='cobertura-instrument')
if result != 0:
raise TaskError("java {0} ... exited non-zero ({1})"
" 'failed to instrument'".format(main, result))
def execute(self):
# The called binary may block for a while, allow concurrent pants activity during this pants
# idle period.
#
# TODO(John Sirois): refactor lock so that I can do:
# with self.context.lock.yield():
# - blocking code
#
# Currently re-acquiring the lock requires a path argument that was set up by the goal
# execution engine. I do not want task code to learn the lock location.
# http://jira.local.twitter.com/browse/AWESOME-1317
target = self.require_single_root_target()
working_dir = None
cwd_opt = self.get_options().cwd
if cwd_opt != _CWD_NOT_PRESENT:
working_dir = self.get_options().cwd
if not working_dir:
working_dir = target.address.spec_path
logger.debug("Working dir is {0}".format(working_dir))
if isinstance(target, JvmApp):
binary = target.binary
else:
binary = target
# We can't throw if binary isn't a JvmBinary, because perhaps we were called on a
# python_binary, in which case we have to no-op and let python_run do its thing.
# TODO(benjy): Some more elegant way to coordinate how tasks claim targets.
if isinstance(binary, JvmBinary):
executor = CommandLineGrabber(
) if self.only_write_cmd_line else None
self.context.release_lock()
exclusives_classpath = self.get_base_classpath_for_target(binary)
result = execute_java(
classpath=(self.classpath(
confs=self.confs,
exclusives_classpath=exclusives_classpath)),
main=binary.main,
executor=executor,
jvm_options=self.jvm_options,
args=self.args,
workunit_factory=self.context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnit.RUN],
cwd=working_dir,
)
if self.only_write_cmd_line:
with safe_open(expand_path(self.only_write_cmd_line),
'w') as outfile:
outfile.write(' '.join(executor.cmd))
elif result != 0:
raise TaskError('java %s ... exited non-zero (%i)' %
(binary.main, result),
exit_code=result)
def instrument(self, targets, tests, junit_classpath):
self._cobertura_classpath = self._task_exports.tool_classpath(
'cobertura')
safe_delete(self._coverage_datafile)
classes_by_target = self._context.products.get_data(
'classes_by_target')
for target in targets:
if self.is_coverage_target(target):
classes_by_rootdir = classes_by_target.get(target)
if classes_by_rootdir:
for root, products in classes_by_rootdir.rel_paths():
self._rootdirs[root].update(products)
# Cobertura uses regular expressions for filters, and even then there are still problems
# with filtering. It turned out to be easier to just select which classes to instrument
# by filtering them here.
# TODO(ji): Investigate again how we can use cobertura's own filtering mechanisms.
if self._coverage_filters:
for basedir, classes in self._rootdirs.items():
updated_classes = []
for cls in classes:
does_match = False
for positive_filter in self._include_filters:
if fnmatch.fnmatchcase(_classfile_to_classname(cls),
positive_filter):
does_match = True
for negative_filter in self._exclude_filters:
if fnmatch.fnmatchcase(_classfile_to_classname(cls),
negative_filter):
does_match = False
if does_match:
updated_classes.append(cls)
self._rootdirs[basedir] = updated_classes
for basedir, classes in self._rootdirs.items():
if not classes:
continue # No point in running instrumentation if there is nothing to instrument!
args = [
'--basedir',
basedir,
'--datafile',
self._coverage_datafile,
]
with temporary_file() as fd:
fd.write('\n'.join(classes) + '\n')
args.append('--listOfFilesToInstrument')
args.append(fd.name)
main = 'net.sourceforge.cobertura.instrument.InstrumentMain'
result = execute_java(
classpath=self._cobertura_classpath + junit_classpath,
main=main,
args=args,
workunit_factory=self._context.new_workunit,
workunit_name='cobertura-instrument')
if result != 0:
raise TaskError("java %s ... exited non-zero (%i)"
" 'failed to instrument'" % (main, result))
def report(self, targets, tests, junit_classpath):
emma_classpath = self._task_exports.tool_classpath(
self._emma_bootstrap_key)
args = [
'report', '-in', self._coverage_metadata_file, '-in',
self._coverage_file, '-exit'
]
source_bases = set()
def collect_source_base(target):
if self.is_coverage_target(target):
source_bases.add(target.target_base)
for target in self._test_target_candidates(targets):
target.walk(collect_source_base)
for source_base in source_bases:
args.extend(['-sp', source_base])
sorting = ['-Dreport.sort', '+name,+class,+method,+block']
if self._coverage_report_console:
args.extend([
'-r', 'txt',
'-Dreport.txt.out.file=%s' % self._coverage_console_file
] + sorting)
if self._coverage_report_xml:
args.extend([
'-r', 'xml',
'-Dreport.xml.out.file=%s' % self._coverage_xml_file
])
if self._coverage_report_html:
args.extend([
'-r', 'html',
'-Dreport.html.out.file=%s' %
self._coverage_html_file, '-Dreport.out.encoding=UTF-8'
] + sorting)
main = 'emma'
result = execute_java(classpath=emma_classpath,
main=main,
args=args,
workunit_factory=self._context.new_workunit,
workunit_name='emma-report')
if result != 0:
raise TaskError("java %s ... exited non-zero (%i)"
" 'failed to generate code coverage reports'" %
(main, result))
if self._coverage_report_console:
with safe_open(self._coverage_console_file) as console_report:
sys.stdout.write(console_report.read())
if self._coverage_report_html_open:
binary_util.ui_open(self._coverage_html_file)
def execute(self):
(accept_predicate,
reject_predicate) = Target.lang_discriminator('java')
targets = self.require_homogeneous_targets(accept_predicate,
reject_predicate)
if targets:
tools_classpath = self.tool_classpath('scala-repl')
self.context.release_lock()
with preserve_stty_settings():
classpath = self.classpath(tools_classpath, confs=self.confs)
print('') # Start REPL output on a new line.
try:
# NOTE: We execute with no workunit, as capturing REPL output makes it very sluggish.
execute_java(classpath=classpath,
main=self.get_options().main,
jvm_options=self.jvm_options,
args=self.args)
except KeyboardInterrupt:
# TODO(John Sirois): Confirm with Steve Gury that finally does not work on mac and an
# explicit catch of KeyboardInterrupt is required.
pass
def _run_tests(self, tests_to_targets, main, extra_jvm_options=None, classpath_prepend=(),
classpath_append=()):
extra_jvm_options = extra_jvm_options or []
tests_by_properties = self._tests_by_properties(tests_to_targets,
self._infer_workdir,
lambda target: target.test_platform)
result = 0
for (workdir, platform), tests in tests_by_properties.items():
for batch in self._partition(tests):
# Batches of test classes will likely exist within the same targets: dedupe them.
relevant_targets = set(map(tests_to_targets.get, batch))
classpath = self._task_exports.classpath(relevant_targets,
cp=self._task_exports.tool_classpath('junit'))
complete_classpath = OrderedSet()
complete_classpath.update(classpath_prepend)
complete_classpath.update(classpath)
complete_classpath.update(classpath_append)
if self._strict_jvm_version:
max_version = Revision(*(platform.target_level.components + [9999]))
distribution = Distribution.cached(minimum_version=platform.target_level,
maximum_version=max_version)
else:
distribution = Distribution.cached(minimum_version=platform.target_level)
with binary_util.safe_args(batch, self._task_exports.task_options) as batch_tests:
self._context.log.debug('CWD = {}'.format(workdir))
self._context.log.debug('platform = {}'.format(platform))
result += abs(execute_java(
classpath=complete_classpath,
main=main,
jvm_options=self._task_exports.jvm_options + extra_jvm_options,
args=self._args + batch_tests + [u'-xmlreport'],
workunit_factory=self._context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnitLabel.TEST],
cwd=workdir,
distribution=distribution,
))
if result != 0 and self._fail_fast:
break
if result != 0:
failed_targets = self._get_failed_targets(tests_to_targets)
raise TestFailedTaskError(
'java {0} ... exited non-zero ({1}); {2} failed targets.'
.format(main, result, len(failed_targets)),
failed_targets=failed_targets
)
def _run_tests(self,
tests_to_targets,
main,
extra_jvm_options=None,
classpath_prepend=(),
classpath_append=()):
extra_jvm_options = extra_jvm_options or []
result = 0
for workdir, tests in self._tests_by_workdir(tests_to_targets).items():
for batch in self._partition(tests):
classpath = self._task_exports.classpath(
map(tests_to_targets.get, batch),
cp=self._task_exports.tool_classpath('junit'))
complete_classpath = OrderedSet()
complete_classpath.update(classpath_prepend)
complete_classpath.update(classpath)
complete_classpath.update(classpath_append)
with binary_util.safe_args(
batch, self._task_exports.task_options) as batch_tests:
self._context.log.debug('CWD = {}'.format(workdir))
result += abs(
execute_java(
classpath=complete_classpath,
main=main,
jvm_options=self._task_exports.jvm_options +
extra_jvm_options,
args=self._args + batch_tests + [u'-xmlreport'],
workunit_factory=self._context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnit.TEST],
cwd=workdir,
))
if result != 0 and self._fail_fast:
break
if result != 0:
failed_targets = self._get_failed_targets(tests_to_targets)
raise TestFailedTaskError(
'java {0} ... exited non-zero ({1}); {2} failed targets.'.
format(main, result, len(failed_targets)),
failed_targets=failed_targets)
def run_tests(tests):
args = ['--color'] if self.color else []
args.append('--specs=%s' % ','.join(tests))
specs_runner_main = 'com.twitter.common.testing.ExplicitSpecsRunnerMain'
bootstrapped_cp = self.tool_classpath('specs')
classpath = self.classpath(bootstrapped_cp, confs=self.confs)
result = execute_java(
classpath=classpath,
main=specs_runner_main,
jvm_options=self.jvm_options,
args=self.args + args,
workunit_factory=self.context.new_workunit,
workunit_name='specs',
workunit_labels=[WorkUnit.TEST])
if result != 0:
raise TaskError('java %s ... exited non-zero (%i)' %
(specs_runner_main, result))
def instrument(self, targets, tests, junit_classpath):
safe_mkdir(self._coverage_instrument_dir, clean=True)
emma_classpath = self._task_exports.tool_classpath(self._emma_bootstrap_key)
with binary_util.safe_args(self.get_coverage_patterns(targets)) as patterns:
args = [
'instr',
'-out', self._coverage_metadata_file,
'-d', self._coverage_instrument_dir,
'-cp', os.pathsep.join(junit_classpath),
'-exit'
]
for pattern in patterns:
args.extend(['-filter', pattern])
main = 'emma'
result = execute_java(classpath=emma_classpath, main=main, args=args,
workunit_factory=self._context.new_workunit,
workunit_name='emma-instrument')
if result != 0:
raise TaskError("java %s ... exited non-zero (%i)"
" 'failed to instrument'" % (main, result))
def _run_tests(self, tests, classpath, main, jvm_args=None):
# TODO(John Sirois): Integrated batching with the test runner. As things stand we get
# results summaries for example for each batch but no overall summary.
# http://jira.local.twitter.com/browse/AWESOME-1114
result = 0
for batch in self._partition(tests):
with binary_util.safe_args(batch) as batch_tests:
result += abs(
execute_java(classpath=classpath,
main=main,
jvm_options=(jvm_args or []) + self._jvm_args,
args=self._opts + batch_tests,
workunit_factory=self._context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnit.TEST]))
if result != 0 and self._fail_fast:
break
if result != 0:
raise TaskError('java %s ... exited non-zero (%i)' %
(main, result))
def execute(self, targets):
# The called binary may block for a while, allow concurrent pants activity during this pants
# idle period.
#
# TODO(John Sirois): refactor lock so that I can do:
# with self.context.lock.yield():
# - blocking code
#
# Currently re-acquiring the lock requires a path argument that was set up by the goal
# execution engine. I do not want task code to learn the lock location.
# http://jira.local.twitter.com/browse/AWESOME-1317
self.context.lock.release()
# Run the first target that is a binary.
binaries = filter(is_binary, targets)
if len(binaries) > 0: # We only run the first one.
main = binaries[0].main
egroups = self.context.products.get_data('exclusives_groups')
group_key = egroups.get_group_key_for_target(binaries[0])
group_classpath = egroups.get_classpath_for_group(group_key)
executor = CommandLineGrabber() if self.only_write_cmd_line else None
result = execute_java(
classpath=(self.classpath(confs=self.confs, exclusives_classpath=group_classpath)),
main=main,
executor=executor,
jvm_options=self.jvm_args,
args=self.args,
workunit_factory=self.context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnit.RUN]
)
if self.only_write_cmd_line:
with safe_open(self.only_write_cmd_line, 'w') as outfile:
outfile.write(' '.join(executor.cmd))
elif result != 0:
raise TaskError('java %s ... exited non-zero (%i)' % (main, result), exit_code=result)
def _run_tests(self, tests, classpath, main, extra_jvm_options=None, cwd=None):
# TODO(John Sirois): Integrated batching with the test runner. As things stand we get
# results summaries for example for each batch but no overall summary.
# http://jira.local.twitter.com/browse/AWESOME-1114
extra_jvm_options = extra_jvm_options or []
result = 0
cwd = cwd or get_buildroot()
for batch in self._partition(tests):
with binary_util.safe_args(batch) as batch_tests:
result += abs(execute_java(
classpath=classpath,
main=main,
jvm_options=self._task_exports.jvm_options + extra_jvm_options,
args=self._args + batch_tests,
workunit_factory=self._context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnit.TEST],
cwd=cwd
))
if result != 0 and self._fail_fast:
break
if result != 0:
raise TaskError('java {0} ... exited non-zero ({1})'.format(main, result))
def report(self, targets, tests, junit_classpath):
# Link files in the real source tree to files named using the classname.
# Do not include class file names containing '$', as these will always have
# a corresponding $-less class file, and they all point back to the same
# source.
# Put all these links to sources under self._coverage_dir/src
all_classes = set()
for basedir, classes in self._rootdirs.items():
all_classes.update([cls for cls in classes if '$' not in cls])
sources_by_class = self._build_sources_by_class()
coverage_source_root_dir = os.path.join(self._coverage_dir, 'src')
safe_rmtree(coverage_source_root_dir)
for cls in all_classes:
source_file = sources_by_class.get(cls)
if source_file:
# the class in @cls
# (e.g., 'com/pants/example/hello/welcome/WelcomeEverybody.class')
# was compiled from the file in @source_file
# (e.g., 'src/scala/com/pants/example/hello/welcome/Welcome.scala')
# Note that, in the case of scala files, the path leading up to Welcome.scala does not
# have to match the path in the corresponding .class file AT ALL. In this example,
# @source_file could very well have been 'src/hello-kitty/Welcome.scala'.
# However, cobertura expects the class file path to match the corresponding source
# file path below the source base directory(ies) (passed as (a) positional argument(s)),
# while it still gets the source file basename from the .class file.
# Here we create a fake hierachy under coverage_dir/src to mimic what cobertura expects.
class_dir = os.path.dirname(
cls) # e.g., 'com/pants/example/hello/welcome'
fake_source_directory = os.path.join(coverage_source_root_dir,
class_dir)
safe_mkdir(fake_source_directory)
fake_source_file = os.path.join(fake_source_directory,
os.path.basename(source_file))
try:
os.symlink(
os.path.relpath(source_file, fake_source_directory),
fake_source_file)
except OSError as e:
# These warnings appear when source files contain multiple classes.
self._context.log.warn('Could not symlink %s to %s: %s' %
(source_file, fake_source_file, e))
else:
self._context.log.error(
'class %s does not exist in a source file!' % cls)
report_formats = []
if self._coverage_report_xml:
report_formats.append('xml')
if self._coverage_report_html:
report_formats.append('html')
for report_format in report_formats:
report_dir = os.path.join(self._coverage_dir, report_format)
safe_mkdir(report_dir, clean=True)
args = [
coverage_source_root_dir,
'--datafile',
self._coverage_datafile,
'--destination',
report_dir,
'--format',
report_format,
]
main = 'net.sourceforge.cobertura.reporting.ReportMain'
result = execute_java(classpath=self._cobertura_classpath,
main=main,
args=args,
workunit_factory=self._context.new_workunit,
workunit_name='cobertura-report-' +
report_format)
if result != 0:
raise TaskError("java %s ... exited non-zero (%i)"
" 'failed to report'" % (main, result))
def execute_java(self, *args, **kwargs): return execute_java(*args, distribution=self, **kwargs)
def execute(self):
exported_targets_map = self.generate_targets_map(
self.context.targets())
export_result = json.dumps(exported_targets_map,
indent=4,
separators=(',', ': '))
with temporary_dir() as tmpdir:
export_outfile = os.path.join(tmpdir, 'export-out.json')
with open(export_outfile, 'wb') as outf:
outf.write(export_result)
ensime_gen_jar = self.context.products.get_data(EnsimeGenJar)
ensime_gen_classpath = [ensime_gen_jar.tool_jar_path]
# TODO: use JvmPlatform for jvm options!
reported_scala_version = self.get_options().reported_scala_version
if not reported_scala_version:
reported_scala_version = self._scala_platform.version
zinc_compile_dir = os.path.join(self.get_options().pants_workdir,
'compile/zinc')
output_file = os.path.join(get_buildroot(),
self.get_options().output_file)
safe_mkdir_for(output_file)
# This is what we depend on in 3rdparty/jvm:ensime-server.
ensime_server_version = '2.0.1'
ensime_server_jars = self.tool_classpath_from_products(
self.context.products,
'ensime-server',
scope=self.options_scope)
scala_compiler_jars = self._scala_platform.compiler_classpath(
self.context.products)
argv = [
get_buildroot(),
reported_scala_version,
self._make_ensime_cache_dir(),
zinc_compile_dir,
output_file,
ensime_server_version,
]
env = {
'SCALAC_ARGS': json.dumps(self.get_options().scalac_options),
'JAVAC_ARGS': json.dumps(self.get_options().javac_options),
'ENSIME_SERVER_JARS_CLASSPATH': ':'.join(ensime_server_jars),
'SCALA_COMPILER_JARS_CLASSPATH': ':'.join(scala_compiler_jars),
}
with open(export_outfile, 'rb') as inf:
with environment_as(**env):
execute_java(ensime_gen_classpath,
'pingpong.ensime.EnsimeFileGen',
args=argv,
workunit_name='ensime-gen-invoke',
workunit_labels=[WorkUnitLabel.TOOL],
distribution=DistributionLocator.cached(),
stdin=inf)
def test_execute_java_no_error(self):
with self.mock_bundled_classpath_helper():
self.assertEquals(0, execute_java(self.TEST_CLASSPATH, self.TEST_MAIN,
executor=self.executor))
def test_execute_java_argument_list_too_long(self):
with self.mock_bundled_classpath_helper(argument_list_too_long=True) as mock_bundled_classpath:
mock_bundled_classpath.side_effect = fake_bundled_classpath
self.runner.run.side_effect = [self.TOO_BIG, 0]
self.assertEquals(0, execute_java(self.TEST_CLASSPATH, self.TEST_MAIN,
executor=self.executor))
def test_execute_java_other_error(self):
self.runner.run.side_effect = self.OTHER_ERROR
with self.mock_bundled_classpath_helper():
with self.assertRaises(type(self.OTHER_ERROR)):
execute_java(self.TEST_CLASSPATH, self.TEST_MAIN, executor=self.executor)
