def errorprone(self, target):
runtime_classpaths = self.context.products.get_data('runtime_classpath')
runtime_classpath = [jar for conf, jar in runtime_classpaths.get_for_targets(target.closure(bfs=True))]
output_dir = os.path.join(self.workdir, target.id)
safe_mkdir(output_dir)
runtime_classpath.append(output_dir)
# Try to run errorprone with the same java version as the target
# The minimum JDK for errorprone is JDK 1.8
min_jdk_version = max(target.platform.target_level, Revision.lenient('1.8'))
if min_jdk_version.components[0] == 1:
max_jdk_version = Revision(min_jdk_version.components[0], min_jdk_version.components[1], '9999')
else:
max_jdk_version = Revision(min_jdk_version.components[0], '9999')
self.set_distribution(minimum_version=min_jdk_version, maximum_version=max_jdk_version, jdk=True)
jvm_options = self.get_options().jvm_options[:]
if self.dist.version < Revision.lenient('9'):
# For Java 8 we need to add the errorprone javac jar to the bootclasspath to
# avoid the "java.lang.NoSuchFieldError: ANNOTATION_PROCESSOR_MODULE_PATH" error
# See https://github.com/google/error-prone/issues/653 for more information
jvm_options.extend(['-Xbootclasspath/p:{}'.format(self.tool_classpath('errorprone-javac')[0])])
args = [
'-d', output_dir,
]
# Errorprone does not recognize source or target 10 yet
if target.platform.source_level < Revision.lenient('10'):
args.extend(['-source', str(target.platform.source_level)])
if target.platform.target_level < Revision.lenient('10'):
args.extend(['-target', str(target.platform.target_level)])
errorprone_classpath_file = os.path.join(self.workdir, '{}.classpath'.format(os.path.basename(output_dir)))
with open(errorprone_classpath_file, 'w') as f:
f.write('-classpath ')
f.write(':'.join(runtime_classpath))
args.append('@{}'.format(errorprone_classpath_file))
for opt in self.get_options().command_line_options:
args.extend(safe_shlex_split(opt))
with argfile.safe_args(self.calculate_sources(target), self.get_options()) as batched_sources:
args.extend(batched_sources)
result = self.runjava(classpath=self.tool_classpath('errorprone'),
main=self._ERRORPRONE_MAIN,
jvm_options=jvm_options,
args=args,
workunit_name='errorprone',
workunit_labels=[WorkUnitLabel.LINT])
self.context.log.debug('java {main} ... exited with result ({result})'.format(
main=self._ERRORPRONE_MAIN, result=result))
return result
def _preferred_jvm_distribution_args(cls,
platforms,
strict=None,
jdk=False):
if not platforms:
return {"jdk": jdk}
if strict is False: # treat all the platforms as non-strict
min_version = max(p.target_level for p in platforms)
set_max_version = False
else:
# treat strict platforms as strict & ensure no non-strict directive is broken
strict_target_levels = {
p.target_level
for p in platforms if p.strict
}
lenient_target_levels = {
p.target_level
for p in platforms if not p.strict
}
if len(strict_target_levels) == 0:
min_version = max(lenient_target_levels)
set_max_version = strict
elif len(strict_target_levels) > 1:
differing = ", ".join(str(t) for t in strict_target_levels)
raise cls.IncompatiblePlatforms(
f"Multiple strict platforms with differing target releases were found:"
f" {differing}")
else:
if len(lenient_target_levels) == 0:
min_version = next(iter(strict_target_levels))
set_max_version = True
else:
strict_level = next(iter(strict_target_levels))
non_strict_max = max(t for t in lenient_target_levels)
if non_strict_max and non_strict_max > strict_level:
raise cls.IncompatiblePlatforms(
f"lenient platform with higher minimum version,"
f" {non_strict_max}, than strict requirement of"
f" {strict_level}")
min_version = strict_level
set_max_version = True
if len(min_version.components
) <= 2: # ensure at least three components.
min_version = Revision(*(min_version.components + [0] *
(3 - len(min_version.components))))
max_version = None
if set_max_version:
max_version = Revision(*(min_version.components[0:2] + [9999]))
return {
"minimum_version": min_version,
"maximum_version": max_version,
"jdk": jdk
}
def preferred_jvm_distribution(self, platforms):
"""Returns a jvm Distribution with a version that should work for all the platforms."""
if not platforms:
return DistributionLocator.cached()
min_version = max(platform.target_level for platform in platforms)
max_version = Revision(*(min_version.components + [9999])) if self._strict_jvm_version else None
return DistributionLocator.cached(minimum_version=min_version, maximum_version=max_version)
def test_preferred_jvm_distributions(self):
with temporary_dir() as strict_jdk_home:
with temporary_dir() as non_strict_jdk_home:
strict_cache_key = (Revision(1, 6), Revision(1, 6,
9999), False)
non_strict_cache_key = (Revision(1, 6), None, False)
DistributionLocator._CACHE[strict_cache_key] = Distribution(
home_path=strict_jdk_home)
DistributionLocator._CACHE[
non_strict_cache_key] = Distribution(
home_path=non_strict_jdk_home)
self.assertEqual(
{
'strict': strict_jdk_home,
'non_strict': non_strict_jdk_home
},
self.execute_export_json()['preferred_jvm_distributions']
['java6'])
def _run_tests(self, tests_to_targets, main, extra_jvm_options=None, classpath_prepend=(),
classpath_append=()):
extra_jvm_options = extra_jvm_options or []
tests_by_properties = self._tests_by_properties(tests_to_targets,
self._infer_workdir,
lambda target: target.test_platform)
result = 0
for (workdir, platform), tests in tests_by_properties.items():
for batch in self._partition(tests):
# Batches of test classes will likely exist within the same targets: dedupe them.
relevant_targets = set(map(tests_to_targets.get, batch))
classpath = self._task_exports.classpath(relevant_targets,
cp=self._task_exports.tool_classpath('junit'))
complete_classpath = OrderedSet()
complete_classpath.update(classpath_prepend)
complete_classpath.update(classpath)
complete_classpath.update(classpath_append)
if self._strict_jvm_version:
max_version = Revision(*(platform.target_level.components + [9999]))
distribution = Distribution.cached(minimum_version=platform.target_level,
maximum_version=max_version)
else:
distribution = Distribution.cached(minimum_version=platform.target_level)
with binary_util.safe_args(batch, self._task_exports.task_options) as batch_tests:
self._context.log.debug('CWD = {}'.format(workdir))
self._context.log.debug('platform = {}'.format(platform))
result += abs(execute_java(
classpath=complete_classpath,
main=main,
jvm_options=self._task_exports.jvm_options + extra_jvm_options,
args=self._args + batch_tests + [u'-xmlreport'],
workunit_factory=self._context.new_workunit,
workunit_name='run',
workunit_labels=[WorkUnitLabel.TEST],
cwd=workdir,
distribution=distribution,
))
if result != 0 and self._fail_fast:
break
if result != 0:
failed_targets = self._get_failed_targets(tests_to_targets)
raise TestFailedTaskError(
'java {0} ... exited non-zero ({1}); {2} failed targets.'
.format(main, result, len(failed_targets)),
failed_targets=failed_targets
)
def preferred_jvm_distribution(cls, platforms, strict=False):
"""Returns a jvm Distribution with a version that should work for all the platforms.
Any one of those distributions whose version is >= all requested platforms' versions
can be returned unless strict flag is set.
:param iterable platforms: An iterable of platform settings.
:param bool strict: If true, only distribution whose version matches the minimum
required version can be returned, i.e, the max target_level of all the requested
platforms.
:returns: Distribution one of the selected distributions.
"""
if not platforms:
return DistributionLocator.cached()
min_version = max(platform.target_level for platform in platforms)
max_version = Revision(*(min_version.components + [9999])) if strict else None
return DistributionLocator.cached(minimum_version=min_version, maximum_version=max_version)
def test_wire_compiler_version_robust(self):
# Here the wire compiler is both indirected, and not 1st in the classpath order.
guava = self.make_target(
'3rdparty:guava',
JarLibrary,
jars=[JarDependency('com.google.guava', 'guava', '18.0')])
wire = self.make_target(
'3rdparty:wire',
JarLibrary,
jars=[
JarDependency('com.squareup.wire', 'wire-compiler',
'3.0.0').exclude('com.google.guava', 'guava')
])
alias = self.make_target('a/random/long/address:spec',
Target,
dependencies=[guava, wire])
self.set_options(wire_compiler='a/random/long/address:spec')
task = self.create_task(self.context(target_roots=[alias]))
self.assertEqual(Revision(3, 0, 0), task.wire_compiler_version)
def parse_java_version(cls, version):
"""Parses the java version (given a string or Revision object).
Handles java version-isms, converting things like '7' -> '1.7' appropriately.
Truncates input versions down to just the major and minor numbers (eg, 1.6), ignoring extra
versioning information after the second number.
:param version: the input version, given as a string or Revision object.
:return: the parsed and cleaned version, suitable as a javac -source or -target argument.
:rtype: Revision
"""
conversion = {str(i): '1.{}'.format(i) for i in cls.SUPPORTED_CONVERSION_VERSIONS}
if str(version) in conversion:
return Revision.lenient(conversion[str(version)])
if not hasattr(version, 'components'):
version = Revision.lenient(version)
if len(version.components) <= 2:
return version
return Revision(*version.components[:2])
def format_args_for_target(self, target):
"""Calculate the arguments to pass to the command line for a single target."""
sources_by_base = self._calculate_sources([target])
if self.codegen_strategy.name() == 'isolated':
sources = OrderedSet(target.sources_relative_to_buildroot())
else:
sources = OrderedSet(
itertools.chain.from_iterable(sources_by_base.values()))
if not self.validate_sources_present(sources, [target]):
return None
relative_sources = OrderedSet()
for source in sources:
source_root = SourceRoot.find_by_path(source)
if not source_root:
source_root = SourceRoot.find(target)
relative_source = os.path.relpath(source, source_root)
relative_sources.add(relative_source)
check_duplicate_conflicting_protos(self, sources_by_base,
relative_sources, self.context.log)
args = ['--java_out={0}'.format(self.codegen_workdir(target))]
# Add all params in payload to args
if target.payload.get_field_value('no_options'):
args.append('--no_options')
def append_service_opts(service_type_name, service_type_value,
options_values):
"""Append --service_writer or --service_factory args as appropriate.
:param str service_type_name: the target parameter/option prefix
:param str service_type_value: class passed to the --service_x= option
:param list options_values: string options to be passed with --service_x_opt
"""
if service_type_value:
args.append('--{0}={1}'.format(service_type_name,
service_type_value))
if options_values:
for opt in options_values:
args.append('--{0}_opt'.format(service_type_name))
args.append(opt)
# A check is done in the java_wire_library target to make sure only one of --service_writer or
# --service_factory is specified.
if self.wire_compiler_version < Revision(2, 0):
if target.payload.service_factory:
raise TaskError(
'{spec} used service_factory, which is not available before Wire 2.0. You '
'should use service_writer instead.'.format(
spec=target.address.spec))
append_service_opts('service_writer',
target.payload.service_writer,
target.payload.service_writer_options)
else:
if target.payload.service_writer:
raise TaskError(
'{spec} used service_writer, which is not available after Wire 2.0. You '
'should use service_factory instead.'.format(
spec=target.address.spec))
append_service_opts('service_factory',
target.payload.service_factory,
target.payload.service_factory_options)
registry_class = target.payload.registry_class
if registry_class:
args.append('--registry_class={0}'.format(registry_class))
if target.payload.roots:
args.append('--roots={0}'.format(','.join(target.payload.roots)))
if target.payload.enum_options:
args.append('--enum_options={0}'.format(','.join(
target.payload.enum_options)))
if self.wire_compiler_version < Revision(2, 0):
args.append('--proto_path={0}'.format(
os.path.join(get_buildroot(), SourceRoot.find(target))))
else:
# NB(gmalmquist): Support for multiple --proto_paths was introduced in Wire 2.0.
for path in self._calculate_proto_paths(target):
args.append('--proto_path={0}'.format(path))
args.extend(relative_sources)
return args
def findbugs(self, target):
runtime_classpaths = self.context.products.get_data(
"runtime_classpath")
runtime_classpath = runtime_classpaths.get_for_targets(
target.closure(bfs=True))
aux_classpath = OrderedSet(jar for conf, jar in runtime_classpath
if conf == "default")
target_jars = OrderedSet(
jar for conf, jar in runtime_classpaths.get_for_target(target)
if conf == "default")
bug_counts = {"error": 0, "high": 0, "normal": 0, "low": 0}
if not target_jars:
self.context.log.info(" No jars to be analyzed")
return bug_counts
output_dir = os.path.join(self.workdir, target.id)
safe_mkdir(output_dir)
output_file = os.path.join(output_dir, "findbugsXml.xml")
aux_classpath_file = os.path.join(
self.workdir, "{}.classpath".format(os.path.basename(output_dir)))
with open(aux_classpath_file, "w") as f:
f.write("\n".join(aux_classpath - target_jars))
args = [
"-auxclasspathFromFile",
aux_classpath_file,
"-projectName",
target.address.spec,
"-xml:withMessages",
"-effort:{}".format(self.get_options().effort),
"-{}".format(self.get_options().threshold),
"-nested:{}".format(
"true" if self.get_options().nested else "false"),
"-output",
output_file,
"-noClassOk",
]
if self.get_options().exclude_filter_file:
args.extend([
"-exclude",
os.path.join(get_buildroot(),
self.get_options().exclude_filter_file)
])
if self.get_options().include_filter_file:
args.extend([
"-include",
os.path.join(get_buildroot(),
self.get_options().include_filter_file)
])
if self.get_options().max_rank:
args.extend(["-maxRank", str(self.get_options().max_rank)])
if self.get_options().relaxed:
args.extend(["-relaxed"])
if self.debug:
args.extend(["-progress"])
args.extend(target_jars)
# Try to run spotbugs with the same java version as the target
# The minimum JDK for spotbugs is JDK 1.8
min_jdk_version = max(target.platform.target_level,
Revision.lenient("1.8"))
if min_jdk_version.components[0] == 1:
max_jdk_version = Revision(min_jdk_version.components[0],
min_jdk_version.components[1], "9999")
else:
max_jdk_version = Revision(min_jdk_version.components[0], "9999")
self.set_distribution(minimum_version=min_jdk_version,
maximum_version=max_jdk_version,
jdk=True)
result = self.runjava(
classpath=self.tool_classpath("findbugs"),
main=self._FINDBUGS_MAIN,
jvm_options=self.get_options().jvm_options,
args=args,
workunit_name="findbugs",
workunit_labels=[WorkUnitLabel.LINT],
)
if result != 0:
raise TaskError(
"java {main} ... exited non-zero ({result})".format(
main=self._FINDBUGS_MAIN, result=result))
xml = XmlParser.from_file(output_file)
for error in xml.parsed.getElementsByTagName("Error"):
self.context.log.warn(
"Error: {msg}".format(msg=error.getElementsByTagName(
"ErrorMessage")[0].firstChild.data))
bug_counts["error"] += 1
for bug_instance in xml.parsed.getElementsByTagName("BugInstance"):
bug_rank = bug_instance.getAttribute("rank")
if int(bug_rank) <= self._HIGH_PRIORITY_LOWEST_RANK:
priority = "high"
elif int(bug_rank) <= self._NORMAL_PRIORITY_LOWEST_RANK:
priority = "normal"
else:
priority = "low"
bug_counts[priority] += 1
source_line = bug_instance.getElementsByTagName(
"Class")[0].getElementsByTagName("SourceLine")[0]
self.context.log.warn(
"Bug[{priority}]: {type} {desc} {line}".format(
priority=priority,
type=bug_instance.getAttribute("type"),
desc=bug_instance.getElementsByTagName(
"LongMessage")[0].firstChild.data,
line=source_line.getElementsByTagName("Message")
[0].firstChild.data,
))
return bug_counts
def findbugs(self, target):
runtime_classpaths = self.context.products.get_data(
'runtime_classpath')
runtime_classpath = runtime_classpaths.get_for_targets(
target.closure(bfs=True))
aux_classpath = OrderedSet(jar for conf, jar in runtime_classpath
if conf == 'default')
target_jars = OrderedSet(
jar for conf, jar in runtime_classpaths.get_for_target(target)
if conf == 'default')
bug_counts = {'error': 0, 'high': 0, 'normal': 0, 'low': 0}
if not target_jars:
self.context.log.info(' No jars to be analyzed')
return bug_counts
output_dir = os.path.join(self.workdir, target.id)
safe_mkdir(output_dir)
output_file = os.path.join(output_dir, 'findbugsXml.xml')
aux_classpath_file = os.path.join(
self.workdir, '{}.classpath'.format(os.path.basename(output_dir)))
with open(aux_classpath_file, 'w') as f:
f.write('\n'.join(aux_classpath - target_jars))
args = [
'-auxclasspathFromFile', aux_classpath_file, '-projectName',
target.address.spec, '-xml:withMessages',
'-effort:{}'.format(self.get_options().effort),
'-{}'.format(self.get_options().threshold), '-nested:{}'.format(
'true' if self.get_options().nested else 'false'), '-output',
output_file, '-noClassOk'
]
if self.get_options().exclude_filter_file:
args.extend([
'-exclude',
os.path.join(get_buildroot(),
self.get_options().exclude_filter_file)
])
if self.get_options().include_filter_file:
args.extend([
'-include',
os.path.join(get_buildroot(),
self.get_options().include_filter_file)
])
if self.get_options().max_rank:
args.extend(['-maxRank', str(self.get_options().max_rank)])
if self.get_options().relaxed:
args.extend(['-relaxed'])
if self.get_options().level == 'debug':
args.extend(['-progress'])
args.extend(target_jars)
# Try to run spotbugs with the same java version as the target
# The minimum JDK for spotbugs is JDK 1.8
min_jdk_version = max(target.platform.target_level,
Revision.lenient('1.8'))
if min_jdk_version.components[0] == 1:
max_jdk_version = Revision(min_jdk_version.components[0],
min_jdk_version.components[1], '9999')
else:
max_jdk_version = Revision(min_jdk_version.components[0], '9999')
self.set_distribution(minimum_version=min_jdk_version,
maximum_version=max_jdk_version,
jdk=True)
result = self.runjava(classpath=self.tool_classpath('findbugs'),
main=self._FINDBUGS_MAIN,
jvm_options=self.get_options().jvm_options,
args=args,
workunit_name='findbugs',
workunit_labels=[WorkUnitLabel.LINT])
if result != 0:
raise TaskError(
'java {main} ... exited non-zero ({result})'.format(
main=self._FINDBUGS_MAIN, result=result))
xml = XmlParser.from_file(output_file)
for error in xml.parsed.getElementsByTagName('Error'):
self.context.log.warn(
'Error: {msg}'.format(msg=error.getElementsByTagName(
'ErrorMessage')[0].firstChild.data))
bug_counts['error'] += 1
for bug_instance in xml.parsed.getElementsByTagName('BugInstance'):
bug_rank = bug_instance.getAttribute('rank')
if int(bug_rank) <= self._HIGH_PRIORITY_LOWEST_RANK:
priority = 'high'
elif int(bug_rank) <= self._NORMAL_PRIORITY_LOWEST_RANK:
priority = 'normal'
else:
priority = 'low'
bug_counts[priority] += 1
source_line = bug_instance.getElementsByTagName(
'Class')[0].getElementsByTagName('SourceLine')[0]
self.context.log.warn(
'Bug[{priority}]: {type} {desc} {line}'.format(
priority=priority,
type=bug_instance.getAttribute('type'),
desc=bug_instance.getElementsByTagName(
'LongMessage')[0].firstChild.data,
line=source_line.getElementsByTagName(
'Message')[0].firstChild.data))
return bug_counts