def test_random_estimator(self):
seedValue = 5
# The number chosen for seedValue doesn't matter, so long as it is the same for the call to
# generate a random test number and the call to create_size_estimators.
random.seed(seedValue)
rand = random.randint(0, 10000)
random.seed(seedValue)
with temporary_file_path() as src:
self.assertEqual(create_size_estimators()['random']([src]), rand)
def test_random_estimator(self):
seedValue = 5
# The number chosen for seedValue doesn't matter, so long as it is the same for the call to
# generate a random test number and the call to create_size_estimators.
random.seed(seedValue)
rand = random.randint(0, 10000)
random.seed(seedValue)
with temporary_file_path() as src:
self.assertEqual(create_size_estimators()['random']([src]), rand)
class JvmDependencyUsage(JvmDependencyAnalyzer):
"""Determines the dependency usage ratios of targets.
Analyzes the relationship between the products a target T produces vs. the products
which T's dependents actually require (this is done by observing analysis files).
If the ratio of required products to available products is low, then this is a sign
that target T isn't factored well.
A graph is formed from these results, where each node of the graph is a target, and
each edge is a product usage ratio between a target and its dependency. The nodes
also contain additional information to guide refactoring -- for example, the estimated
job size of each target, which indicates the impact a poorly factored target has on
the build times. (see DependencyUsageGraph->to_json)
The graph is either summarized for local analysis or outputted as a JSON file for
aggregation and analysis on a larger scale.
"""
size_estimators = create_size_estimators()
@classmethod
def register_options(cls, register):
super(JvmDependencyUsage, cls).register_options(register)
register(
'--internal-only',
default=True,
type=bool,
help=
'Specifies that only internal dependencies should be included in the graph '
'output (no external jars).')
register(
'--summary',
default=True,
type=bool,
help=
'When set, outputs a summary of the "worst" dependencies; otherwise, '
'outputs a JSON report.')
register('--size-estimator',
choices=list(cls.size_estimators.keys()),
default='filesize',
help='The method of target size estimation.')
register('--transitive',
default=True,
type=bool,
help='Score all targets in the build graph transitively.')
register('--output-file',
type=str,
help='Output destination. When unset, outputs to <stdout>.')
register(
'--use-cached',
type=bool,
help='Use cached dependency data to compute analysis result. '
'When set, skips `resolve` and `compile` steps. '
'Useful for computing analysis for a lot of targets, but '
'result can differ from direct execution because cached information '
'doesn\'t depend on 3rdparty libraries versions.')
@classmethod
def prepare(cls, options, round_manager):
if not options.use_cached:
super(JvmDependencyUsage, cls).prepare(options, round_manager)
round_manager.require_data('classes_by_source')
round_manager.require_data('runtime_classpath')
round_manager.require_data('product_deps_by_src')
else:
# We want to have synthetic targets in build graph to deserialize nodes properly.
round_manager.require_data('java')
round_manager.require_data('scala')
round_manager.require_data('deferred_sources')
@classmethod
def skip(cls, options):
"""This task is always explicitly requested."""
return False
def execute(self):
graph = self.create_dep_usage_graph(self.context.targets(
) if self.get_options().transitive else self.context.target_roots)
output_file = self.get_options().output_file
if output_file:
self.context.log.info(
'Writing dependency usage to {}'.format(output_file))
with open(output_file, 'w') as fh:
self._render(graph, fh)
else:
sys.stdout.write(b'\n')
self._render(graph, sys.stdout)
@classmethod
def implementation_version(cls):
return super(JvmDependencyUsage, cls).implementation_version() + [
('JvmDependencyUsage', 4)
]
def _render(self, graph, fh):
chunks = graph.to_summary() if self.get_options(
).summary else graph.to_json()
for chunk in chunks:
fh.write(chunk)
fh.flush()
def _resolve_aliases(self, target):
"""Recursively resolve `target` aliases.
:param Target target: target whose dependencies are to be resolved recursively.
:returns: An iterator of (resolved_dependency, resolved_from) tuples.
`resolved_from` is the top level target alias that depends on `resolved_dependency`,
and `None` if `resolved_dependency` is not a dependency of a target alias.
When there are nested aliases, this implementation returns just the top level,
consider returning the entire path to allow more fine grained alias usage analysis.
"""
for declared in target.dependencies:
if type(declared) in (Target, AliasTarget):
for r, _ in self._resolve_aliases(declared):
yield r, declared
else:
yield declared, None
def _is_declared_dep(self, target, dep):
"""Returns true if the given dep target should be considered a declared dep of target."""
return dep in [
resolved for resolved, _ in self._resolve_aliases(target)
]
def _select(self, target):
if self.get_options().internal_only and isinstance(target, JarLibrary):
return False
elif isinstance(target, Resources) or type(target) == Target:
return False
else:
return True
def _normalize_product_dep(self, buildroot, classes_by_source, dep):
"""Normalizes the given product dep from the given dep into a set of classfiles.
Product deps arrive as sources, jars, and classfiles: this method normalizes them to classfiles.
TODO: This normalization should happen in the super class.
"""
if dep.endswith(".jar"):
# TODO: post sbt/zinc jar output patch, binary deps will be reported directly as classfiles
return set()
elif dep.endswith(".class"):
return set([dep])
else:
# assume a source file and convert to classfiles
rel_src = fast_relpath(dep, buildroot)
return set(p
for _, paths in classes_by_source[rel_src].rel_paths()
for p in paths)
def _count_products(self, classpath_products, target):
contents = ClasspathUtil.classpath_contents((target, ),
classpath_products)
# Generators don't implement len.
return sum(1 for _ in contents)
def create_dep_usage_graph(self, targets):
"""Creates a graph of concrete targets, with their sum of products and dependencies.
Synthetic targets contribute products and dependencies to their concrete target.
"""
with self.invalidated(
targets, invalidate_dependents=True) as invalidation_check:
target_to_vts = {}
for vts in invalidation_check.all_vts:
target_to_vts[vts.target] = vts
if not self.get_options().use_cached:
node_creator = self.calculating_node_creator(
self.context.products.get_data('classes_by_source'),
self.context.products.get_data('runtime_classpath'),
self.context.products.get_data('product_deps_by_src'),
target_to_vts)
else:
node_creator = self.cached_node_creator(target_to_vts)
return DependencyUsageGraph(
self.create_dep_usage_nodes(targets, node_creator),
self.size_estimators[self.get_options().size_estimator])
def calculating_node_creator(self, classes_by_source, runtime_classpath,
product_deps_by_src, target_to_vts):
"""Strategy directly computes dependency graph node based on
`classes_by_source`, `runtime_classpath`, `product_deps_by_src` parameters and
stores the result to the build cache.
"""
def creator(target):
node = self.create_dep_usage_node(
target, get_buildroot(), classes_by_source, runtime_classpath,
product_deps_by_src, self._compute_transitive_deps_by_target())
vt = target_to_vts[target]
with open(self.nodes_json(vt.results_dir), mode='w') as fp:
json.dump(node.to_cacheable_dict(),
fp,
indent=2,
sort_keys=True)
vt.update()
return node
return creator
def cached_node_creator(self, target_to_vts):
"""Strategy restores dependency graph node from the build cache.
"""
def creator(target):
vt = target_to_vts[target]
if vt.valid and os.path.exists(self.nodes_json(vt.results_dir)):
try:
with open(self.nodes_json(vt.results_dir)) as fp:
return Node.from_cacheable_dict(
json.load(fp), lambda spec: self.context.resolve(
spec).__iter__().next())
except Exception:
self.context.log.warn(
"Can't deserialize json for target {}".format(target))
return Node(target.concrete_derived_from)
else:
self.context.log.warn("No cache entry for {}".format(target))
return Node(target.concrete_derived_from)
return creator
def nodes_json(self, target_results_dir):
return os.path.join(target_results_dir, 'node.json')
def create_dep_usage_nodes(self, targets, node_creator):
nodes = dict()
for target in targets:
if not self._select(target):
continue
# Create or extend a Node for the concrete version of this target.
concrete_target = target.concrete_derived_from
node = node_creator(target)
if concrete_target in nodes:
nodes[concrete_target].combine(node)
else:
nodes[concrete_target] = node
# Prune any Nodes with 0 products.
for concrete_target, node in nodes.items()[:]:
if node.products_total == 0:
nodes.pop(concrete_target)
return nodes
def cache_target_dirs(self):
return True
def create_dep_usage_node(self, target, buildroot, classes_by_source,
runtime_classpath, product_deps_by_src,
transitive_deps_by_target):
concrete_target = target.concrete_derived_from
products_total = self._count_products(runtime_classpath, target)
node = Node(concrete_target)
node.add_derivation(target, products_total)
# Record declared Edges.
for dep_tgt, aliased_from in self._resolve_aliases(target):
derived_from = dep_tgt.concrete_derived_from
if self._select(derived_from):
node.add_edge(Edge(is_declared=True, products_used=set()),
derived_from, aliased_from)
# Record the used products and undeclared Edges for this target. Note that some of
# these may be self edges, which are considered later.
target_product_deps_by_src = product_deps_by_src.get(target, dict())
for src in target.sources_relative_to_buildroot():
for product_dep in target_product_deps_by_src.get(
os.path.join(buildroot, src), []):
for dep_tgt in self.targets_by_file.get(product_dep, []):
derived_from = dep_tgt.concrete_derived_from
if not self._select(derived_from):
continue
# Create edge only for those direct or transitive dependencies in order to
# disqualify irrelevant targets that happen to share some file in sources,
# not uncommon when globs especially rglobs is used.
if not derived_from in transitive_deps_by_target.get(
concrete_target):
continue
is_declared = self._is_declared_dep(target, dep_tgt)
normalized_deps = self._normalize_product_dep(
buildroot, classes_by_source, product_dep)
node.add_edge(
Edge(is_declared=is_declared,
products_used=normalized_deps), derived_from)
return node
def test_line_count_estimator(self):
with temporary_file_path() as src:
self.assertEqual(create_size_estimators()['linecount']([src]), 0)
class JvmCompile(CompilerOptionSetsMixin, NailgunTaskBase):
"""A common framework for JVM compilation.
To subclass for a specific JVM language, implement the static values and methods
mentioned below under "Subclasses must implement".
"""
size_estimators = create_size_estimators()
@classmethod
def size_estimator_by_name(cls, estimation_strategy_name):
return cls.size_estimators[estimation_strategy_name]
@classmethod
def register_options(cls, register):
super(JvmCompile, cls).register_options(register)
register('--args',
advanced=True,
type=list,
default=list(cls.get_args_default(register.bootstrap)),
fingerprint=True,
help='Pass these extra args to the compiler.')
register(
'--clear-invalid-analysis',
advanced=True,
type=bool,
help=
'When set, any invalid/incompatible analysis files will be deleted '
'automatically. When unset, an error is raised instead.')
register('--warnings',
default=True,
type=bool,
fingerprint=True,
help='Compile with all configured warnings enabled.')
register('--warning-args',
advanced=True,
type=list,
fingerprint=True,
default=list(cls.get_warning_args_default()),
help='Extra compiler args to use when warnings are enabled.')
register('--no-warning-args',
advanced=True,
type=list,
fingerprint=True,
default=list(cls.get_no_warning_args_default()),
help='Extra compiler args to use when warnings are disabled.')
register('--debug-symbols',
type=bool,
fingerprint=True,
help='Compile with debug symbol enabled.')
register('--debug-symbol-args',
advanced=True,
type=list,
fingerprint=True,
default=['-C-g:lines,source,vars'],
help='Extra args to enable debug symbol.')
register(
'--delete-scratch',
advanced=True,
default=True,
type=bool,
help=
'Leave intermediate scratch files around, for debugging build problems.'
)
register('--worker-count',
advanced=True,
type=int,
default=cpu_count(),
help='The number of concurrent workers to use when '
'compiling with {task}. Defaults to the '
'current machine\'s CPU count.'.format(task=cls._name))
register(
'--size-estimator',
advanced=True,
choices=list(cls.size_estimators.keys()),
default='filesize',
help=
'The method of target size estimation. The size estimator estimates the size '
'of targets in order to build the largest targets first (subject to dependency '
'constraints). Choose \'random\' to choose random sizes for each target, which '
'may be useful for distributed builds.')
register(
'--capture-classpath',
advanced=True,
type=bool,
default=True,
fingerprint=True,
help=
'Capture classpath to per-target newline-delimited text files. These files will '
'be packaged into any jar artifacts that are created from the jvm targets.'
)
register(
'--suggest-missing-deps',
type=bool,
help=
'Suggest missing dependencies on a best-effort basis from target\'s transitive'
'deps for compilation failures that are due to class not found.')
register(
'--buildozer',
help='Path to buildozer for suggest-missing-deps command lines. '
'If absent, no command line will be suggested to fix missing deps.'
)
register(
'--missing-deps-not-found-msg',
advanced=True,
type=str,
help=
'The message to print when pants can\'t find any suggestions for targets '
'containing the classes not found during compilation. This should '
'likely include a link to documentation about dependency management.',
default=
'Please see https://www.pantsbuild.org/3rdparty_jvm.html#strict-dependencies '
'for more information.')
register(
'--class-not-found-error-patterns',
advanced=True,
type=list,
default=CLASS_NOT_FOUND_ERROR_PATTERNS,
help=
'List of regular expression patterns that extract class not found '
'compile errors.')
register(
'--use-classpath-jars',
advanced=True,
type=bool,
fingerprint=True,
help=
'Use jar files on the compile_classpath. Note: Using this option degrades '
'incremental compile between targets.')
@classmethod
def implementation_version(cls):
return super(JvmCompile,
cls).implementation_version() + [('JvmCompile', 3)]
@classmethod
def prepare(cls, options, round_manager):
super(JvmCompile, cls).prepare(options, round_manager)
round_manager.require_data('compile_classpath')
# Require codegen we care about
# TODO(John Sirois): roll this up in Task - if the list of labels we care about for a target
# predicate to filter the full build graph is exposed, the requirement can be made automatic
# and in turn codegen tasks could denote the labels they produce automating wiring of the
# produce side
round_manager.optional_data('java')
round_manager.optional_data('scala')
# Allow the deferred_sources_mapping to take place first
round_manager.optional_data('deferred_sources')
# Subclasses must implement.
# --------------------------
_name = None
# The name used in JvmPlatform to refer to this compiler task.
compiler_name = None
@classmethod
def subsystem_dependencies(cls):
return super(JvmCompile, cls).subsystem_dependencies() + (
DependencyContext, Java, JvmPlatform, ScalaPlatform, Zinc.Factory)
@classmethod
def name(cls):
return cls._name
@classmethod
def get_args_default(cls, bootstrap_option_values):
"""Override to set default for --args option.
:param bootstrap_option_values: The values of the "bootstrap options" (e.g., pants_workdir).
Implementations can use these when generating the default.
See src/python/pants/options/options_bootstrapper.py for
details.
"""
return ()
@classmethod
def get_warning_args_default(cls):
"""Override to set default for --warning-args option."""
return ()
@classmethod
def get_no_warning_args_default(cls):
"""Override to set default for --no-warning-args option."""
return ()
@property
def cache_target_dirs(self):
return True
@memoized_property
def _zinc(self):
return Zinc.Factory.global_instance().create(self.context.products,
self.execution_strategy)
def _zinc_tool_classpath(self, toolname):
return self._zinc.tool_classpath_from_products(
self.context.products, toolname, scope=self.options_scope)
def _zinc_tool_jar(self, toolname):
return self._zinc.tool_jar_from_products(self.context.products,
toolname,
scope=self.options_scope)
def select(self, target):
raise NotImplementedError()
def select_source(self, source_file_path):
raise NotImplementedError()
def compile(self, ctx, args, dependency_classpath, upstream_analysis,
settings, compiler_option_sets, zinc_file_manager,
javac_plugin_map, scalac_plugin_map):
"""Invoke the compiler.
Subclasses must implement. Must raise TaskError on compile failure.
:param CompileContext ctx: A CompileContext for the target to compile.
:param list args: Arguments to the compiler (such as javac or zinc).
:param list dependency_classpath: List of classpath entries of type ClasspathEntry for
dependencies.
:param upstream_analysis: A map from classpath entry to analysis file for dependencies.
:param JvmPlatformSettings settings: platform settings determining the -source, -target, etc for
javac to use.
:param list compiler_option_sets: The compiler_option_sets flags for the target.
:param zinc_file_manager: whether to use zinc provided file manager.
:param javac_plugin_map: Map of names of javac plugins to use to their arguments.
:param scalac_plugin_map: Map of names of scalac plugins to use to their arguments.
"""
raise NotImplementedError()
# Subclasses may override.
# ------------------------
def extra_compile_time_classpath_elements(self):
"""Extra classpath elements common to all compiler invocations.
These should be of type ClasspathEntry, but strings are also supported for backwards
compatibility.
E.g., jars for compiler plugins.
These are added at the end of the classpath, after any dependencies, so that if they
overlap with any explicit dependencies, the compiler sees those first. This makes
missing dependency accounting much simpler.
"""
return []
def scalac_plugin_classpath_elements(self):
"""Classpath entries containing scalac plugins."""
return []
def write_extra_resources(self, compile_context):
"""Writes any extra, out-of-band resources for a target to its classes directory.
E.g., targets that produce scala compiler plugins or annotation processor files
produce an info file. The resources will be added to the runtime_classpath.
Returns a list of pairs (root, [absolute paths of files under root]).
"""
pass
def create_empty_extra_products(self):
"""Create any products the subclass task supports in addition to the runtime_classpath.
The runtime_classpath is constructed by default.
"""
def register_extra_products_from_contexts(self, targets, compile_contexts):
"""Allows subclasses to register additional products for targets.
It is called for valid targets at start, then for each completed invalid target,
separately, during compilation.
"""
def select_runtime_context(self, ccs):
"""Select the context that contains the paths for runtime classpath artifacts.
Subclasses may have more than one type of context."""
return ccs
def __init__(self, *args, **kwargs):
super(JvmCompile, self).__init__(*args, **kwargs)
self._targets_to_compile_settings = None
# JVM options for running the compiler.
self._jvm_options = self.get_options().jvm_options
self._args = list(self.get_options().args)
if self.get_options().warnings:
self._args.extend(self.get_options().warning_args)
else:
self._args.extend(self.get_options().no_warning_args)
if self.get_options().debug_symbols:
self._args.extend(self.get_options().debug_symbol_args)
# The ivy confs for which we're building.
self._confs = Zinc.DEFAULT_CONFS
# Determines which sources are relevant to this target.
self._sources_predicate = self.select_source
self._delete_scratch = self.get_options().delete_scratch
self._clear_invalid_analysis = self.get_options(
).clear_invalid_analysis
try:
worker_count = self.get_options().worker_count
except AttributeError:
# tasks that don't support concurrent execution have no worker_count registered
worker_count = 1
self._worker_count = worker_count
self._size_estimator = self.size_estimator_by_name(
self.get_options().size_estimator)
@memoized_property
def _missing_deps_finder(self):
dep_analyzer = JvmDependencyAnalyzer(
get_buildroot(),
self.context.products.get_data('runtime_classpath'))
return MissingDependencyFinder(
dep_analyzer,
CompileErrorExtractor(
self.get_options().class_not_found_error_patterns))
def create_compile_context(self, target, target_workdir):
return CompileContext(
target, os.path.join(target_workdir, 'z.analysis'),
ClasspathEntry(os.path.join(target_workdir, 'classes')),
ClasspathEntry(os.path.join(target_workdir, 'z.jar')),
os.path.join(target_workdir, 'logs'),
os.path.join(target_workdir, 'zinc_args'),
self._compute_sources_for_target(target))
def execute(self):
if JvmPlatform.global_instance().get_options(
).compiler != self.compiler_name:
# If the requested compiler is not the one supported by this task,
# bail early.
return
# In case we have no relevant targets and return early, create the requested product maps.
self.create_empty_extra_products()
relevant_targets = list(self.context.targets(predicate=self.select))
if not relevant_targets:
return
# Clone the compile_classpath to the runtime_classpath.
classpath_product = self.create_runtime_classpath()
fingerprint_strategy = DependencyContext.global_instance(
).create_fingerprint_strategy(classpath_product)
# Note, JVM targets are validated (`vts.update()`) as they succeed. As a result,
# we begin writing artifacts out to the cache immediately instead of waiting for
# all targets to finish.
with self.invalidated(relevant_targets,
invalidate_dependents=True,
fingerprint_strategy=fingerprint_strategy,
topological_order=True) as invalidation_check:
compile_contexts = {
vt.target: self.create_compile_context(vt.target,
vt.results_dir)
for vt in invalidation_check.all_vts
}
self.do_compile(
invalidation_check,
compile_contexts,
classpath_product,
)
if not self.get_options().use_classpath_jars:
# Once compilation has completed, replace the classpath entry for each target with
# its jar'd representation.
for ccs in compile_contexts.values():
cc = self.select_runtime_context(ccs)
for conf in self._confs:
classpath_product.remove_for_target(
cc.target, [(conf, cc.classes_dir.path)])
classpath_product.add_for_target(
cc.target, [(conf, cc.jar_file.path)])
def _classpath_for_context(self, context):
if self.get_options().use_classpath_jars:
return context.jar_file
return context.classes_dir
def create_runtime_classpath(self):
compile_classpath = self.context.products.get_data('compile_classpath')
classpath_product = self.context.products.get_data('runtime_classpath')
if not classpath_product:
classpath_product = self.context.products.get_data(
'runtime_classpath', compile_classpath.copy)
else:
classpath_product.update(compile_classpath)
return classpath_product
def do_compile(self, invalidation_check, compile_contexts,
classpath_product):
"""Executes compilations for the invalid targets contained in a single chunk."""
invalid_targets = [vt.target for vt in invalidation_check.invalid_vts]
valid_targets = [
vt.target for vt in invalidation_check.all_vts if vt.valid
]
if self.execution_strategy == self.HERMETIC:
self._set_directory_digests_for_valid_target_classpath_directories(
valid_targets, compile_contexts)
for valid_target in valid_targets:
cc = self.select_runtime_context(compile_contexts[valid_target])
classpath_product.add_for_target(
valid_target,
[(conf, self._classpath_for_context(cc))
for conf in self._confs],
)
self.register_extra_products_from_contexts(valid_targets,
compile_contexts)
if not invalid_targets:
return
# This ensures the workunit for the worker pool is set before attempting to compile.
with self.context.new_workunit('isolation-{}-pool-bootstrap'.format(self.name())) \
as workunit:
# This uses workunit.parent as the WorkerPool's parent so that child workunits
# of different pools will show up in order in the html output. This way the current running
# workunit is on the bottom of the page rather than possibly in the middle.
worker_pool = WorkerPool(workunit.parent, self.context.run_tracker,
self._worker_count)
# Prepare the output directory for each invalid target, and confirm that analysis is valid.
for target in invalid_targets:
cc = self.select_runtime_context(compile_contexts[target])
safe_mkdir(cc.classes_dir.path)
# Now create compile jobs for each invalid target one by one, using the classpath
# generated by upstream JVM tasks and our own prepare_compile().
jobs = self._create_compile_jobs(compile_contexts, invalid_targets,
invalidation_check.invalid_vts,
classpath_product)
exec_graph = ExecutionGraph(
jobs,
self.get_options().print_exception_stacktrace)
try:
exec_graph.execute(worker_pool, self.context.log)
except ExecutionFailure as e:
raise TaskError("Compilation failure: {}".format(e))
def _record_compile_classpath(self, classpath, target, outdir):
relative_classpaths = [
fast_relpath(path,
self.get_options().pants_workdir)
for path in classpath
]
text = '\n'.join(relative_classpaths)
path = os.path.join(outdir, 'compile_classpath',
'{}.txt'.format(target.id))
safe_mkdir(os.path.dirname(path), clean=False)
with open(path, 'w') as f:
f.write(text)
def _set_directory_digests_for_valid_target_classpath_directories(
self, valid_targets, compile_contexts):
snapshots = self.context._scheduler.capture_snapshots(
tuple(
PathGlobsAndRoot(
PathGlobs([
self._get_relative_classes_dir_from_target(
target, compile_contexts)
]), get_buildroot()) for target in valid_targets))
[
self._set_directory_digest_for_compile_context(
snapshot.directory_digest, target, compile_contexts)
for target, snapshot in list(zip(valid_targets, snapshots))
]
def _get_relative_classes_dir_from_target(self, target, compile_contexts):
cc = self.select_runtime_context(compile_contexts[target])
return fast_relpath(cc.classes_dir.path, get_buildroot()) + '/**'
def _set_directory_digest_for_compile_context(self, directory_digest,
target, compile_contexts):
cc = self.select_runtime_context(compile_contexts[target])
new_classpath_entry = ClasspathEntry(cc.classes_dir.path,
directory_digest)
cc.classes_dir = new_classpath_entry
def _compile_vts(self, vts, ctx, upstream_analysis, dependency_classpath,
progress_message, settings, compiler_option_sets,
zinc_file_manager, counter):
"""Compiles sources for the given vts into the given output dir.
:param vts: VersionedTargetSet with one entry for the target.
:param ctx: - A CompileContext instance for the target.
:param dependency_classpath: A list of classpath entries of type ClasspathEntry for dependencies
May be invoked concurrently on independent target sets.
Postcondition: The individual targets in vts are up-to-date, as if each were
compiled individually.
"""
if not ctx.sources:
self.context.log.warn(
'Skipping {} compile for targets with no sources:\n {}'.
format(self.name(), vts.targets))
else:
counter_val = str(counter()).rjust(counter.format_length(), ' ')
counter_str = '[{}/{}] '.format(counter_val, counter.size)
# Do some reporting.
self.context.log.info(
counter_str, 'Compiling ',
items_to_report_element(ctx.sources,
'{} source'.format(self.name())),
' in ',
items_to_report_element(
[t.address.reference() for t in vts.targets], 'target'),
' (', progress_message, ').')
with self.context.new_workunit('compile',
labels=[WorkUnitLabel.COMPILER
]) as compile_workunit:
try:
directory_digest = self.compile(
ctx,
self._args,
dependency_classpath,
upstream_analysis,
settings,
compiler_option_sets,
zinc_file_manager,
self._get_plugin_map('javac', Java.global_instance(),
ctx.target),
self._get_plugin_map('scalac',
ScalaPlatform.global_instance(),
ctx.target),
)
self._capture_logs(compile_workunit, ctx.log_dir)
return directory_digest
except TaskError:
if self.get_options().suggest_missing_deps:
logs = [
path for _, name, _, path in self._find_logs(
compile_workunit) if name == self.name()
]
if logs:
self._find_missing_deps(logs, ctx.target)
raise
def _capture_logs(self, workunit, destination):
safe_mkdir(destination, clean=True)
for idx, name, output_name, path in self._find_logs(workunit):
os.link(
path,
os.path.join(destination,
'{}-{}-{}.log'.format(name, idx, output_name)))
def _get_plugin_map(self, compiler, options_src, target):
"""Returns a map of plugin to args, for the given compiler.
Only plugins that must actually be activated will be present as keys in the map.
Plugins with no arguments will have an empty list as a value.
Active plugins and their args will be gathered from (in order of precedence):
- The <compiler>_plugins and <compiler>_plugin_args fields of the target, if it has them.
- The <compiler>_plugins and <compiler>_plugin_args options of this task, if it has them.
- The <compiler>_plugins and <compiler>_plugin_args fields of this task, if it has them.
Note that in-repo plugins will not be returned, even if requested, when building
themselves. Use published versions of those plugins for that.
See:
- examples/src/java/org/pantsbuild/example/javac/plugin/README.md.
- examples/src/scala/org/pantsbuild/example/scalac/plugin/README.md
:param compiler: one of 'javac', 'scalac'.
:param options_src: A JvmToolMixin instance providing plugin options.
:param target: The target whose plugins we compute.
"""
# Note that we get() options and getattr() target fields and task methods,
# so we're robust when those don't exist (or are None).
plugins_key = '{}_plugins'.format(compiler)
requested_plugins = (
tuple(getattr(self, plugins_key, []) or []) +
tuple(options_src.get_options().get(plugins_key, []) or []) +
tuple((getattr(target, plugins_key, []) or [])))
# Allow multiple flags and also comma-separated values in a single flag.
requested_plugins = {
p
for val in requested_plugins for p in val.split(',')
}
plugin_args_key = '{}_plugin_args'.format(compiler)
available_plugin_args = {}
available_plugin_args.update(getattr(self, plugin_args_key, {}) or {})
available_plugin_args.update(
options_src.get_options().get(plugin_args_key, {}) or {})
available_plugin_args.update(
getattr(target, plugin_args_key, {}) or {})
# From all available args, pluck just the ones for the selected plugins.
plugin_map = {}
for plugin in requested_plugins:
# Don't attempt to use a plugin while building that plugin.
# This avoids a bootstrapping problem. Note that you can still
# use published plugins on themselves, just not in-repo plugins.
if target not in self._plugin_targets(compiler).get(plugin, {}):
plugin_map[plugin] = available_plugin_args.get(plugin, [])
return plugin_map
def _find_logs(self, compile_workunit):
"""Finds all logs under the given workunit."""
for idx, workunit in enumerate(compile_workunit.children):
for output_name, outpath in workunit.output_paths().items():
if output_name in ('stdout', 'stderr'):
yield idx, workunit.name, output_name, outpath
def _find_missing_deps(self, compile_logs, target):
with self.context.new_workunit('missing-deps-suggest',
labels=[WorkUnitLabel.COMPILER]):
compile_failure_log = '\n'.join(
read_file(log) for log in compile_logs)
missing_dep_suggestions, no_suggestions = self._missing_deps_finder.find(
compile_failure_log, target)
if missing_dep_suggestions:
self.context.log.info(
'Found the following deps from target\'s transitive '
'dependencies that provide the missing classes:')
suggested_deps = set()
for classname, candidates in missing_dep_suggestions.items():
suggested_deps.add(list(candidates)[0])
self.context.log.info(' {}: {}'.format(
classname, ', '.join(candidates)))
# We format the suggested deps with single quotes and commas so that
# they can be easily cut/pasted into a BUILD file.
formatted_suggested_deps = [
"'%s'," % dep for dep in suggested_deps
]
suggestion_msg = (
'\nIf the above information is correct, '
'please add the following to the dependencies of ({}):\n {}\n'
.format(
target.address.spec,
'\n '.join(sorted(list(formatted_suggested_deps)))))
path_to_buildozer = self.get_options().buildozer
if path_to_buildozer:
suggestion_msg += (
"\nYou can do this by running:\n"
" {buildozer} 'add dependencies {deps}' {target}".
format(buildozer=path_to_buildozer,
deps=" ".join(sorted(suggested_deps)),
target=target.address.spec))
self.context.log.info(suggestion_msg)
if no_suggestions:
self.context.log.warn(
'Unable to find any deps from target\'s transitive '
'dependencies that provide the following missing classes:')
no_suggestion_msg = '\n '.join(sorted(list(no_suggestions)))
self.context.log.warn(' {}'.format(no_suggestion_msg))
self.context.log.warn(
self.get_options().missing_deps_not_found_msg)
def _upstream_analysis(self, compile_contexts, classpath_entries):
"""Returns tuples of classes_dir->analysis_file for the closure of the target."""
# Reorganize the compile_contexts by class directory.
compile_contexts_by_directory = {}
for compile_context in compile_contexts.values():
compile_context = self.select_runtime_context(compile_context)
compile_contexts_by_directory[
compile_context.classes_dir.path] = compile_context
# If we have a compile context for the target, include it.
for entry in classpath_entries:
path = entry.path
if not path.endswith('.jar'):
compile_context = compile_contexts_by_directory.get(path)
if not compile_context:
self.context.log.debug(
'Missing upstream analysis for {}'.format(path))
else:
yield compile_context.classes_dir.path, compile_context.analysis_file
def exec_graph_key_for_target(self, compile_target):
return "compile({})".format(compile_target.address.spec)
def _create_compile_jobs(self, compile_contexts, invalid_targets,
invalid_vts, classpath_product):
class Counter(object):
def __init__(self, size, initial=0):
self.size = size
self.count = initial
def __call__(self):
self.count += 1
return self.count
def format_length(self):
return len(str(self.size))
counter = Counter(len(invalid_vts))
jobs = []
jobs.extend(self.pre_compile_jobs(counter))
invalid_target_set = set(invalid_targets)
for ivts in invalid_vts:
# Invalidated targets are a subset of relevant targets: get the context for this one.
compile_target = ivts.target
invalid_dependencies = self._collect_invalid_compile_dependencies(
compile_target, invalid_target_set, compile_contexts)
jobs.extend(
self.create_compile_jobs(compile_target, compile_contexts,
invalid_dependencies, ivts, counter,
classpath_product))
counter.size = len(jobs)
return jobs
def pre_compile_jobs(self, counter):
"""Override this to provide jobs that are not related to particular targets.
This is only called when there are invalid targets."""
return []
def create_compile_jobs(self, compile_target, all_compile_contexts,
invalid_dependencies, ivts, counter,
classpath_product):
context_for_target = all_compile_contexts[compile_target]
compile_context = self.select_runtime_context(context_for_target)
job = Job(
self.exec_graph_key_for_target(compile_target),
functools.partial(self._default_work_for_vts, ivts,
compile_context, 'runtime_classpath', counter,
all_compile_contexts, classpath_product),
[
self.exec_graph_key_for_target(target)
for target in invalid_dependencies
],
self._size_estimator(compile_context.sources),
# If compilation and analysis work succeeds, validate the vts.
# Otherwise, fail it.
on_success=ivts.update,
on_failure=ivts.force_invalidate)
return [job]
def check_cache(self, vts, counter):
"""Manually checks the artifact cache (usually immediately before compilation.)
Returns true if the cache was hit successfully, indicating that no compilation is necessary.
"""
if not self.artifact_cache_reads_enabled():
return False
cached_vts, _, _ = self.check_artifact_cache([vts])
if not cached_vts:
self.context.log.debug(
'Missed cache during double check for {}'.format(
vts.target.address.spec))
return False
assert cached_vts == [
vts
], ('Cache returned unexpected target: {} vs {}'.format(
cached_vts, [vts]))
self.context.log.info('Hit cache during double check for {}'.format(
vts.target.address.spec))
counter()
return True
def should_compile_incrementally(self, vts, ctx):
"""Check to see if the compile should try to re-use the existing analysis.
Returns true if we should try to compile the target incrementally.
"""
if not vts.is_incremental:
return False
if not self._clear_invalid_analysis:
return True
return os.path.exists(ctx.analysis_file)
def _record_target_stats(self, target, classpath_len, sources_len,
compiletime, is_incremental, stats_key):
def record(k, v):
self.context.run_tracker.report_target_info(
self.options_scope, target, [stats_key, k], v)
record('time', compiletime)
record('classpath_len', classpath_len)
record('sources_len', sources_len)
record('incremental', is_incremental)
def _collect_invalid_compile_dependencies(self, compile_target,
invalid_target_set,
compile_contexts):
# Collects all invalid dependencies that are not dependencies of other invalid dependencies
# within the closure of compile_target.
invalid_dependencies = OrderedSet()
def work(target):
pass
def predicate(target):
if target is compile_target:
return True
if target in invalid_target_set:
invalid_dependencies.add(target)
return self._on_invalid_compile_dependency(
target, compile_target, compile_contexts)
return True
compile_target.walk(work, predicate)
return invalid_dependencies
def _on_invalid_compile_dependency(self, dep, compile_target,
compile_contexts):
"""Decide whether to continue searching for invalid targets to use in the execution graph.
By default, don't recurse because once we have an invalid dependency, we can rely on its
dependencies having been compiled already.
Override to adjust this behavior."""
return False
def _create_context_jar(self, compile_context):
"""Jar up the compile_context to its output jar location.
TODO(stuhood): In the medium term, we hope to add compiler support for this step, which would
allow the jars to be used as compile _inputs_ as well. Currently using jar'd compile outputs as
compile inputs would make the compiler's analysis useless.
see https://github.com/twitter-forks/sbt/tree/stuhood/output-jars
"""
root = compile_context.classes_dir.path
with compile_context.open_jar(mode='w') as jar:
for abs_sub_dir, dirnames, filenames in safe_walk(root):
for name in dirnames + filenames:
abs_filename = os.path.join(abs_sub_dir, name)
arcname = fast_relpath(abs_filename, root)
jar.write(abs_filename, arcname)
def _compute_sources_for_target(self, target):
"""Computes and returns the sources (relative to buildroot) for the given target."""
def resolve_target_sources(target_sources):
resolved_sources = []
for tgt in target_sources:
if tgt.has_sources():
resolved_sources.extend(
tgt.sources_relative_to_buildroot())
return resolved_sources
sources = [
s for s in target.sources_relative_to_buildroot()
if self._sources_predicate(s)
]
# TODO: Make this less hacky. Ideally target.java_sources will point to sources, not targets.
if hasattr(target, 'java_sources') and target.java_sources:
sources.extend(resolve_target_sources(target.java_sources))
return sources
@memoized_property
def _extra_compile_time_classpath(self):
"""Compute any extra compile-time-only classpath elements."""
def extra_compile_classpath_iter():
for conf in self._confs:
for jar in self.extra_compile_time_classpath_elements():
yield (conf, jar)
return list(extra_compile_classpath_iter())
@memoized_method
def _plugin_targets(self, compiler):
"""Returns a map from plugin name to the targets that build that plugin."""
if compiler == 'javac':
plugin_cls = JavacPlugin
elif compiler == 'scalac':
plugin_cls = ScalacPlugin
else:
raise TaskError('Unknown JVM compiler: {}'.format(compiler))
plugin_tgts = self.context.targets(
predicate=lambda t: isinstance(t, plugin_cls))
return {t.plugin: t.closure() for t in plugin_tgts}
@staticmethod
def _local_jvm_distribution(settings=None):
settings_args = [settings] if settings else []
try:
local_distribution = JvmPlatform.preferred_jvm_distribution(
settings_args, strict=True)
except DistributionLocator.Error:
local_distribution = JvmPlatform.preferred_jvm_distribution(
settings_args, strict=False)
return local_distribution
class _HermeticDistribution(object):
def __init__(self, home_path, distribution):
self._underlying = distribution
self._home = home_path
def find_libs(self, names):
underlying_libs = self._underlying.find_libs(names)
return [self._rehome(l) for l in underlying_libs]
def find_libs_path_globs(self, names):
libs_abs = self._underlying.find_libs(names)
libs_unrooted = [self._unroot_lib_path(l) for l in libs_abs]
path_globs = PathGlobsAndRoot(PathGlobs(tuple(libs_unrooted)),
text_type(self._underlying.home))
return (libs_unrooted, path_globs)
@property
def java(self):
return os.path.join(self._home, 'bin', 'java')
@property
def home(self):
return self._home
@property
def underlying_home(self):
return self._underlying.home
def _unroot_lib_path(self, path):
return path[len(self._underlying.home) + 1:]
def _rehome(self, l):
return os.path.join(self._home, self._unroot_lib_path(l))
def _get_jvm_distribution(self):
# TODO We may want to use different jvm distributions depending on what
# java version the target expects to be compiled against.
# See: https://github.com/pantsbuild/pants/issues/6416 for covering using
# different jdks in remote builds.
local_distribution = self._local_jvm_distribution()
return self.execution_strategy_enum.resolve_for_enum_variant({
self.SUBPROCESS:
lambda: local_distribution,
self.NAILGUN:
lambda: local_distribution,
self.HERMETIC:
lambda: self._HermeticDistribution('.jdk', local_distribution),
})()
def _default_work_for_vts(self, vts, ctx, input_classpath_product_key,
counter, all_compile_contexts,
output_classpath_product):
progress_message = ctx.target.address.spec
# Double check the cache before beginning compilation
hit_cache = self.check_cache(vts, counter)
if not hit_cache:
# Compute the compile classpath for this target.
dependency_cp_entries = self._zinc.compile_classpath_entries(
input_classpath_product_key,
ctx.target,
extra_cp_entries=self._extra_compile_time_classpath,
)
upstream_analysis = dict(
self._upstream_analysis(all_compile_contexts,
dependency_cp_entries))
is_incremental = self.should_compile_incrementally(vts, ctx)
if not is_incremental:
# Purge existing analysis file in non-incremental mode.
safe_delete(ctx.analysis_file)
# Work around https://github.com/pantsbuild/pants/issues/3670
safe_rmtree(ctx.classes_dir.path)
dep_context = DependencyContext.global_instance()
tgt, = vts.targets
compiler_option_sets = dep_context.defaulted_property(
tgt, 'compiler_option_sets')
zinc_file_manager = dep_context.defaulted_property(
tgt, 'zinc_file_manager')
with Timer() as timer:
directory_digest = self._compile_vts(
vts, ctx, upstream_analysis, dependency_cp_entries,
progress_message, tgt.platform, compiler_option_sets,
zinc_file_manager, counter)
ctx.classes_dir = ClasspathEntry(ctx.classes_dir.path,
directory_digest)
self._record_target_stats(tgt, len(dependency_cp_entries),
len(ctx.sources), timer.elapsed,
is_incremental, 'compile')
# Write any additional resources for this target to the target workdir.
self.write_extra_resources(ctx)
# Jar the compiled output.
self._create_context_jar(ctx)
# Update the products with the latest classes.
output_classpath_product.add_for_target(
ctx.target,
[(conf, self._classpath_for_context(ctx)) for conf in self._confs],
)
self.register_extra_products_from_contexts([ctx.target],
all_compile_contexts)
class JvmCompile(NailgunTaskBase):
"""A common framework for JVM compilation.
To subclass for a specific JVM language, implement the static values and methods
mentioned below under "Subclasses must implement".
"""
size_estimators = create_size_estimators()
@classmethod
def size_estimator_by_name(cls, estimation_strategy_name):
return cls.size_estimators[estimation_strategy_name]
@staticmethod
def _analysis_for_target(analysis_dir, target):
return os.path.join(analysis_dir, target.id + '.analysis')
@staticmethod
def _portable_analysis_for_target(analysis_dir, target):
return JvmCompile._analysis_for_target(analysis_dir,
target) + '.portable'
@classmethod
def register_options(cls, register):
super(JvmCompile, cls).register_options(register)
register('--args',
advanced=True,
type=list,
default=list(cls.get_args_default(register.bootstrap)),
fingerprint=True,
help='Pass these extra args to the compiler.')
register('--confs',
advanced=True,
type=list,
default=['default'],
help='Compile for these Ivy confs.')
# TODO: Stale analysis should be automatically ignored via Task identities:
# https://github.com/pantsbuild/pants/issues/1351
register(
'--clear-invalid-analysis',
advanced=True,
type=bool,
help=
'When set, any invalid/incompatible analysis files will be deleted '
'automatically. When unset, an error is raised instead.')
register('--warnings',
default=True,
type=bool,
fingerprint=True,
help='Compile with all configured warnings enabled.')
register('--warning-args',
advanced=True,
type=list,
fingerprint=True,
default=list(cls.get_warning_args_default()),
help='Extra compiler args to use when warnings are enabled.')
register('--no-warning-args',
advanced=True,
type=list,
fingerprint=True,
default=list(cls.get_no_warning_args_default()),
help='Extra compiler args to use when warnings are disabled.')
register(
'--fatal-warnings-enabled-args',
advanced=True,
type=list,
fingerprint=True,
default=list(cls.get_fatal_warnings_enabled_args_default()),
help='Extra compiler args to use when fatal warnings are enabled.')
register(
'--fatal-warnings-disabled-args',
advanced=True,
type=list,
fingerprint=True,
default=list(cls.get_fatal_warnings_disabled_args_default()),
help='Extra compiler args to use when fatal warnings are disabled.'
)
register('--debug-symbols',
type=bool,
fingerprint=True,
help='Compile with debug symbol enabled.')
register('--debug-symbol-args',
advanced=True,
type=list,
fingerprint=True,
default=['-C-g:lines,source,vars'],
help='Extra args to enable debug symbol.')
register(
'--delete-scratch',
advanced=True,
default=True,
type=bool,
help=
'Leave intermediate scratch files around, for debugging build problems.'
)
register('--worker-count',
advanced=True,
type=int,
default=cpu_count(),
help='The number of concurrent workers to use when '
'compiling with {task}. Defaults to the '
'current machine\'s CPU count.'.format(task=cls._name))
register(
'--size-estimator',
advanced=True,
choices=list(cls.size_estimators.keys()),
default='filesize',
help=
'The method of target size estimation. The size estimator estimates the size '
'of targets in order to build the largest targets first (subject to dependency '
'constraints). Choose \'random\' to choose random sizes for each target, which '
'may be useful for distributed builds.')
register('--capture-log',
advanced=True,
type=bool,
fingerprint=True,
help='Capture compilation output to per-target logs.')
register(
'--capture-classpath',
advanced=True,
type=bool,
default=True,
fingerprint=True,
help=
'Capture classpath to per-target newline-delimited text files. These files will '
'be packaged into any jar artifacts that are created from the jvm targets.'
)
register(
'--unused-deps',
choices=['ignore', 'warn', 'fatal'],
default='warn',
fingerprint=True,
help=
'Controls whether unused deps are checked, and whether they cause warnings or '
'errors.')
register(
'--use-classpath-jars',
advanced=True,
type=bool,
fingerprint=True,
help=
'Use jar files on the compile_classpath. Note: Using this option degrades '
'incremental compile between targets.')
@classmethod
def prepare(cls, options, round_manager):
super(JvmCompile, cls).prepare(options, round_manager)
round_manager.require_data('compile_classpath')
# Require codegen we care about
# TODO(John Sirois): roll this up in Task - if the list of labels we care about for a target
# predicate to filter the full build graph is exposed, the requirement can be made automatic
# and in turn codegen tasks could denote the labels they produce automating wiring of the
# produce side
round_manager.require_data('java')
round_manager.require_data('scala')
# Allow the deferred_sources_mapping to take place first
round_manager.require_data('deferred_sources')
# Subclasses must implement.
# --------------------------
_name = None
_supports_concurrent_execution = None
@classmethod
def subsystem_dependencies(cls):
return super(JvmCompile, cls).subsystem_dependencies() + (
Java, JvmPlatform, ScalaPlatform)
@classmethod
def name(cls):
return cls._name
@classmethod
def compiler_plugin_types(cls):
"""A tuple of target types which are compiler plugins."""
return ()
@classmethod
def get_args_default(cls, bootstrap_option_values):
"""Override to set default for --args option.
:param bootstrap_option_values: The values of the "bootstrap options" (e.g., pants_workdir).
Implementations can use these when generating the default.
See src/python/pants/options/options_bootstrapper.py for
details.
"""
return ()
@classmethod
def get_warning_args_default(cls):
"""Override to set default for --warning-args option."""
return ()
@classmethod
def get_no_warning_args_default(cls):
"""Override to set default for --no-warning-args option."""
return ()
@classmethod
def get_fatal_warnings_enabled_args_default(cls):
"""Override to set default for --fatal-warnings-enabled-args option."""
return ()
@classmethod
def get_fatal_warnings_disabled_args_default(cls):
"""Override to set default for --fatal-warnings-disabled-args option."""
return ()
@property
def cache_target_dirs(self):
return True
def select(self, target):
raise NotImplementedError()
def select_source(self, source_file_path):
raise NotImplementedError()
def create_analysis_tools(self):
"""Returns an AnalysisTools implementation.
Subclasses must implement.
"""
raise NotImplementedError()
def compile(self, args, classpath, sources, classes_output_dir,
upstream_analysis, analysis_file, log_file, settings,
fatal_warnings, javac_plugins_to_exclude):
"""Invoke the compiler.
Must raise TaskError on compile failure.
Subclasses must implement.
:param list args: Arguments to the compiler (such as jmake or zinc).
:param list classpath: List of classpath entries.
:param list sources: Source files.
:param str classes_output_dir: Where to put the compiled output.
:param upstream_analysis:
:param analysis_file: Where to write the compile analysis.
:param log_file: Where to write logs.
:param JvmPlatformSettings settings: platform settings determining the -source, -target, etc for
javac to use.
:param fatal_warnings: whether to convert compilation warnings to errors.
:param javac_plugins_to_exclude: A list of names of javac plugins that mustn't be used in
this compilation, even if requested (typically because
this compilation is building those same plugins).
"""
raise NotImplementedError()
# Subclasses may override.
# ------------------------
def extra_compile_time_classpath_elements(self):
"""Extra classpath elements common to all compiler invocations.
E.g., jars for compiler plugins.
These are added at the end of the classpath, after any dependencies, so that if they
overlap with any explicit dependencies, the compiler sees those first. This makes
missing dependency accounting much simpler.
"""
return []
def write_extra_resources(self, compile_context):
"""Writes any extra, out-of-band resources for a target to its classes directory.
E.g., targets that produce scala compiler plugins or annotation processor files
produce an info file. The resources will be added to the runtime_classpath.
Returns a list of pairs (root, [absolute paths of files under root]).
"""
pass
def __init__(self, *args, **kwargs):
super(JvmCompile, self).__init__(*args, **kwargs)
self._targets_to_compile_settings = None
# JVM options for running the compiler.
self._jvm_options = self.get_options().jvm_options
self._args = list(self.get_options().args)
if self.get_options().warnings:
self._args.extend(self.get_options().warning_args)
else:
self._args.extend(self.get_options().no_warning_args)
if self.get_options().debug_symbols:
self._args.extend(self.get_options().debug_symbol_args)
# The ivy confs for which we're building.
self._confs = self.get_options().confs
# Determines which sources are relevant to this target.
self._sources_predicate = self.select_source
self._capture_log = self.get_options().capture_log
self._delete_scratch = self.get_options().delete_scratch
self._clear_invalid_analysis = self.get_options(
).clear_invalid_analysis
try:
worker_count = self.get_options().worker_count
except AttributeError:
# tasks that don't support concurrent execution have no worker_count registered
worker_count = 1
self._worker_count = worker_count
self._size_estimator = self.size_estimator_by_name(
self.get_options().size_estimator)
self._analysis_tools = self.create_analysis_tools()
self._dep_context = DependencyContext(
self.compiler_plugin_types(),
dict(include_scopes=Scopes.JVM_COMPILE_SCOPES,
respect_intransitive=True))
@property
def _unused_deps_check_enabled(self):
return self.get_options().unused_deps != 'ignore'
@memoized_property
def _dep_analyzer(self):
return JvmDependencyAnalyzer(
get_buildroot(),
self.context.products.get_data('runtime_classpath'),
self.context.products.get_data('product_deps_by_src'))
@property
def _analysis_parser(self):
return self._analysis_tools.parser
def _fingerprint_strategy(self, classpath_products):
return ResolvedJarAwareTaskIdentityFingerprintStrategy(
self, classpath_products)
def _compile_context(self, target, target_workdir):
analysis_file = JvmCompile._analysis_for_target(target_workdir, target)
portable_analysis_file = JvmCompile._portable_analysis_for_target(
target_workdir, target)
classes_dir = os.path.join(target_workdir, 'classes')
jar_file = os.path.join(target_workdir, 'z.jar')
log_file = os.path.join(target_workdir, 'debug.log')
strict_deps = self._compute_language_property(target,
lambda x: x.strict_deps)
return CompileContext(target, analysis_file, portable_analysis_file,
classes_dir, jar_file, log_file,
self._compute_sources_for_target(target),
strict_deps)
def execute(self):
# In case we have no relevant targets and return early create the requested product maps.
self._create_empty_products()
relevant_targets = list(self.context.targets(predicate=self.select))
if not relevant_targets:
return
# Clone the compile_classpath to the runtime_classpath.
compile_classpath = self.context.products.get_data('compile_classpath')
classpath_product = self.context.products.get_data(
'runtime_classpath', compile_classpath.copy)
def classpath_for_context(context):
if self.get_options().use_classpath_jars:
return context.jar_file
return context.classes_dir
fingerprint_strategy = self._fingerprint_strategy(classpath_product)
# Note, JVM targets are validated (`vts.update()`) as they succeed. As a result,
# we begin writing artifacts out to the cache immediately instead of waiting for
# all targets to finish.
with self.invalidated(relevant_targets,
invalidate_dependents=True,
fingerprint_strategy=fingerprint_strategy,
topological_order=True) as invalidation_check:
# Initialize the classpath for all targets.
compile_contexts = {
vt.target: self._compile_context(vt.target, vt.results_dir)
for vt in invalidation_check.all_vts
}
for cc in compile_contexts.values():
classpath_product.add_for_target(
cc.target, [(conf, classpath_for_context(cc))
for conf in self._confs])
# Register products for valid targets.
valid_targets = [
vt.target for vt in invalidation_check.all_vts if vt.valid
]
self._register_vts([compile_contexts[t] for t in valid_targets])
# Build any invalid targets (which will register products in the background).
if invalidation_check.invalid_vts:
self.do_compile(
invalidation_check,
compile_contexts,
self.extra_compile_time_classpath_elements(),
)
if not self.get_options().use_classpath_jars:
# Once compilation has completed, replace the classpath entry for each target with
# its jar'd representation.
for cc in compile_contexts.values():
for conf in self._confs:
classpath_product.remove_for_target(
cc.target, [(conf, cc.classes_dir)])
classpath_product.add_for_target(
cc.target, [(conf, cc.jar_file)])
def do_compile(self, invalidation_check, compile_contexts,
extra_compile_time_classpath_elements):
"""Executes compilations for the invalid targets contained in a single chunk."""
invalid_targets = [vt.target for vt in invalidation_check.invalid_vts]
assert invalid_targets, "compile_chunk should only be invoked if there are invalid targets."
# This ensures the workunit for the worker pool is set before attempting to compile.
with self.context.new_workunit('isolation-{}-pool-bootstrap'.format(self._name)) \
as workunit:
# This uses workunit.parent as the WorkerPool's parent so that child workunits
# of different pools will show up in order in the html output. This way the current running
# workunit is on the bottom of the page rather than possibly in the middle.
worker_pool = WorkerPool(workunit.parent, self.context.run_tracker,
self._worker_count)
# Prepare the output directory for each invalid target, and confirm that analysis is valid.
for target in invalid_targets:
cc = compile_contexts[target]
safe_mkdir(cc.classes_dir)
self.validate_analysis(cc.analysis_file)
# Get the classpath generated by upstream JVM tasks and our own prepare_compile().
classpath_products = self.context.products.get_data(
'runtime_classpath')
extra_compile_time_classpath = self._compute_extra_classpath(
extra_compile_time_classpath_elements)
# Now create compile jobs for each invalid target one by one.
jobs = self._create_compile_jobs(classpath_products, compile_contexts,
extra_compile_time_classpath,
invalid_targets,
invalidation_check.invalid_vts)
exec_graph = ExecutionGraph(jobs)
try:
exec_graph.execute(worker_pool, self.context.log)
except ExecutionFailure as e:
raise TaskError("Compilation failure: {}".format(e))
def _record_compile_classpath(self, classpath, targets, outdir):
text = '\n'.join(classpath)
for target in targets:
path = os.path.join(outdir, 'compile_classpath',
'{}.txt'.format(target.id))
safe_mkdir(os.path.dirname(path), clean=False)
with open(path, 'w') as f:
f.write(text.encode('utf-8'))
def _compile_vts(self, vts, sources, analysis_file, upstream_analysis,
classpath, outdir, log_file, progress_message, settings,
fatal_warnings, counter):
"""Compiles sources for the given vts into the given output dir.
vts - versioned target set
sources - sources for this target set
analysis_file - the analysis file to manipulate
classpath - a list of classpath entries
outdir - the output dir to send classes to
May be invoked concurrently on independent target sets.
Postcondition: The individual targets in vts are up-to-date, as if each were
compiled individually.
"""
if not sources:
self.context.log.warn(
'Skipping {} compile for targets with no sources:\n {}'.
format(self.name(), vts.targets))
else:
counter_val = str(counter()).rjust(counter.format_length(), b' ')
counter_str = '[{}/{}] '.format(counter_val, counter.size)
# Do some reporting.
self.context.log.info(
counter_str, 'Compiling ',
items_to_report_element(sources,
'{} source'.format(self.name())),
' in ',
items_to_report_element(
[t.address.reference() for t in vts.targets], 'target'),
' (', progress_message, ').')
with self.context.new_workunit('compile',
labels=[WorkUnitLabel.COMPILER]):
# The compiler may delete classfiles, then later exit on a compilation error. Then if the
# change triggering the error is reverted, we won't rebuild to restore the missing
# classfiles. So we force-invalidate here, to be on the safe side.
vts.force_invalidate()
if self.get_options().capture_classpath:
self._record_compile_classpath(classpath, vts.targets,
outdir)
# If compiling a plugin, don't try to use it on itself.
javac_plugins_to_exclude = (t.plugin for t in vts.targets
if isinstance(t, JavacPlugin))
self.compile(self._args, classpath, sources, outdir,
upstream_analysis, analysis_file, log_file,
settings, fatal_warnings,
javac_plugins_to_exclude)
def check_artifact_cache(self, vts):
"""Localizes the fetched analysis for targets we found in the cache."""
def post_process(cached_vts):
for vt in cached_vts:
cc = self._compile_context(vt.target, vt.results_dir)
safe_delete(cc.analysis_file)
self._analysis_tools.localize(cc.portable_analysis_file,
cc.analysis_file)
return self.do_check_artifact_cache(
vts, post_process_cached_vts=post_process)
def _create_empty_products(self):
if self.context.products.is_required_data('classes_by_source'):
make_products = lambda: defaultdict(MultipleRootedProducts)
self.context.products.safe_create_data('classes_by_source',
make_products)
if self.context.products.is_required_data('product_deps_by_src') \
or self._unused_deps_check_enabled:
self.context.products.safe_create_data('product_deps_by_src', dict)
def compute_classes_by_source(self, compile_contexts):
"""Compute a map of (context->(src->classes)) for the given compile_contexts.
It's possible (although unfortunate) for multiple targets to own the same sources, hence
the top level division. Srcs are relative to buildroot. Classes are absolute paths.
Returning classes with 'None' as their src indicates that the compiler analysis indicated
that they were un-owned. This case is triggered when annotation processors generate
classes (or due to bugs in classfile tracking in zinc/jmake.)
"""
buildroot = get_buildroot()
# Build a mapping of srcs to classes for each context.
classes_by_src_by_context = defaultdict(dict)
for compile_context in compile_contexts:
# Walk the context's jar to build a set of unclaimed classfiles.
unclaimed_classes = set()
with compile_context.open_jar(mode='r') as jar:
for name in jar.namelist():
if not name.endswith('/'):
unclaimed_classes.add(
os.path.join(compile_context.classes_dir, name))
# Grab the analysis' view of which classfiles were generated.
classes_by_src = classes_by_src_by_context[compile_context]
if os.path.exists(compile_context.analysis_file):
products = self._analysis_parser.parse_products_from_path(
compile_context.analysis_file, compile_context.classes_dir)
for src, classes in products.items():
relsrc = os.path.relpath(src, buildroot)
classes_by_src[relsrc] = classes
unclaimed_classes.difference_update(classes)
# Any remaining classfiles were unclaimed by sources/analysis.
classes_by_src[None] = list(unclaimed_classes)
return classes_by_src_by_context
def classname_for_classfile(self, compile_context, class_file_name):
assert class_file_name.startswith(compile_context.classes_dir)
rel_classfile_path = class_file_name[len(compile_context.classes_dir) +
1:]
return ClasspathUtil.classname_for_rel_classfile(rel_classfile_path)
def _register_vts(self, compile_contexts):
classes_by_source = self.context.products.get_data('classes_by_source')
product_deps_by_src = self.context.products.get_data(
'product_deps_by_src')
# Register a mapping between sources and classfiles (if requested).
if classes_by_source is not None:
ccbsbc = self.compute_classes_by_source(compile_contexts).items()
for compile_context, computed_classes_by_source in ccbsbc:
classes_dir = compile_context.classes_dir
for source in compile_context.sources:
classes = computed_classes_by_source.get(source, [])
classes_by_source[source].add_abs_paths(
classes_dir, classes)
# Register classfile product dependencies (if requested).
if product_deps_by_src is not None:
for compile_context in compile_contexts:
product_deps_by_src[compile_context.target] = \
self._analysis_parser.parse_deps_from_path(compile_context.analysis_file)
def _check_unused_deps(self, compile_context):
"""Uses `product_deps_by_src` to check unused deps and warn or error."""
with self.context.new_workunit('unused-check',
labels=[WorkUnitLabel.COMPILER]):
# Compute replacement deps.
replacement_deps = self._dep_analyzer.compute_unused_deps(
compile_context.target)
if not replacement_deps:
return
# Warn or error for unused.
def joined_dep_msg(deps):
return '\n '.join('\'{}\','.format(dep.address.spec)
for dep in sorted(deps))
flat_replacements = set(
r for replacements in replacement_deps.values()
for r in replacements)
replacements_msg = ''
if flat_replacements:
replacements_msg = 'Suggested replacements:\n {}\n'.format(
joined_dep_msg(flat_replacements))
unused_msg = (
'unused dependencies:\n {}\n{}'
'(If you\'re seeing this message in error, you might need to '
'change the `scope` of the dependencies.)'.format(
joined_dep_msg(replacement_deps.keys()),
replacements_msg,
))
if self.get_options().unused_deps == 'fatal':
raise TaskError(unused_msg)
else:
self.context.log.warn('Target {} had {}\n'.format(
compile_context.target.address.spec, unused_msg))
def _upstream_analysis(self, compile_contexts, classpath_entries):
"""Returns tuples of classes_dir->analysis_file for the closure of the target."""
# Reorganize the compile_contexts by class directory.
compile_contexts_by_directory = {}
for compile_context in compile_contexts.values():
compile_contexts_by_directory[
compile_context.classes_dir] = compile_context
# If we have a compile context for the target, include it.
for entry in classpath_entries:
if not entry.endswith('.jar'):
compile_context = compile_contexts_by_directory.get(entry)
if not compile_context:
self.context.log.debug(
'Missing upstream analysis for {}'.format(entry))
else:
yield compile_context.classes_dir, compile_context.analysis_file
def exec_graph_key_for_target(self, compile_target):
return "compile({})".format(compile_target.address.spec)
def _create_compile_jobs(self, classpath_products, compile_contexts,
extra_compile_time_classpath, invalid_targets,
invalid_vts):
class Counter(object):
def __init__(self, size, initial=0):
self.size = size
self.count = initial
def __call__(self):
self.count += 1
return self.count
def format_length(self):
return len(str(self.size))
counter = Counter(len(invalid_vts))
def check_cache(vts):
"""Manually checks the artifact cache (usually immediately before compilation.)
Returns true if the cache was hit successfully, indicating that no compilation is necessary.
"""
if not self.artifact_cache_reads_enabled():
return False
cached_vts, _, _ = self.check_artifact_cache([vts])
if not cached_vts:
self.context.log.debug(
'Missed cache during double check for {}'.format(
vts.target.address.spec))
return False
assert cached_vts == [
vts
], ('Cache returned unexpected target: {} vs {}'.format(
cached_vts, [vts]))
self.context.log.info(
'Hit cache during double check for {}'.format(
vts.target.address.spec))
counter()
return True
def should_compile_incrementally(vts, ctx):
"""Check to see if the compile should try to re-use the existing analysis.
Returns true if we should try to compile the target incrementally.
"""
if not vts.is_incremental:
return False
if not self._clear_invalid_analysis:
return True
return os.path.exists(ctx.analysis_file)
def work_for_vts(vts, ctx):
progress_message = ctx.target.address.spec
# Capture a compilation log if requested.
log_file = ctx.log_file if self._capture_log else None
# Double check the cache before beginning compilation
hit_cache = check_cache(vts)
if not hit_cache:
# Compute the compile classpath for this target.
cp_entries = [ctx.classes_dir]
cp_entries.extend(
ClasspathUtil.compute_classpath(
ctx.dependencies(self._dep_context),
classpath_products, extra_compile_time_classpath,
self._confs))
upstream_analysis = dict(
self._upstream_analysis(compile_contexts, cp_entries))
if not should_compile_incrementally(vts, ctx):
# Purge existing analysis file in non-incremental mode.
safe_delete(ctx.analysis_file)
# Work around https://github.com/pantsbuild/pants/issues/3670
safe_rmtree(ctx.classes_dir)
tgt, = vts.targets
fatal_warnings = self._compute_language_property(
tgt, lambda x: x.fatal_warnings)
self._compile_vts(vts, ctx.sources, ctx.analysis_file,
upstream_analysis, cp_entries,
ctx.classes_dir, log_file, progress_message,
tgt.platform, fatal_warnings, counter)
self._analysis_tools.relativize(ctx.analysis_file,
ctx.portable_analysis_file)
# Write any additional resources for this target to the target workdir.
self.write_extra_resources(ctx)
# Jar the compiled output.
self._create_context_jar(ctx)
# Update the products with the latest classes.
self._register_vts([ctx])
# Once products are registered, check for unused dependencies (if enabled).
if not hit_cache and self._unused_deps_check_enabled:
self._check_unused_deps(ctx)
jobs = []
invalid_target_set = set(invalid_targets)
for ivts in invalid_vts:
# Invalidated targets are a subset of relevant targets: get the context for this one.
compile_target = ivts.target
compile_context = compile_contexts[compile_target]
invalid_dependencies = self._collect_invalid_compile_dependencies(
compile_target, compile_contexts, invalid_target_set)
jobs.append(
Job(
self.exec_graph_key_for_target(compile_target),
functools.partial(work_for_vts, ivts, compile_context),
[
self.exec_graph_key_for_target(target)
for target in invalid_dependencies
],
self._size_estimator(compile_context.sources),
# If compilation and analysis work succeeds, validate the vts.
# Otherwise, fail it.
on_success=ivts.update,
on_failure=ivts.force_invalidate))
return jobs
def _collect_invalid_compile_dependencies(self, compile_target,
compile_contexts,
invalid_target_set):
# Collects just the direct invalid compile dependencies, traversing non-compile targets.
invalid_dependencies = OrderedSet()
def work(target):
pass
def predicate(target):
if target in compile_contexts and target is not compile_target:
if target in invalid_target_set:
invalid_dependencies.add(target)
return False
else:
return True
compile_target.walk(work, predicate)
return invalid_dependencies
def _create_context_jar(self, compile_context):
"""Jar up the compile_context to its output jar location.
TODO(stuhood): In the medium term, we hope to add compiler support for this step, which would
allow the jars to be used as compile _inputs_ as well. Currently using jar'd compile outputs as
compile inputs would make the compiler's analysis useless.
see https://github.com/twitter-forks/sbt/tree/stuhood/output-jars
"""
root = compile_context.classes_dir
with compile_context.open_jar(mode='w') as jar:
for abs_sub_dir, dirnames, filenames in safe_walk(root):
for name in dirnames + filenames:
abs_filename = os.path.join(abs_sub_dir, name)
arcname = fast_relpath(abs_filename, root)
jar.write(abs_filename, arcname)
def validate_analysis(self, path):
"""Throws a TaskError for invalid analysis files."""
try:
self._analysis_parser.validate_analysis(path)
except Exception as e:
if self._clear_invalid_analysis:
self.context.log.warn(
"Invalid analysis detected at path {} ... pants will remove these "
"automatically, but\nyou may experience spurious warnings until "
"clean-all is executed.\n{}".format(path, e))
safe_delete(path)
else:
raise TaskError(
"An internal build directory contains invalid/mismatched analysis: please "
"run `clean-all` if your tools versions changed recently:\n{}"
.format(e))
def _compute_sources_for_target(self, target):
"""Computes and returns the sources (relative to buildroot) for the given target."""
def resolve_target_sources(target_sources):
resolved_sources = []
for tgt in target_sources:
if tgt.has_sources():
resolved_sources.extend(
tgt.sources_relative_to_buildroot())
return resolved_sources
sources = [
s for s in target.sources_relative_to_buildroot()
if self._sources_predicate(s)
]
# TODO: Make this less hacky. Ideally target.java_sources will point to sources, not targets.
if hasattr(target, 'java_sources') and target.java_sources:
sources.extend(resolve_target_sources(target.java_sources))
return sources
def _compute_language_property(self, target, selector):
"""Computes the a language property setting for the given target sources.
:param target The target whose language property will be calculated.
:param selector A function that takes a target or platform and returns the boolean value of the
property for that target or platform, or None if that target or platform does
not directly define the property.
If the target does not override the language property, returns true iff the property
is true for any of the matched languages for the target.
"""
if selector(target) is not None:
return selector(target)
prop = False
if target.has_sources('.java'):
prop |= selector(Java.global_instance())
if target.has_sources('.scala'):
prop |= selector(ScalaPlatform.global_instance())
return prop
def _compute_extra_classpath(self, extra_compile_time_classpath_elements):
"""Compute any extra compile-time-only classpath elements.
TODO(benjy): Model compile-time vs. runtime classpaths more explicitly.
"""
def extra_compile_classpath_iter():
for conf in self._confs:
for jar in extra_compile_time_classpath_elements:
yield (conf, jar)
return list(extra_compile_classpath_iter())
def test_line_count_estimator(self):
with temporary_file_path() as src:
self.assertEqual(create_size_estimators()['linecount']([src]), 0)
class JvmDependencyUsage(Task):
"""Determines the dependency usage ratios of targets.
Analyzes the relationship between the products a target T produces vs. the products
which T's dependents actually require (this is done by observing analysis files).
If the ratio of required products to available products is low, then this is a sign
that target T isn't factored well.
A graph is formed from these results, where each node of the graph is a target, and
each edge is a product usage ratio between a target and its dependency. The nodes
also contain additional information to guide refactoring -- for example, the estimated
job size of each target, which indicates the impact a poorly factored target has on
the build times. (see DependencyUsageGraph->to_json)
The graph is either summarized for local analysis or outputted as a JSON file for
aggregation and analysis on a larger scale.
"""
size_estimators = create_size_estimators()
@classmethod
def register_options(cls, register):
super(JvmDependencyUsage, cls).register_options(register)
register(
'--internal-only',
default=False,
type=bool,
fingerprint=True,
help=
'Specifies that only internal dependencies should be included in the graph '
'output (no external jars).')
register(
'--summary',
default=True,
type=bool,
help=
'When set, outputs a summary of the "worst" dependencies; otherwise, '
'outputs a JSON report.')
register('--size-estimator',
choices=list(cls.size_estimators.keys()),
default='filesize',
fingerprint=True,
help='The method of target size estimation.')
register('--transitive',
default=True,
type=bool,
help='Score all targets in the build graph transitively.')
register('--output-file',
type=str,
help='Output destination. When unset, outputs to <stdout>.')
register(
'--use-cached',
type=bool,
help='Use cached dependency data to compute analysis result. '
'When set, skips `resolve` and `compile` steps. '
'Useful for computing analysis for a lot of targets, but '
'result can differ from direct execution because cached information '
'doesn\'t depend on 3rdparty libraries versions.')
@classmethod
def subsystem_dependencies(cls):
return super(JvmDependencyUsage,
cls).subsystem_dependencies() + (DistributionLocator, )
@classmethod
def prepare(cls, options, round_manager):
super(JvmDependencyUsage, cls).prepare(options, round_manager)
if not options.use_cached:
round_manager.require_data('classes_by_source')
round_manager.require_data('runtime_classpath')
round_manager.require_data('product_deps_by_src')
else:
# We want to have synthetic targets in build graph to deserialize nodes properly.
round_manager.optional_data('java')
round_manager.optional_data('scala')
round_manager.optional_data('deferred_sources')
@classmethod
def skip(cls, options):
"""This task is always explicitly requested."""
return False
def execute(self):
graph = self.create_dep_usage_graph(self.context.targets(
) if self.get_options().transitive else self.context.target_roots)
output_file = self.get_options().output_file
if output_file:
self.context.log.info(
'Writing dependency usage to {}'.format(output_file))
with open(output_file, 'w') as fh:
self._render(graph, fh)
else:
sys.stdout.write('\n')
self._render(graph, sys.stdout)
@classmethod
def implementation_version(cls):
return super(JvmDependencyUsage, cls).implementation_version() + [
('JvmDependencyUsage', 7)
]
def _render(self, graph, fh):
chunks = graph.to_summary() if self.get_options(
).summary else graph.to_json()
for chunk in chunks:
fh.write(chunk)
fh.flush()
def _dep_type(self, target, dep, declared_deps, eligible_unused_deps,
is_used):
"""Returns a tuple of a 'declared'/'undeclared' boolean, and 'used'/'unused' boolean.
These values are related, because some declared deps are not eligible to be considered unused.
:param target: The source target.
:param dep: The dependency to compute a type for.
:param declared_deps: The declared dependencies of the target.
:param eligible_unused_deps: The declared dependencies of the target that are eligible
to be considered unused; this is generally only 'DEFAULT' scoped dependencies.
:param is_used: True if the dep was actually used at compile time.
"""
if target == dep:
return True, True
return (dep in declared_deps), (is_used
or dep not in eligible_unused_deps)
def _select(self, target):
if self.get_options().internal_only and isinstance(target, JarLibrary):
return False
elif isinstance(target,
Resources) or type(target) in (AliasTarget, Target):
return False
else:
return True
def create_dep_usage_graph(self, targets):
"""Creates a graph of concrete targets, with their sum of products and dependencies.
Synthetic targets contribute products and dependencies to their concrete target.
"""
with self.invalidated(
targets, invalidate_dependents=True) as invalidation_check:
target_to_vts = {}
for vts in invalidation_check.all_vts:
target_to_vts[vts.target] = vts
if not self.get_options().use_cached:
node_creator = self.calculating_node_creator(target_to_vts)
else:
node_creator = self.cached_node_creator(target_to_vts)
return DependencyUsageGraph(
self.create_dep_usage_nodes(targets, node_creator),
self.size_estimators[self.get_options().size_estimator])
@memoized_property
def _analyzer(self):
return JvmDependencyAnalyzer(
get_buildroot(), DistributionLocator.cached(),
self.context.products.get_data('runtime_classpath'))
def calculating_node_creator(self, target_to_vts):
"""Strategy directly computes dependency graph node based on
`classes_by_source`, `runtime_classpath`, `product_deps_by_src` parameters and
stores the result to the build cache.
"""
targets = self.context.targets()
targets_by_file = self._analyzer.targets_by_file(targets)
transitive_deps_by_target = self._analyzer.compute_transitive_deps_by_target(
targets)
def creator(target):
transitive_deps = set(transitive_deps_by_target.get(target))
node = self.create_dep_usage_node(target, targets_by_file,
transitive_deps)
vt = target_to_vts[target]
mode = 'w' if PY3 else 'wb'
with open(self.nodes_json(vt.results_dir), mode=mode) as fp:
json.dump(node.to_cacheable_dict(),
fp,
indent=2,
sort_keys=True)
vt.update()
return node
return creator
def cached_node_creator(self, target_to_vts):
"""Strategy restores dependency graph node from the build cache.
"""
def creator(target):
vt = target_to_vts[target]
if vt.valid and os.path.exists(self.nodes_json(vt.results_dir)):
try:
with open(self.nodes_json(vt.results_dir), 'r') as fp:
return Node.from_cacheable_dict(
json.load(fp), lambda spec: next(
self.context.resolve(spec).__iter__()))
except Exception:
self.context.log.warn(
"Can't deserialize json for target {}".format(target))
return Node(target.concrete_derived_from)
else:
self.context.log.warn("No cache entry for {}".format(target))
return Node(target.concrete_derived_from)
return creator
def nodes_json(self, target_results_dir):
return os.path.join(target_results_dir, 'node.json')
def create_dep_usage_nodes(self, targets, node_creator):
nodes = {}
for target in targets:
if not self._select(target):
continue
# Create or extend a Node for the concrete version of this target.
concrete_target = target.concrete_derived_from
node = node_creator(target)
if concrete_target in nodes:
nodes[concrete_target].combine(node)
else:
nodes[concrete_target] = node
# Prune any Nodes with 0 products.
for concrete_target, node in list(
nodes.items()): # copy because mutation
if node.products_total == 0:
nodes.pop(concrete_target)
return nodes
def cache_target_dirs(self):
return True
def create_dep_usage_node(self, target, targets_by_file, transitive_deps):
declared_deps_with_aliases = set(
self._analyzer.resolve_aliases(target))
eligible_unused_deps = {
d
for d, _ in self._analyzer.resolve_aliases(target,
scope=Scopes.DEFAULT)
}
concrete_target = target.concrete_derived_from
declared_deps = [
resolved for resolved, _ in declared_deps_with_aliases
]
products_total = self._analyzer.count_products(target)
node = Node(concrete_target)
node.add_derivation(target, products_total)
def _construct_edge(dep_tgt, products_used):
is_declared, is_used = self._dep_type(target, dep_tgt,
declared_deps,
eligible_unused_deps,
len(products_used) > 0)
return Edge(is_declared=is_declared,
is_used=is_used,
products_used=products_used)
# Record declared Edges, initially all as "unused" or "declared".
for dep_tgt, aliased_from in declared_deps_with_aliases:
derived_from = dep_tgt.concrete_derived_from
if self._select(derived_from):
node.add_edge(_construct_edge(dep_tgt, products_used=set()),
derived_from, aliased_from)
# Record the used products and undeclared Edges for this target. Note that some of
# these may be self edges, which are considered later.
product_deps_by_src = self.context.products.get_data(
'product_deps_by_src')
target_product_deps_by_src = product_deps_by_src.get(target, {})
for product_deps in target_product_deps_by_src.values():
for product_dep in product_deps:
for dep_tgt in targets_by_file.get(product_dep, []):
derived_from = dep_tgt.concrete_derived_from
if not self._select(derived_from):
continue
# Create edge only for those direct or transitive dependencies in order to
# disqualify irrelevant targets that happen to share some file in sources,
# not uncommon when globs especially rglobs is used.
if not derived_from in transitive_deps:
continue
node.add_edge(
_construct_edge(dep_tgt, products_used={product_dep}),
derived_from)
return node
class JvmCompileIsolatedStrategy(JvmCompileStrategy):
"""A strategy for JVM compilation that uses per-target classpaths and analysis."""
size_estimators = create_size_estimators()
@classmethod
def size_estimator_by_name(cls, estimation_strategy_name):
return cls.size_estimators[estimation_strategy_name]
@classmethod
def register_options(cls, register, compile_task_name,
supports_concurrent_execution):
if supports_concurrent_execution:
register(
'--worker-count',
advanced=True,
type=int,
default=1,
help=
'The number of concurrent workers to use compiling with {task} with the '
'isolated strategy.'.format(task=compile_task_name))
register('--size-estimator',
advanced=True,
choices=list(cls.size_estimators.keys()),
default='filesize',
help='The method of target size estimation.')
register('--capture-log',
advanced=True,
action='store_true',
default=False,
fingerprint=True,
help='Capture compilation output to per-target logs.')
def __init__(self, context, options, workdir, analysis_tools,
compile_task_name, sources_predicate):
super(JvmCompileIsolatedStrategy,
self).__init__(context, options, workdir, analysis_tools,
compile_task_name, sources_predicate)
# Various working directories.
self._analysis_dir = os.path.join(workdir, 'isolated-analysis')
self._classes_dir = os.path.join(workdir, 'isolated-classes')
self._logs_dir = os.path.join(workdir, 'isolated-logs')
self._jars_dir = os.path.join(workdir, 'jars')
self._capture_log = options.capture_log
try:
worker_count = options.worker_count
except AttributeError:
# tasks that don't support concurrent execution have no worker_count registered
worker_count = 1
self._worker_count = worker_count
self._size_estimator = self.size_estimator_by_name(
options.size_estimator)
self._worker_pool = None
def name(self):
return 'isolated'
def compile_context(self, target):
analysis_file = JvmCompileStrategy._analysis_for_target(
self._analysis_dir, target)
classes_dir = os.path.join(self._classes_dir, target.id)
# Generate a short unique path for the jar to allow for shorter classpaths.
# TODO: likely unnecessary after https://github.com/pantsbuild/pants/issues/1988
jar_file = os.path.join(
self._jars_dir, '{}.jar'.format(sha1(target.id).hexdigest()[:12]))
return IsolatedCompileContext(target, analysis_file,
classes_dir, jar_file,
self._sources_for_target(target))
def _create_compile_contexts_for_targets(self, targets):
compile_contexts = OrderedDict()
for target in targets:
compile_context = self.compile_context(target)
compile_contexts[target] = compile_context
return compile_contexts
def pre_compile(self):
super(JvmCompileIsolatedStrategy, self).pre_compile()
safe_mkdir(self._analysis_dir)
safe_mkdir(self._classes_dir)
safe_mkdir(self._logs_dir)
safe_mkdir(self._jars_dir)
def prepare_compile(self, cache_manager, all_targets, relevant_targets):
super(JvmCompileIsolatedStrategy,
self).prepare_compile(cache_manager, all_targets,
relevant_targets)
# Update the classpath by adding relevant target's classes directories to its classpath.
compile_classpaths = self.context.products.get_data(
'compile_classpath')
with self.context.new_workunit('validate-{}-analysis'.format(
self._compile_task_name)):
for target in relevant_targets:
cc = self.compile_context(target)
safe_mkdir(cc.classes_dir)
compile_classpaths.add_for_target(target,
[(conf, cc.classes_dir)
for conf in self._confs])
self.validate_analysis(cc.analysis_file)
# This ensures the workunit for the worker pool is set
with self.context.new_workunit('isolation-{}-pool-bootstrap'.format(self._compile_task_name)) \
as workunit:
# This uses workunit.parent as the WorkerPool's parent so that child workunits
# of different pools will show up in order in the html output. This way the current running
# workunit is on the bottom of the page rather than possibly in the middle.
self._worker_pool = WorkerPool(workunit.parent,
self.context.run_tracker,
self._worker_count)
def finalize_compile(self, targets):
# Replace the classpath entry for each target with its jar'd representation.
compile_classpaths = self.context.products.get_data(
'compile_classpath')
for target in targets:
cc = self.compile_context(target)
for conf in self._confs:
compile_classpaths.remove_for_target(target,
[(conf, cc.classes_dir)])
compile_classpaths.add_for_target(target,
[(conf, cc.jar_file)])
def invalidation_hints(self, relevant_targets):
# No partitioning.
return (0, None)
def compute_classes_by_source(self, compile_contexts):
buildroot = get_buildroot()
# Build a mapping of srcs to classes for each context.
classes_by_src_by_context = defaultdict(dict)
for compile_context in compile_contexts:
# Walk the context's jar to build a set of unclaimed classfiles.
unclaimed_classes = set()
with compile_context.open_jar(mode='r') as jar:
for name in jar.namelist():
unclaimed_classes.add(
os.path.join(compile_context.classes_dir, name))
# Grab the analysis' view of which classfiles were generated.
classes_by_src = classes_by_src_by_context[compile_context]
if os.path.exists(compile_context.analysis_file):
products = self._analysis_parser.parse_products_from_path(
compile_context.analysis_file, compile_context.classes_dir)
for src, classes in products.items():
relsrc = os.path.relpath(src, buildroot)
classes_by_src[relsrc] = classes
unclaimed_classes.difference_update(classes)
# Any remaining classfiles were unclaimed by sources/analysis.
classes_by_src[None] = list(unclaimed_classes)
return classes_by_src_by_context
def _compute_classpath_entries(self, compile_classpaths, target_closure,
compile_context,
extra_compile_time_classpath):
# Generate a classpath specific to this compile and target.
return ClasspathUtil.compute_classpath_for_target(
compile_context.target, compile_classpaths,
extra_compile_time_classpath, self._confs, target_closure)
def _upstream_analysis(self, compile_contexts, classpath_entries):
"""Returns tuples of classes_dir->analysis_file for the closure of the target."""
# Reorganize the compile_contexts by class directory.
compile_contexts_by_directory = {}
for compile_context in compile_contexts.values():
compile_contexts_by_directory[
compile_context.classes_dir] = compile_context
# If we have a compile context for the target, include it.
for entry in classpath_entries:
if not entry.endswith('.jar'):
compile_context = compile_contexts_by_directory.get(entry)
if not compile_context:
self.context.log.debug(
'Missing upstream analysis for {}'.format(entry))
else:
yield compile_context.classes_dir, compile_context.analysis_file
def _capture_log_file(self, target):
if self._capture_log:
return os.path.join(self._logs_dir, "{}.log".format(target.id))
return None
def exec_graph_key_for_target(self, compile_target):
return "compile({})".format(compile_target.address.spec)
def _create_compile_jobs(self, compile_classpaths, compile_contexts,
extra_compile_time_classpath, invalid_targets,
invalid_vts_partitioned, compile_vts,
register_vts, update_artifact_cache_vts_work):
def create_work_for_vts(vts, compile_context, target_closure):
def work():
progress_message = compile_context.target.address.spec
cp_entries = self._compute_classpath_entries(
compile_classpaths, target_closure, compile_context,
extra_compile_time_classpath)
upstream_analysis = dict(
self._upstream_analysis(compile_contexts, cp_entries))
# Capture a compilation log if requested.
log_file = self._capture_log_file(compile_context.target)
# Mutate analysis within a temporary directory, and move it to the final location
# on success.
tmpdir = os.path.join(self.analysis_tmpdir,
compile_context.target.id)
safe_mkdir(tmpdir)
tmp_analysis_file = JvmCompileStrategy._analysis_for_target(
tmpdir, compile_context.target)
if os.path.exists(compile_context.analysis_file):
shutil.copy(compile_context.analysis_file,
tmp_analysis_file)
target, = vts.targets
compile_vts(vts, compile_context.sources, tmp_analysis_file,
upstream_analysis, cp_entries,
compile_context.classes_dir, log_file,
progress_message, target.platform)
atomic_copy(tmp_analysis_file, compile_context.analysis_file)
# Jar the compiled output.
self._create_context_jar(compile_context)
# Update the products with the latest classes.
register_vts([compile_context])
# Kick off the background artifact cache write.
if update_artifact_cache_vts_work:
self._write_to_artifact_cache(
vts, compile_context, update_artifact_cache_vts_work)
return work
jobs = []
invalid_target_set = set(invalid_targets)
for vts in invalid_vts_partitioned:
assert len(vts.targets) == 1, (
"Requested one target per partition, got {}".format(vts))
# Invalidated targets are a subset of relevant targets: get the context for this one.
compile_target = vts.targets[0]
compile_context = compile_contexts[compile_target]
compile_target_closure = compile_target.closure()
# dependencies of the current target which are invalid for this chunk
invalid_dependencies = (compile_target_closure
& invalid_target_set) - [compile_target]
jobs.append(
Job(
self.exec_graph_key_for_target(compile_target),
create_work_for_vts(vts, compile_context,
compile_target_closure),
[
self.exec_graph_key_for_target(target)
for target in invalid_dependencies
],
self._size_estimator(compile_context.sources),
# If compilation and analysis work succeeds, validate the vts.
# Otherwise, fail it.
on_success=vts.update,
on_failure=vts.force_invalidate))
return jobs
def compile_chunk(self, invalidation_check, all_targets, relevant_targets,
invalid_targets, extra_compile_time_classpath_elements,
compile_vts, register_vts,
update_artifact_cache_vts_work):
"""Executes compilations for the invalid targets contained in a single chunk."""
assert invalid_targets, "compile_chunk should only be invoked if there are invalid targets."
# Get the classpath generated by upstream JVM tasks and our own prepare_compile().
compile_classpaths = self.context.products.get_data(
'compile_classpath')
extra_compile_time_classpath = self._compute_extra_classpath(
extra_compile_time_classpath_elements)
compile_contexts = self._create_compile_contexts_for_targets(
all_targets)
# Now create compile jobs for each invalid target one by one.
jobs = self._create_compile_jobs(
compile_classpaths, compile_contexts, extra_compile_time_classpath,
invalid_targets, invalidation_check.invalid_vts_partitioned,
compile_vts, register_vts, update_artifact_cache_vts_work)
exec_graph = ExecutionGraph(jobs)
try:
exec_graph.execute(self._worker_pool, self.context.log)
except ExecutionFailure as e:
raise TaskError("Compilation failure: {}".format(e))
def compute_resource_mapping(self, compile_contexts):
return ResourceMapping(self._classes_dir)
def post_process_cached_vts(self, cached_vts):
"""Localizes the fetched analysis for targets we found in the cache.
This is the complement of `_write_to_artifact_cache`.
"""
compile_contexts = []
for vt in cached_vts:
for target in vt.targets:
compile_contexts.append(self.compile_context(target))
for compile_context in compile_contexts:
portable_analysis_file = JvmCompileStrategy._portable_analysis_for_target(
self._analysis_dir, compile_context.target)
if os.path.exists(portable_analysis_file):
self._analysis_tools.localize(portable_analysis_file,
compile_context.analysis_file)
def _create_context_jar(self, compile_context):
"""Jar up the compile_context to its output jar location.
TODO(stuhood): In the medium term, we hope to add compiler support for this step, which would
allow the jars to be used as compile _inputs_ as well. Currently using jar'd compile outputs as
compile inputs would make the compiler's analysis useless.
see https://github.com/twitter-forks/sbt/tree/stuhood/output-jars
"""
root = compile_context.classes_dir
with compile_context.open_jar(mode='w') as jar:
for abs_sub_dir, _, filenames in safe_walk(root):
for name in filenames:
abs_filename = os.path.join(abs_sub_dir, name)
arcname = os.path.relpath(abs_filename, root)
jar.write(abs_filename, arcname)
def _write_to_artifact_cache(self, vts, compile_context,
get_update_artifact_cache_work):
assert len(vts.targets) == 1
assert vts.targets[0] == compile_context.target
# Noop if the target is uncacheable.
if (compile_context.target.has_label('no_cache')):
return
vt = vts.versioned_targets[0]
# Set up args to relativize analysis in the background.
portable_analysis_file = JvmCompileStrategy._portable_analysis_for_target(
self._analysis_dir, compile_context.target)
relativize_args_tuple = (compile_context.analysis_file,
portable_analysis_file)
# Collect the artifacts for this target.
artifacts = []
def add_abs_products(p):
if p:
for _, paths in p.abs_paths():
artifacts.extend(paths)
# Resources.
resources_by_target = self.context.products.get_data(
'resources_by_target')
add_abs_products(resources_by_target.get(compile_context.target))
# Classes.
classes_by_target = self.context.products.get_data('classes_by_target')
add_abs_products(classes_by_target.get(compile_context.target))
# Log file.
log_file = self._capture_log_file(compile_context.target)
if log_file and os.path.exists(log_file):
artifacts.append(log_file)
# Jar.
artifacts.append(compile_context.jar_file)
# Get the 'work' that will publish these artifacts to the cache.
# NB: the portable analysis_file won't exist until we finish.
vts_artifactfiles_pair = (vt, artifacts + [portable_analysis_file])
update_artifact_cache_work = get_update_artifact_cache_work(
[vts_artifactfiles_pair])
# And execute it.
if update_artifact_cache_work:
work_chain = [
Work(self._analysis_tools.relativize, [relativize_args_tuple],
'relativize'), update_artifact_cache_work
]
self.context.submit_background_work_chain(
work_chain, parent_workunit_name='cache')
class JvmCompile(NailgunTaskBase, GroupMember):
"""A common framework for JVM compilation.
To subclass for a specific JVM language, implement the static values and methods
mentioned below under "Subclasses must implement".
"""
size_estimators = create_size_estimators()
@classmethod
def size_estimator_by_name(cls, estimation_strategy_name):
return cls.size_estimators[estimation_strategy_name]
@staticmethod
def _analysis_for_target(analysis_dir, target):
return os.path.join(analysis_dir, target.id + '.analysis')
@staticmethod
def _portable_analysis_for_target(analysis_dir, target):
return JvmCompile._analysis_for_target(analysis_dir,
target) + '.portable'
@classmethod
def register_options(cls, register):
super(JvmCompile, cls).register_options(register)
register('--jvm-options',
advanced=True,
type=list_option,
default=[],
help='Run the compiler with these JVM options.')
register('--args',
advanced=True,
action='append',
default=list(cls.get_args_default(register.bootstrap)),
fingerprint=True,
help='Pass these args to the compiler.')
register('--confs',
advanced=True,
type=list_option,
default=['default'],
help='Compile for these Ivy confs.')
# TODO: Stale analysis should be automatically ignored via Task identities:
# https://github.com/pantsbuild/pants/issues/1351
register(
'--clear-invalid-analysis',
advanced=True,
default=False,
action='store_true',
help=
'When set, any invalid/incompatible analysis files will be deleted '
'automatically. When unset, an error is raised instead.')
register('--warnings',
default=True,
action='store_true',
help='Compile with all configured warnings enabled.')
register('--warning-args',
advanced=True,
action='append',
default=list(cls.get_warning_args_default()),
help='Extra compiler args to use when warnings are enabled.')
register('--no-warning-args',
advanced=True,
action='append',
default=list(cls.get_no_warning_args_default()),
help='Extra compiler args to use when warnings are disabled.')
register(
'--delete-scratch',
advanced=True,
default=True,
action='store_true',
help=
'Leave intermediate scratch files around, for debugging build problems.'
)
register('--worker-count',
advanced=True,
type=int,
default=1,
help='The number of concurrent workers to use when '
'compiling with {task}.'.format(task=cls._name))
register('--size-estimator',
advanced=True,
choices=list(cls.size_estimators.keys()),
default='filesize',
help='The method of target size estimation.')
register('--capture-log',
advanced=True,
action='store_true',
default=False,
fingerprint=True,
help='Capture compilation output to per-target logs.')
# TODO: Defaulting to false due to a few upstream issues for which we haven't pulled down fixes:
# https://github.com/sbt/sbt/pull/2085
# https://github.com/sbt/sbt/pull/2160
register(
'--incremental-caching',
advanced=True,
action='store_true',
default=False,
help=
'When set, the results of incremental compiles will be written to the cache. '
'This is unset by default, because it is generally a good precaution to cache '
'only clean/cold builds.')
@classmethod
def product_types(cls):
raise TaskError(
'Expected to be installed in GroupTask, which has its own '
'product_types implementation.')
@classmethod
def prepare(cls, options, round_manager):
super(JvmCompile, cls).prepare(options, round_manager)
round_manager.require_data('compile_classpath')
# Require codegen we care about
# TODO(John Sirois): roll this up in Task - if the list of labels we care about for a target
# predicate to filter the full build graph is exposed, the requirement can be made automatic
# and in turn codegen tasks could denote the labels they produce automating wiring of the
# produce side
round_manager.require_data('java')
round_manager.require_data('scala')
# Allow the deferred_sources_mapping to take place first
round_manager.require_data('deferred_sources')
# Subclasses must implement.
# --------------------------
_name = None
_file_suffix = None
_supports_concurrent_execution = None
@classmethod
def task_subsystems(cls):
# NB(gmalmquist): This is only used to make sure the JvmTargets get properly fingerprinted.
# See: java_zinc_compile_jvm_platform_integration#test_compile_stale_platform_settings.
return super(JvmCompile, cls).task_subsystems() + (JvmPlatform, )
@classmethod
def name(cls):
return cls._name
@classmethod
def get_args_default(cls, bootstrap_option_values):
"""Override to set default for --args option.
:param bootstrap_option_values: The values of the "bootstrap options" (e.g., pants_workdir).
Implementations can use these when generating the default.
See src/python/pants/options/options_bootstrapper.py for
details.
"""
return ()
@classmethod
def get_warning_args_default(cls):
"""Override to set default for --warning-args option."""
return ()
@classmethod
def get_no_warning_args_default(cls):
"""Override to set default for --no-warning-args option."""
return ()
@property
def config_section(self):
return self.options_scope
def select(self, target):
return target.has_sources(self._file_suffix)
def select_source(self, source_file_path):
"""Source predicate for this task."""
return source_file_path.endswith(self._file_suffix)
def create_analysis_tools(self):
"""Returns an AnalysisTools implementation.
Subclasses must implement.
"""
raise NotImplementedError()
def compile(self, args, classpath, sources, classes_output_dir,
upstream_analysis, analysis_file, log_file, settings):
"""Invoke the compiler.
Must raise TaskError on compile failure.
Subclasses must implement.
:param list args: Arguments to the compiler (such as jmake or zinc).
:param list classpath: List of classpath entries.
:param list sources: Source files.
:param str classes_output_dir: Where to put the compiled output.
:param upstream_analysis:
:param analysis_file: Where to write the compile analysis.
:param log_file: Where to write logs.
:param JvmPlatformSettings settings: platform settings determining the -source, -target, etc for
javac to use.
"""
raise NotImplementedError()
# Subclasses may override.
# ------------------------
def extra_compile_time_classpath_elements(self):
"""Extra classpath elements common to all compiler invocations.
E.g., jars for compiler plugins.
These are added at the end of the classpath, after any dependencies, so that if they
overlap with any explicit dependencies, the compiler sees those first. This makes
missing dependency accounting much simpler.
"""
return []
def extra_products(self, target):
"""Any extra, out-of-band resources created for a target.
E.g., targets that produce scala compiler plugins or annotation processor files
produce an info file. The resources will be added to the runtime_classpath.
Returns a list of pairs (root, [absolute paths of files under root]).
"""
return []
def __init__(self, *args, **kwargs):
super(JvmCompile, self).__init__(*args, **kwargs)
self._targets_to_compile_settings = None
# JVM options for running the compiler.
self._jvm_options = self.get_options().jvm_options
self._args = list(self.get_options().args)
if self.get_options().warnings:
self._args.extend(self.get_options().warning_args)
else:
self._args.extend(self.get_options().no_warning_args)
# The ivy confs for which we're building.
self._confs = self.get_options().confs
# Maps CompileContext --> dict of upstream class to paths.
self._upstream_class_to_paths = {}
# Mapping of relevant (as selected by the predicate) sources by target.
self._sources_by_target = None
self._sources_predicate = self.select_source
# Various working directories.
self._analysis_dir = os.path.join(self.workdir, 'isolated-analysis')
self._classes_dir = os.path.join(self.workdir, 'isolated-classes')
self._logs_dir = os.path.join(self.workdir, 'isolated-logs')
self._jars_dir = os.path.join(self.workdir, 'jars')
self._capture_log = self.get_options().capture_log
self._delete_scratch = self.get_options().delete_scratch
self._clear_invalid_analysis = self.get_options(
).clear_invalid_analysis
try:
worker_count = self.get_options().worker_count
except AttributeError:
# tasks that don't support concurrent execution have no worker_count registered
worker_count = 1
self._worker_count = worker_count
self._size_estimator = self.size_estimator_by_name(
self.get_options().size_estimator)
self._worker_pool = None
self._analysis_tools = self.create_analysis_tools()
@property
def _analysis_parser(self):
return self._analysis_tools.parser
def _fingerprint_strategy(self, classpath_products):
return ResolvedJarAwareTaskIdentityFingerprintStrategy(
self, classpath_products)
def ensure_analysis_tmpdir(self):
"""Work in a tmpdir so we don't stomp the main analysis files on error.
A temporary, but well-known, dir in which to munge analysis/dependency files in before
caching. It must be well-known so we know where to find the files when we retrieve them from
the cache. The tmpdir is cleaned up in a shutdown hook, because background work
may need to access files we create there even after this method returns
:return: path of temporary analysis directory
"""
analysis_tmpdir = os.path.join(self._workdir, 'analysis_tmpdir')
if self._delete_scratch:
self.context.background_worker_pool().add_shutdown_hook(
lambda: safe_rmtree(analysis_tmpdir))
safe_mkdir(analysis_tmpdir)
return analysis_tmpdir
def pre_execute(self):
# Only create these working dirs during execution phase, otherwise, they
# would be wiped out by clean-all goal/task if it's specified.
self.analysis_tmpdir = self.ensure_analysis_tmpdir()
safe_mkdir(self._analysis_dir)
safe_mkdir(self._classes_dir)
safe_mkdir(self._logs_dir)
safe_mkdir(self._jars_dir)
# TODO(John Sirois): Ensuring requested product maps are available - if empty - should probably
# be lifted to Task infra.
# In case we have no relevant targets and return early create the requested product maps.
self._create_empty_products()
def prepare_execute(self, chunks):
relevant_targets = list(itertools.chain(*chunks))
# Target -> sources (relative to buildroot).
# TODO(benjy): Should sources_by_target be available in all Tasks?
self._sources_by_target = self._compute_sources_by_target(
relevant_targets)
# Update the classpath by adding relevant target's classes directories to its classpath.
compile_classpath = self.context.products.get_data('compile_classpath')
runtime_classpath = self.context.products.get_data(
'runtime_classpath', compile_classpath.copy)
with self.context.new_workunit('validate-{}-analysis'.format(
self._name)):
for target in relevant_targets:
cc = self.compile_context(target)
safe_mkdir(cc.classes_dir)
runtime_classpath.add_for_target(target,
[(conf, cc.classes_dir)
for conf in self._confs])
self.validate_analysis(cc.analysis_file)
# This ensures the workunit for the worker pool is set
with self.context.new_workunit('isolation-{}-pool-bootstrap'.format(self._name)) \
as workunit:
# This uses workunit.parent as the WorkerPool's parent so that child workunits
# of different pools will show up in order in the html output. This way the current running
# workunit is on the bottom of the page rather than possibly in the middle.
self._worker_pool = WorkerPool(workunit.parent,
self.context.run_tracker,
self._worker_count)
def compile_context(self, target):
analysis_file = JvmCompile._analysis_for_target(
self._analysis_dir, target)
classes_dir = os.path.join(self._classes_dir, target.id)
# Generate a short unique path for the jar to allow for shorter classpaths.
# TODO: likely unnecessary after https://github.com/pantsbuild/pants/issues/1988
jar_file = os.path.join(
self._jars_dir, '{}.jar'.format(sha1(target.id).hexdigest()[:12]))
return CompileContext(target, analysis_file, classes_dir, jar_file,
self._sources_for_target(target))
def execute_chunk(self, relevant_targets):
if not relevant_targets:
return
classpath_product = self.context.products.get_data('runtime_classpath')
fingerprint_strategy = self._fingerprint_strategy(classpath_product)
# Invalidation check. Everything inside the with block must succeed for the
# invalid targets to become valid.
partition_size_hint, locally_changed_targets = (0, None)
with self.invalidated(relevant_targets,
invalidate_dependents=True,
partition_size_hint=partition_size_hint,
locally_changed_targets=locally_changed_targets,
fingerprint_strategy=fingerprint_strategy,
topological_order=True) as invalidation_check:
if invalidation_check.invalid_vts:
# Find the invalid targets for this chunk.
invalid_targets = [
vt.target for vt in invalidation_check.invalid_vts
]
# Register products for all the valid targets.
# We register as we go, so dependency checking code can use this data.
valid_targets = [
vt.target for vt in invalidation_check.all_vts if vt.valid
]
valid_compile_contexts = [
self.compile_context(t) for t in valid_targets
]
self._register_vts(valid_compile_contexts)
# Execute compilations for invalid targets.
check_vts = (self.check_artifact_cache
if self.artifact_cache_reads_enabled() else None)
update_artifact_cache_vts_work = (
self.get_update_artifact_cache_work
if self.artifact_cache_writes_enabled() else None)
self.compile_chunk(
invalidation_check, self.context.targets(),
relevant_targets, invalid_targets,
self.extra_compile_time_classpath_elements(), check_vts,
self._compile_vts, self._register_vts,
update_artifact_cache_vts_work)
else:
# Nothing to build. Register products for all the targets in one go.
self._register_vts(
[self.compile_context(t) for t in relevant_targets])
def compile_chunk(self, invalidation_check, all_targets, relevant_targets,
invalid_targets, extra_compile_time_classpath_elements,
check_vts, compile_vts, register_vts,
update_artifact_cache_vts_work):
"""Executes compilations for the invalid targets contained in a single chunk."""
assert invalid_targets, "compile_chunk should only be invoked if there are invalid targets."
# Get the classpath generated by upstream JVM tasks and our own prepare_compile().
classpath_products = self.context.products.get_data(
'runtime_classpath')
extra_compile_time_classpath = self._compute_extra_classpath(
extra_compile_time_classpath_elements)
compile_contexts = self._create_compile_contexts_for_targets(
all_targets)
# Now create compile jobs for each invalid target one by one.
jobs = self._create_compile_jobs(
classpath_products, compile_contexts, extra_compile_time_classpath,
invalid_targets, invalidation_check.invalid_vts_partitioned,
check_vts, compile_vts, register_vts,
update_artifact_cache_vts_work)
exec_graph = ExecutionGraph(jobs)
try:
exec_graph.execute(self._worker_pool, self.context.log)
except ExecutionFailure as e:
raise TaskError("Compilation failure: {}".format(e))
def finalize_execute(self, chunks):
targets = list(itertools.chain(*chunks))
# Replace the classpath entry for each target with its jar'd representation.
classpath_products = self.context.products.get_data(
'runtime_classpath')
for target in targets:
cc = self.compile_context(target)
for conf in self._confs:
classpath_products.remove_for_target(target,
[(conf, cc.classes_dir)])
classpath_products.add_for_target(target,
[(conf, cc.jar_file)])
def _compile_vts(self, vts, sources, analysis_file, upstream_analysis,
classpath, outdir, log_file, progress_message, settings):
"""Compiles sources for the given vts into the given output dir.
vts - versioned target set
sources - sources for this target set
analysis_file - the analysis file to manipulate
classpath - a list of classpath entries
outdir - the output dir to send classes to
May be invoked concurrently on independent target sets.
Postcondition: The individual targets in vts are up-to-date, as if each were
compiled individually.
"""
if not sources:
self.context.log.warn(
'Skipping {} compile for targets with no sources:\n {}'.
format(self.name(), vts.targets))
else:
# Do some reporting.
self.context.log.info(
'Compiling ',
items_to_report_element(sources,
'{} source'.format(self.name())),
' in ',
items_to_report_element(
[t.address.reference() for t in vts.targets], 'target'),
' (', progress_message, ').')
with self.context.new_workunit('compile',
labels=[WorkUnitLabel.COMPILER]):
# The compiler may delete classfiles, then later exit on a compilation error. Then if the
# change triggering the error is reverted, we won't rebuild to restore the missing
# classfiles. So we force-invalidate here, to be on the safe side.
vts.force_invalidate()
self.compile(self._args, classpath, sources, outdir,
upstream_analysis, analysis_file, log_file,
settings)
def check_artifact_cache(self, vts):
post_process_cached_vts = lambda cvts: self.post_process_cached_vts(
cvts)
cache_hit_callback = self.create_cache_hit_callback(vts)
return self.do_check_artifact_cache(
vts,
post_process_cached_vts=post_process_cached_vts,
cache_hit_callback=cache_hit_callback)
def create_cache_hit_callback(self, vts):
cache_key_to_classes_dir = {
v.cache_key: self.compile_context(v.target).classes_dir
for v in vts
}
return CacheHitCallback(cache_key_to_classes_dir)
def post_process_cached_vts(self, cached_vts):
"""Localizes the fetched analysis for targets we found in the cache.
This is the complement of `_write_to_artifact_cache`.
"""
compile_contexts = []
for vt in cached_vts:
for target in vt.targets:
compile_contexts.append(self.compile_context(target))
for compile_context in compile_contexts:
portable_analysis_file = JvmCompile._portable_analysis_for_target(
self._analysis_dir, compile_context.target)
if os.path.exists(portable_analysis_file):
self._analysis_tools.localize(portable_analysis_file,
compile_context.analysis_file)
def _create_empty_products(self):
if self.context.products.is_required_data('classes_by_source'):
make_products = lambda: defaultdict(MultipleRootedProducts)
self.context.products.safe_create_data('classes_by_source',
make_products)
if self.context.products.is_required_data('product_deps_by_src'):
self.context.products.safe_create_data('product_deps_by_src', dict)
def compute_classes_by_source(self, compile_contexts):
"""Compute a map of (context->(src->classes)) for the given compile_contexts.
It's possible (although unfortunate) for multiple targets to own the same sources, hence
the top level division. Srcs are relative to buildroot. Classes are absolute paths.
Returning classes with 'None' as their src indicates that the compiler analysis indicated
that they were un-owned. This case is triggered when annotation processors generate
classes (or due to bugs in classfile tracking in zinc/jmake.)
"""
buildroot = get_buildroot()
# Build a mapping of srcs to classes for each context.
classes_by_src_by_context = defaultdict(dict)
for compile_context in compile_contexts:
# Walk the context's jar to build a set of unclaimed classfiles.
unclaimed_classes = set()
with compile_context.open_jar(mode='r') as jar:
for name in jar.namelist():
if not name.endswith('/'):
unclaimed_classes.add(
os.path.join(compile_context.classes_dir, name))
# Grab the analysis' view of which classfiles were generated.
classes_by_src = classes_by_src_by_context[compile_context]
if os.path.exists(compile_context.analysis_file):
products = self._analysis_parser.parse_products_from_path(
compile_context.analysis_file, compile_context.classes_dir)
for src, classes in products.items():
relsrc = os.path.relpath(src, buildroot)
classes_by_src[relsrc] = classes
unclaimed_classes.difference_update(classes)
# Any remaining classfiles were unclaimed by sources/analysis.
classes_by_src[None] = list(unclaimed_classes)
return classes_by_src_by_context
def classname_for_classfile(self, compile_context, class_file_name):
assert class_file_name.startswith(compile_context.classes_dir)
return ClasspathUtil.classname_for_rel_classfile(
class_file_name[len(compile_context.classes_dir) + 1:])
def _register_vts(self, compile_contexts):
classes_by_source = self.context.products.get_data('classes_by_source')
product_deps_by_src = self.context.products.get_data(
'product_deps_by_src')
runtime_classpath = self.context.products.get_data('runtime_classpath')
# Register a mapping between sources and classfiles (if requested).
if classes_by_source is not None:
ccbsbc = self.compute_classes_by_source(compile_contexts).items()
for compile_context, computed_classes_by_source in ccbsbc:
target = compile_context.target
classes_dir = compile_context.classes_dir
for source in compile_context.sources:
classes = computed_classes_by_source.get(source, [])
classes_by_source[source].add_abs_paths(
classes_dir, classes)
# Register resource products.
for compile_context in compile_contexts:
extra_resources = self.extra_products(compile_context.target)
entries = [(conf, root) for conf in self._confs
for root, _ in extra_resources]
runtime_classpath.add_for_target(compile_context.target, entries)
# And classfile product dependencies (if requested).
if product_deps_by_src is not None:
product_deps_by_src[compile_context.target] = \
self._analysis_parser.parse_deps_from_path(compile_context.analysis_file)
def _create_compile_contexts_for_targets(self, targets):
compile_contexts = OrderedDict()
for target in targets:
compile_context = self.compile_context(target)
compile_contexts[target] = compile_context
return compile_contexts
def _compute_classpath_entries(self, classpath_products, target_closure,
compile_context,
extra_compile_time_classpath):
# Generate a classpath specific to this compile and target.
return ClasspathUtil.compute_classpath_for_target(
compile_context.target, classpath_products,
extra_compile_time_classpath, self._confs, target_closure)
def _upstream_analysis(self, compile_contexts, classpath_entries):
"""Returns tuples of classes_dir->analysis_file for the closure of the target."""
# Reorganize the compile_contexts by class directory.
compile_contexts_by_directory = {}
for compile_context in compile_contexts.values():
compile_contexts_by_directory[
compile_context.classes_dir] = compile_context
# If we have a compile context for the target, include it.
for entry in classpath_entries:
if not entry.endswith('.jar'):
compile_context = compile_contexts_by_directory.get(entry)
if not compile_context:
self.context.log.debug(
'Missing upstream analysis for {}'.format(entry))
else:
yield compile_context.classes_dir, compile_context.analysis_file
def _capture_log_file(self, target):
if self._capture_log:
return os.path.join(self._logs_dir, "{}.log".format(target.id))
return None
def exec_graph_key_for_target(self, compile_target):
return "compile({})".format(compile_target.address.spec)
def _create_compile_jobs(self, classpath_products, compile_contexts,
extra_compile_time_classpath, invalid_targets,
invalid_vts_partitioned, check_vts, compile_vts,
register_vts, update_artifact_cache_vts_work):
def check_cache(vts):
"""Manually checks the artifact cache (usually immediately before compilation.)
Returns true if the cache was hit successfully, indicating that no compilation is necessary.
"""
if not check_vts:
return False
cached_vts, uncached_vts = check_vts([vts])
if not cached_vts:
self.context.log.debug(
'Missed cache during double check for {}'.format(
vts.target.address.spec))
return False
assert cached_vts == [
vts
], ('Cache returned unexpected target: {} vs {}'.format(
cached_vts, [vts]))
self.context.log.info(
'Hit cache during double check for {}'.format(
vts.target.address.spec))
return True
def work_for_vts(vts, compile_context, target_closure):
progress_message = compile_context.target.address.spec
cp_entries = self._compute_classpath_entries(
classpath_products, target_closure, compile_context,
extra_compile_time_classpath)
upstream_analysis = dict(
self._upstream_analysis(compile_contexts, cp_entries))
# Capture a compilation log if requested.
log_file = self._capture_log_file(compile_context.target)
# Double check the cache before beginning compilation
hit_cache = check_cache(vts)
incremental = False
if not hit_cache:
# Mutate analysis within a temporary directory, and move it to the final location
# on success.
tmpdir = os.path.join(self.analysis_tmpdir,
compile_context.target.id)
safe_mkdir(tmpdir)
tmp_analysis_file = self._analysis_for_target(
tmpdir, compile_context.target)
# If the analysis exists for this context, it is an incremental compile.
if os.path.exists(compile_context.analysis_file):
incremental = True
shutil.copy(compile_context.analysis_file,
tmp_analysis_file)
target, = vts.targets
compile_vts(vts, compile_context.sources, tmp_analysis_file,
upstream_analysis, cp_entries,
compile_context.classes_dir, log_file,
progress_message, target.platform)
atomic_copy(tmp_analysis_file, compile_context.analysis_file)
# Jar the compiled output.
self._create_context_jar(compile_context)
# Update the products with the latest classes.
register_vts([compile_context])
# We write to the cache only if we didn't hit during the double check, and optionally
# only for clean builds.
is_cacheable = not hit_cache and (
self.get_options().incremental_caching or not incremental)
self.context.log.debug(
'Completed compile for {}. '
'Hit cache: {}, was incremental: {}, is cacheable: {}, cache writes enabled: {}.'
.format(compile_context.target.address.spec, hit_cache,
incremental, is_cacheable,
update_artifact_cache_vts_work is not None))
if is_cacheable and update_artifact_cache_vts_work:
# Kick off the background artifact cache write.
self._write_to_artifact_cache(vts, compile_context,
update_artifact_cache_vts_work)
jobs = []
invalid_target_set = set(invalid_targets)
for vts in invalid_vts_partitioned:
assert len(vts.targets) == 1, (
"Requested one target per partition, got {}".format(vts))
# Invalidated targets are a subset of relevant targets: get the context for this one.
compile_target = vts.targets[0]
compile_context = compile_contexts[compile_target]
compile_target_closure = compile_target.closure()
# dependencies of the current target which are invalid for this chunk
invalid_dependencies = (compile_target_closure
& invalid_target_set) - [compile_target]
jobs.append(
Job(
self.exec_graph_key_for_target(compile_target),
functools.partial(work_for_vts, vts, compile_context,
compile_target_closure),
[
self.exec_graph_key_for_target(target)
for target in invalid_dependencies
],
self._size_estimator(compile_context.sources),
# If compilation and analysis work succeeds, validate the vts.
# Otherwise, fail it.
on_success=vts.update,
on_failure=vts.force_invalidate))
return jobs
def _create_context_jar(self, compile_context):
"""Jar up the compile_context to its output jar location.
TODO(stuhood): In the medium term, we hope to add compiler support for this step, which would
allow the jars to be used as compile _inputs_ as well. Currently using jar'd compile outputs as
compile inputs would make the compiler's analysis useless.
see https://github.com/twitter-forks/sbt/tree/stuhood/output-jars
"""
root = compile_context.classes_dir
with compile_context.open_jar(mode='w') as jar:
for abs_sub_dir, dirnames, filenames in safe_walk(root):
for name in dirnames + filenames:
abs_filename = os.path.join(abs_sub_dir, name)
arcname = fast_relpath(abs_filename, root)
jar.write(abs_filename, arcname)
def _write_to_artifact_cache(self, vts, compile_context,
get_update_artifact_cache_work):
assert len(vts.targets) == 1
assert vts.targets[0] == compile_context.target
# Noop if the target is uncacheable.
if (compile_context.target.has_label('no_cache')):
return
vt = vts.versioned_targets[0]
# Set up args to relativize analysis in the background.
portable_analysis_file = self._portable_analysis_for_target(
self._analysis_dir, compile_context.target)
relativize_args_tuple = (compile_context.analysis_file,
portable_analysis_file)
# Collect the artifacts for this target.
artifacts = []
# Intransitive classpath entries.
target_classpath = ClasspathUtil.classpath_entries(
(compile_context.target, ),
self.context.products.get_data('runtime_classpath'), ('default', ),
transitive=False)
for entry in target_classpath:
if ClasspathUtil.is_jar(entry):
artifacts.append(entry)
elif ClasspathUtil.is_dir(entry):
for rel_file in ClasspathUtil.classpath_entries_contents(
[entry]):
artifacts.append(os.path.join(entry, rel_file))
else:
# non-jar and non-directory classpath entries should be ignored
pass
# Log file.
log_file = self._capture_log_file(compile_context.target)
if log_file and os.path.exists(log_file):
artifacts.append(log_file)
# Jar.
artifacts.append(compile_context.jar_file)
# Get the 'work' that will publish these artifacts to the cache.
# NB: the portable analysis_file won't exist until we finish.
vts_artifactfiles_pair = (vt, artifacts + [portable_analysis_file])
update_artifact_cache_work = get_update_artifact_cache_work(
[vts_artifactfiles_pair])
# And execute it.
if update_artifact_cache_work:
work_chain = [
Work(self._analysis_tools.relativize, [relativize_args_tuple],
'relativize'), update_artifact_cache_work
]
self.context.submit_background_work_chain(
work_chain, parent_workunit_name='cache')
def validate_analysis(self, path):
"""Throws a TaskError for invalid analysis files."""
try:
self._analysis_parser.validate_analysis(path)
except Exception as e:
if self._clear_invalid_analysis:
self.context.log.warn(
"Invalid analysis detected at path {} ... pants will remove these "
"automatically, but\nyou may experience spurious warnings until "
"clean-all is executed.\n{}".format(path, e))
safe_delete(path)
else:
raise TaskError(
"An internal build directory contains invalid/mismatched analysis: please "
"run `clean-all` if your tools versions changed recently:\n{}"
.format(e))
def _compute_sources_by_target(self, targets):
"""Computes and returns a map target->sources (relative to buildroot)."""
def resolve_target_sources(target_sources):
resolved_sources = []
for target in target_sources:
if target.has_sources():
resolved_sources.extend(
target.sources_relative_to_buildroot())
return resolved_sources
def calculate_sources(target):
sources = [
s for s in target.sources_relative_to_buildroot()
if self._sources_predicate(s)
]
# TODO: Make this less hacky. Ideally target.java_sources will point to sources, not targets.
if hasattr(target, 'java_sources') and target.java_sources:
sources.extend(resolve_target_sources(target.java_sources))
return sources
return {t: calculate_sources(t) for t in targets}
def _sources_for_targets(self, targets):
"""Returns a cached map of target->sources for the specified targets."""
if self._sources_by_target is None:
raise TaskError('self._sources_by_target not computed yet.')
return {t: self._sources_by_target.get(t, []) for t in targets}
def _sources_for_target(self, target):
"""Returns the cached sources for the given target."""
if self._sources_by_target is None:
raise TaskError('self._sources_by_target not computed yet.')
return self._sources_by_target.get(target, [])
def _compute_extra_classpath(self, extra_compile_time_classpath_elements):
"""Compute any extra compile-time-only classpath elements.
TODO(benjy): Model compile-time vs. runtime classpaths more explicitly.
TODO(benjy): Add a pre-execute goal for injecting deps into targets, so e.g.,
we can inject a dep on the scala runtime library and still have it ivy-resolve.
"""
def extra_compile_classpath_iter():
for conf in self._confs:
for jar in extra_compile_time_classpath_elements:
yield (conf, jar)
return list(extra_compile_classpath_iter())
class JvmDependencyUsage(JvmDependencyAnalyzer):
"""Determines the dependency usage ratios of targets.
Analyzes the relationship between the products a target T produces vs. the products
which T's dependents actually require (this is done by observing analysis files).
If the ratio of required products to available products is low, then this is a sign
that target T isn't factored well.
A graph is formed from these results, where each node of the graph is a target, and
each edge is a product usage ratio between a target and its dependency. The nodes
also contain additional information to guide refactoring -- for example, the estimated
job size of each target, which indicates the impact a poorly factored target has on
the build times. (see DependencyUsageGraph->to_json)
The graph is either summarized for local analysis or outputted as a JSON file for
aggregation and analysis on a larger scale.
"""
size_estimators = create_size_estimators()
@classmethod
def register_options(cls, register):
super(JvmDependencyUsage, cls).register_options(register)
register(
'--internal-only',
default=True,
action='store_true',
help=
'Specifies that only internal dependencies should be included in the graph '
'output (no external jars).')
register(
'--summary',
default=True,
action='store_true',
help=
'When set, outputs a summary of the "worst" dependencies; otherwise, '
'outputs a JSON report.')
register('--size-estimator',
choices=list(cls.size_estimators.keys()),
default='filesize',
help='The method of target size estimation.')
register('--transitive',
default=True,
action='store_true',
help='Score all targets in the build graph transitively.')
register('--output-file',
type=str,
help='Output destination. When unset, outputs to <stdout>.')
@classmethod
def prepare(cls, options, round_manager):
super(JvmDependencyUsage, cls).prepare(options, round_manager)
round_manager.require_data('classes_by_source')
round_manager.require_data('runtime_classpath')
round_manager.require_data('product_deps_by_src')
@classmethod
def skip(cls, options):
"""This task is always explicitly requested."""
return False
def execute(self):
targets = (self.context.targets() if self.get_options().transitive else
self.context.target_roots)
graph = self.create_dep_usage_graph(targets, get_buildroot())
output_file = self.get_options().output_file
if output_file:
self.context.log.info(
'Writing dependency usage to {}'.format(output_file))
with open(output_file, 'w') as fh:
self._render(graph, fh)
else:
sys.stdout.write(b'\n')
self._render(graph, sys.stdout)
def _render(self, graph, fh):
chunks = graph.to_summary() if self.get_options(
).summary else graph.to_json()
for chunk in chunks:
fh.write(chunk)
fh.flush()
def _resolve_aliases(self, target):
"""Recursively resolve `target` aliases."""
for declared in target.dependencies:
if type(declared) == Target:
for r in self._resolve_aliases(declared):
yield r
else:
yield declared
def _is_declared_dep(self, target, dep):
"""Returns true if the given dep target should be considered a declared dep of target."""
return dep in self._resolve_aliases(target)
def _select(self, target):
if self.get_options().internal_only and isinstance(target, JarLibrary):
return False
elif isinstance(target, Resources) or type(target) == Target:
return False
else:
return True
def _normalize_product_dep(self, buildroot, classes_by_source, dep):
"""Normalizes the given product dep from the given dep into a set of classfiles.
Product deps arrive as sources, jars, and classfiles: this method normalizes them to classfiles.
TODO: This normalization should happen in the super class.
"""
if dep.endswith(".jar"):
# TODO: post sbt/zinc jar output patch, binary deps will be reported directly as classfiles
return set()
elif dep.endswith(".class"):
return set([dep])
else:
# assume a source file and convert to classfiles
rel_src = fast_relpath(dep, buildroot)
return set(p
for _, paths in classes_by_source[rel_src].rel_paths()
for p in paths)
def _count_products(self, classpath_products, target):
contents = ClasspathUtil.classpath_contents((target, ),
classpath_products)
# Generators don't implement len.
return sum(1 for _ in contents)
def create_dep_usage_graph(self, targets, buildroot):
"""Creates a graph of concrete targets, with their sum of products and dependencies.
Synthetic targets contribute products and dependencies to their concrete target.
"""
# Initialize all Nodes.
classes_by_source = self.context.products.get_data('classes_by_source')
runtime_classpath = self.context.products.get_data('runtime_classpath')
product_deps_by_src = self.context.products.get_data(
'product_deps_by_src')
nodes = dict()
for target in targets:
if not self._select(target):
continue
# Create or extend a Node for the concrete version of this target.
concrete_target = target.concrete_derived_from
products_total = self._count_products(runtime_classpath, target)
node = nodes.get(concrete_target)
if not node:
node = nodes.setdefault(concrete_target, Node(concrete_target))
node.add_derivation(target, products_total)
# Record declared Edges.
for dep_tgt in self._resolve_aliases(target):
derived_from = dep_tgt.concrete_derived_from
if self._select(derived_from):
node.add_edge(Edge(is_declared=True, products_used=set()),
derived_from)
# Record the used products and undeclared Edges for this target. Note that some of
# these may be self edges, which are considered later.
target_product_deps_by_src = product_deps_by_src.get(
target, dict())
for src in target.sources_relative_to_buildroot():
for product_dep in target_product_deps_by_src.get(
os.path.join(buildroot, src), []):
for dep_tgt in self.targets_by_file.get(product_dep, []):
derived_from = dep_tgt.concrete_derived_from
if not self._select(derived_from):
continue
is_declared = self._is_declared_dep(target, dep_tgt)
normalized_deps = self._normalize_product_dep(
buildroot, classes_by_source, product_dep)
node.add_edge(
Edge(is_declared=is_declared,
products_used=normalized_deps), derived_from)
# Prune any Nodes with 0 products.
for concrete_target, node in nodes.items()[:]:
if node.products_total == 0:
nodes.pop(concrete_target)
return DependencyUsageGraph(
nodes, self.size_estimators[self.get_options().size_estimator])
