def calculating_node_creator(self, classes_by_source, runtime_classpath,
product_deps_by_src, target_to_vts):
"""Strategy directly computes dependency graph node based on
`classes_by_source`, `runtime_classpath`, `product_deps_by_src` parameters and
stores the result to the build cache.
"""
analyzer = JvmDependencyAnalyzer(get_buildroot(), runtime_classpath)
targets = self.context.targets()
targets_by_file = analyzer.targets_by_file(targets)
transitive_deps_by_target = analyzer.compute_transitive_deps_by_target(
targets)
def creator(target):
transitive_deps = set(transitive_deps_by_target.get(target))
node = self.create_dep_usage_node(target, analyzer,
product_deps_by_src,
classes_by_source,
targets_by_file, transitive_deps)
vt = target_to_vts[target]
mode = 'w' if PY3 else 'wb'
with open(self.nodes_json(vt.results_dir), mode=mode) as fp:
json.dump(node.to_cacheable_dict(),
fp,
indent=2,
sort_keys=True)
vt.update()
return node
return creator
def calculating_node_creator(self, classes_by_source, runtime_classpath, product_deps_by_src,
target_to_vts):
"""Strategy directly computes dependency graph node based on
`classes_by_source`, `runtime_classpath`, `product_deps_by_src` parameters and
stores the result to the build cache.
"""
analyzer = JvmDependencyAnalyzer(get_buildroot(), runtime_classpath)
targets = self.context.targets()
targets_by_file = analyzer.targets_by_file(targets)
transitive_deps_by_target = analyzer.compute_transitive_deps_by_target(targets)
def creator(target):
transitive_deps = set(transitive_deps_by_target.get(target))
node = self.create_dep_usage_node(target,
analyzer,
product_deps_by_src,
classes_by_source,
targets_by_file,
transitive_deps)
vt = target_to_vts[target]
with open(self.nodes_json(vt.results_dir), mode='w') as fp:
json.dump(node.to_cacheable_dict(), fp, indent=2, sort_keys=True)
vt.update()
return node
return creator
def _missing_deps_finder(self):
dep_analyzer = JvmDependencyAnalyzer(
get_buildroot(),
self.context.products.get_data('runtime_classpath'))
return MissingDependencyFinder(
dep_analyzer,
CompileErrorExtractor(
self.get_options().class_not_found_error_patterns))
def create_graph(self, task, targets):
classes_by_source = task.context.products.get_data('classes_by_source')
runtime_classpath = task.context.products.get_data('runtime_classpath')
product_deps_by_src = task.context.products.get_data('product_deps_by_src')
analyzer = JvmDependencyAnalyzer('', runtime_classpath, product_deps_by_src)
targets_by_file = analyzer.targets_by_file(targets)
transitive_deps_by_target = analyzer.compute_transitive_deps_by_target(targets)
def node_creator(target):
transitive_deps = set(transitive_deps_by_target.get(target))
return task.create_dep_usage_node(target,
analyzer,
classes_by_source,
targets_by_file,
transitive_deps)
return DependencyUsageGraph(task.create_dep_usage_nodes(targets, node_creator),
task.size_estimators[task.get_options().size_estimator])
def create_graph(self, task, targets):
classes_by_source = task.context.products.get_data('classes_by_source')
runtime_classpath = task.context.products.get_data('runtime_classpath')
product_deps_by_src = task.context.products.get_data('product_deps_by_src')
analyzer = JvmDependencyAnalyzer('', runtime_classpath)
targets_by_file = analyzer.targets_by_file(targets)
transitive_deps_by_target = analyzer.compute_transitive_deps_by_target(targets)
def node_creator(target):
transitive_deps = set(transitive_deps_by_target.get(target))
return task.create_dep_usage_node(target,
analyzer,
product_deps_by_src,
classes_by_source,
targets_by_file,
transitive_deps)
return DependencyUsageGraph(task.create_dep_usage_nodes(targets, node_creator),
task.size_estimators[task.get_options().size_estimator])
def _dep_analyzer(self):
return JvmDependencyAnalyzer(
get_buildroot(),
self.context.products.get_data('runtime_classpath'),
self.context.products.get_data('product_deps_by_src'))
def _analyzer(self):
return JvmDependencyAnalyzer(
get_buildroot(), DistributionLocator.cached(),
self.context.products.get_data('runtime_classpath'))
def _compute_missing_deps(self, src_tgt, actual_deps):
"""Computes deps that are used by the compiler but not specified in a BUILD file.
These deps are bugs waiting to happen: the code may happen to compile because the dep was
brought in some other way (e.g., by some other root target), but that is obviously fragile.
Note that in practice we're OK with reliance on indirect deps that are only brought in
transitively. E.g., in Scala type inference can bring in such a dep subtly. Fortunately these
cases aren't as fragile as a completely missing dependency. It's still a good idea to have
explicit direct deps where relevant, so we optionally warn about indirect deps, to make them
easy to find and reason about.
- actual_deps: a map src -> list of actual deps (source, class or jar file) as noted by the
compiler.
Returns a triple (missing_file_deps, missing_tgt_deps, missing_direct_tgt_deps) where:
- missing_file_deps: a list of dep_files where src_tgt requires dep_file, and we're unable
to map to a target (because its target isn't in the total set of targets in play,
and we don't want to parse every BUILD file in the workspace just to find it).
- missing_tgt_deps: a list of dep_tgt where src_tgt is missing a necessary transitive
dependency on dep_tgt.
- missing_direct_tgt_deps: a list of dep_tgts where src_tgt is missing a direct dependency
on dep_tgt but has a transitive dep on it.
All paths in the input and output are absolute.
"""
analyzer = JvmDependencyAnalyzer(
get_buildroot(),
self.context.products.get_data('runtime_classpath'),
self.context.products.get_data('product_deps_by_src'))
def must_be_explicit_dep(dep):
# We don't require explicit deps on the java runtime, so we shouldn't consider that
# a missing dep.
return (dep not in analyzer.bootstrap_jar_classfiles and
not dep.startswith(DistributionLocator.cached().real_home))
def target_or_java_dep_in_targets(target, targets):
# We want to check if the target is in the targets collection
#
# However, for the special case of scala_library that has a java_sources
# reference we're ok if that exists in targets even if the scala_library does not.
if target in targets:
return True
elif isinstance(target, ScalaLibrary):
return any(t in targets for t in target.java_sources)
else:
return False
# TODO: If recomputing these every time becomes a performance issue, memoize for
# already-seen targets and incrementally compute for new targets not seen in a previous
# partition, in this or a previous chunk.
transitive_deps_by_target = analyzer.compute_transitive_deps_by_target(
self.context.targets())
# Find deps that are actual but not specified.
missing_file_deps = OrderedSet() # (src, src).
missing_tgt_deps_map = defaultdict(
list) # (tgt, tgt) -> a list of (src, src) as evidence.
missing_direct_tgt_deps_map = defaultdict(
list) # The same, but for direct deps.
targets_by_file = analyzer.targets_by_file(self.context.targets())
buildroot = get_buildroot()
abs_srcs = [
os.path.join(buildroot, src)
for src in src_tgt.sources_relative_to_buildroot()
]
for src in abs_srcs:
for actual_dep in filter(must_be_explicit_dep,
actual_deps.get(src, [])):
actual_dep_tgts = targets_by_file.get(actual_dep)
# actual_dep_tgts is usually a singleton. If it's not, we only need one of these
# to be in our declared deps to be OK.
if actual_dep_tgts is None:
missing_file_deps.add((src_tgt, actual_dep))
elif not target_or_java_dep_in_targets(src_tgt,
actual_dep_tgts):
# Obviously intra-target deps are fine.
canonical_actual_dep_tgt = next(iter(actual_dep_tgts))
if actual_dep_tgts.isdisjoint(
transitive_deps_by_target.get(src_tgt, [])):
missing_tgt_deps_map[(
src_tgt, canonical_actual_dep_tgt)].append(
(src, actual_dep))
elif canonical_actual_dep_tgt not in src_tgt.dependencies:
# The canonical dep is the only one a direct dependency makes sense on.
missing_direct_tgt_deps_map[(
src_tgt, canonical_actual_dep_tgt)].append(
(src, actual_dep))
return (list(missing_file_deps), missing_tgt_deps_map.items(),
missing_direct_tgt_deps_map.items())
def _analyzer(self):
return JvmDependencyAnalyzer(
get_buildroot(), self._distribution, self.context.products.get_data("runtime_classpath")
)
def _compute_missing_deps(self, src_tgt, actual_deps):
"""Computes deps that are used by the compiler but not specified in a BUILD file.
These deps are bugs waiting to happen: the code may happen to compile because the dep was
brought in some other way (e.g., by some other root target), but that is obviously fragile.
Note that in practice we're OK with reliance on indirect deps that are only brought in
transitively. E.g., in Scala type inference can bring in such a dep subtly. Fortunately these
cases aren't as fragile as a completely missing dependency. It's still a good idea to have
explicit direct deps where relevant, so we optionally warn about indirect deps, to make them
easy to find and reason about.
- actual_deps: a map src -> list of actual deps (source, class or jar file) as noted by the
compiler.
Returns a triple (missing_file_deps, missing_tgt_deps, missing_direct_tgt_deps) where:
- missing_file_deps: a list of dep_files where src_tgt requires dep_file, and we're unable
to map to a target (because its target isn't in the total set of targets in play,
and we don't want to parse every BUILD file in the workspace just to find it).
- missing_tgt_deps: a list of dep_tgt where src_tgt is missing a necessary transitive
dependency on dep_tgt.
- missing_direct_tgt_deps: a list of dep_tgts where src_tgt is missing a direct dependency
on dep_tgt but has a transitive dep on it.
All paths in the input and output are absolute.
"""
analyzer = JvmDependencyAnalyzer(get_buildroot(),
self.context.products.get_data('runtime_classpath'),
self.context.products.get_data('product_deps_by_src'))
def must_be_explicit_dep(dep):
# We don't require explicit deps on the java runtime, so we shouldn't consider that
# a missing dep.
return (dep not in analyzer.bootstrap_jar_classfiles
and not dep.startswith(DistributionLocator.cached().real_home))
def target_or_java_dep_in_targets(target, targets):
# We want to check if the target is in the targets collection
#
# However, for the special case of scala_library that has a java_sources
# reference we're ok if that exists in targets even if the scala_library does not.
if target in targets:
return True
elif isinstance(target, ScalaLibrary):
return any(t in targets for t in target.java_sources)
else:
return False
# TODO: If recomputing these every time becomes a performance issue, memoize for
# already-seen targets and incrementally compute for new targets not seen in a previous
# partition, in this or a previous chunk.
transitive_deps_by_target = analyzer.compute_transitive_deps_by_target(self.context.targets())
# Find deps that are actual but not specified.
missing_file_deps = OrderedSet() # (src, src).
missing_tgt_deps_map = defaultdict(list) # (tgt, tgt) -> a list of (src, src) as evidence.
missing_direct_tgt_deps_map = defaultdict(list) # The same, but for direct deps.
targets_by_file = analyzer.targets_by_file(self.context.targets())
buildroot = get_buildroot()
abs_srcs = [os.path.join(buildroot, src) for src in src_tgt.sources_relative_to_buildroot()]
for src in abs_srcs:
for actual_dep in filter(must_be_explicit_dep, actual_deps.get(src, [])):
actual_dep_tgts = targets_by_file.get(actual_dep)
# actual_dep_tgts is usually a singleton. If it's not, we only need one of these
# to be in our declared deps to be OK.
if actual_dep_tgts is None:
missing_file_deps.add((src_tgt, actual_dep))
elif not target_or_java_dep_in_targets(src_tgt, actual_dep_tgts):
# Obviously intra-target deps are fine.
canonical_actual_dep_tgt = next(iter(actual_dep_tgts))
if actual_dep_tgts.isdisjoint(transitive_deps_by_target.get(src_tgt, [])):
missing_tgt_deps_map[(src_tgt, canonical_actual_dep_tgt)].append((src, actual_dep))
elif canonical_actual_dep_tgt not in src_tgt.dependencies:
# The canonical dep is the only one a direct dependency makes sense on.
missing_direct_tgt_deps_map[(src_tgt, canonical_actual_dep_tgt)].append(
(src, actual_dep))
return (list(missing_file_deps),
missing_tgt_deps_map.items(),
missing_direct_tgt_deps_map.items())
