def invalidated(self,
targets,
invalidate_dependents=False,
partition_size_hint=sys.maxint,
silent=False,
locally_changed_targets=None,
fingerprint_strategy=None,
topological_order=False):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
targets: The targets to check for changes.
invalidate_dependents: If True then any targets depending on changed targets are invalidated.
partition_size_hint: Each VersionedTargetSet in the yielded list will represent targets
containing roughly this number of source files, if possible. Set to
sys.maxint for a single VersionedTargetSet. Set to 0 for one
VersionedTargetSet per target. It is up to the caller to do the right
thing with whatever partitioning it asks for.
locally_changed_targets: Targets that we've edited locally. If specified, and there aren't too
many of them, we keep these in separate partitions from other targets,
as these are more likely to have build errors, and so to be rebuilt over
and over, and partitioning them separately is a performance win.
fingerprint_strategy: A FingerprintStrategy instance, which can do per task, finer grained
fingerprinting of a given Target.
Yields an InvalidationCheck object reflecting the (partitioned) targets.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
"""
# TODO(benjy): Compute locally_changed_targets here instead of passing it in? We currently pass
# it in because JvmCompile already has the source->target mapping for other reasons, and also
# to selectively enable this feature.
cache_manager = self.create_cache_manager(invalidate_dependents,
fingerprint_strategy=fingerprint_strategy)
# We separate locally-modified targets from others by coloring them differently.
# This can be a performance win, because these targets are more likely to be iterated
# over, and this preserves "chunk stability" for them.
colors = {}
# But we only do so if there aren't too many, or this optimization will backfire.
locally_changed_target_limit = 10
if locally_changed_targets and len(locally_changed_targets) < locally_changed_target_limit:
for t in targets:
if t in locally_changed_targets:
colors[t] = 'locally_changed'
else:
colors[t] = 'not_locally_changed'
invalidation_check = cache_manager.check(targets, partition_size_hint, colors, topological_order=topological_order)
if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled():
with self.context.new_workunit('cache'):
cached_vts, uncached_vts = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
for t in cached_targets:
self.context.run_tracker.artifact_cache_stats.add_hit('default', t)
if not silent:
self._report_targets('Using cached artifacts for ', cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
for t in uncached_targets:
self.context.run_tracker.artifact_cache_stats.add_miss('default', t)
if not silent:
self._report_targets('No cached artifacts for ', uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts, partition_size_hint, colors)
if not silent:
targets = []
num_invalid_partitions = len(invalidation_check.invalid_vts_partitioned)
for vt in invalidation_check.invalid_vts_partitioned:
targets.extend(vt.targets)
payloads = [t.payload for t in targets]
if len(targets):
msg_elements = ['Invalidated ',
items_to_report_element([t.address.reference() for t in targets], 'target')]
if len(payloads) > 0:
msg_elements.append(' containing ')
msg_elements.append(items_to_report_element(payloads, 'payload file'))
if num_invalid_partitions > 1:
msg_elements.append(' in %d target partitions' % num_invalid_partitions)
msg_elements.append('.')
self.context.log.info(*msg_elements)
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
for vt in invalidation_check.invalid_vts:
vt.update() # In case the caller doesn't update.
def test_partition(self):
# The default EmptyPayload chunking unit happens to be 1, so each of these Targets
# has a chunking unit contribution of 1
a = self.make_target(':a', dependencies=[])
b = self.make_target(':b', dependencies=[a])
c = self.make_target(':c', dependencies=[b])
d = self.make_target(':d', dependencies=[c, a])
e = self.make_target(':e', dependencies=[d])
targets = [a, b, c, d, e]
def print_partitions(partitions):
strs = []
for partition in partitions:
strs.append('(%s)' %
', '.join([t.id for t in partition.targets]))
print('[%s]' % ' '.join(strs))
# Verify basic data structure soundness.
all_vts = self.cache_manager._sort_and_validate_targets(targets)
invalid_vts = filter(lambda vt: not vt.valid, all_vts)
self.assertEquals(5, len(invalid_vts))
self.assertEquals(5, len(all_vts))
vts_targets = [vt.targets[0] for vt in all_vts]
self.assertEquals(set(targets), set(vts_targets))
# Test a simple partition.
ic = InvalidationCheck(all_vts, [], 3)
partitioned = ic.all_vts_partitioned
print_partitions(partitioned)
# Several correct partitionings are possible, but in all cases 4 1-source targets will be
# added to the first partition before it exceeds the limit of 3, and the final target will
# be in a partition by itself.
self.assertEquals(2, len(partitioned))
self.assertEquals(4, len(partitioned[0].targets))
self.assertEquals(1, len(partitioned[1].targets))
# Test partition with colors.
red = 'red'
blue = 'blue'
colors = {a: blue, b: red, c: red, d: red, e: blue}
# As a reference, we partition without colors.
ic = InvalidationCheck(all_vts, [], 2)
partitioned = ic.all_vts_partitioned
print_partitions(partitioned)
self.assertEquals(2, len(partitioned))
self.assertEquals(3, len(partitioned[0].targets))
self.assertEquals(2, len(partitioned[1].targets))
# Now apply color restrictions.
ic = InvalidationCheck(all_vts, [], 2, target_colors=colors)
partitioned = ic.all_vts_partitioned
print_partitions(partitioned)
self.assertEquals(3, len(partitioned))
self.assertEquals(1, len(partitioned[0].targets))
self.assertEquals(3, len(partitioned[1].targets))
self.assertEquals(1, len(partitioned[2].targets))