def _do_invalidation_check(self,
fingerprint_strategy,
invalidate_dependents,
targets,
topological_order):
if self._cache_factory.ignore:
cache_key_generator = UncacheableCacheKeyGenerator()
else:
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(self.workdir,
cache_key_generator,
self._build_invalidator,
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=self._task_name,
task_version_slug=self.implementation_version_slug(),
artifact_write_callback=self.maybe_write_artifact)
# If this Task's execution has been forced, invalidate all our target fingerprints.
if self._cache_factory.ignore and not self._force_invalidated:
self.invalidate()
self._force_invalidated = True
return cache_manager.check(targets, topological_order=topological_order)
def _do_invalidation_check(self,
fingerprint_strategy,
invalidate_dependents,
targets,
topological_order):
if self._cache_factory.ignore:
cache_key_generator = UncacheableCacheKeyGenerator()
else:
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(self.workdir,
cache_key_generator,
self._build_invalidator(),
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=self._task_name,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
# If this Task's execution has been forced, invalidate all our target fingerprints.
if self._cache_factory.ignore and not self._force_invalidated:
self.invalidate()
self._force_invalidated = True
return cache_manager.check(targets, topological_order=topological_order)
def invalidated(self,
targets,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=None,
topological_order=False):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
:API: public
:param targets: The targets to check for changes.
:param invalidate_dependents: If True then any targets depending on changed targets are
invalidated.
:param silent: If true, suppress logging information about target invalidation.
:param fingerprint_strategy: A FingerprintStrategy instance, which can do per task,
finer grained fingerprinting of a given Target.
:param topological_order: Whether to invalidate in dependency order.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
:returns: Yields an InvalidationCheck object reflecting the targets.
:rtype: InvalidationCheck
"""
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(self.workdir,
cache_key_generator,
self._build_invalidator_dir,
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=type(self).__name__,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
invalidation_check = cache_manager.check(targets, topological_order=topological_order)
self._maybe_create_results_dirs(invalidation_check.all_vts)
if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled():
with self.context.new_workunit('cache'):
cached_vts, uncached_vts, uncached_causes = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
self.context.run_tracker.artifact_cache_stats.add_hits(cache_manager.task_name,
cached_targets)
if not silent:
self._report_targets('Using cached artifacts for ', cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
self.context.run_tracker.artifact_cache_stats.add_misses(cache_manager.task_name,
uncached_targets,
uncached_causes)
if not silent:
self._report_targets('No cached artifacts for ', uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts)
if not silent:
targets = []
for vt in invalidation_check.invalid_vts:
targets.extend(vt.targets)
if len(targets):
msg_elements = ['Invalidated ',
items_to_report_element([t.address.reference() for t in targets], 'target'),
'.']
self.context.log.info(*msg_elements)
invalidation_report = self.context.invalidation_report
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager, vts.targets, vts.cache_key, vts.valid,
phase='pre-check')
# Cache has been checked to create the full list of invalid VTs.
# Only copy previous_results for this subset of VTs.
for vts in invalidation_check.invalid_vts:
if self.incremental:
vts.copy_previous_results(self.workdir)
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager, vts.targets, vts.cache_key, vts.valid,
phase='post-check')
for vt in invalidation_check.invalid_vts:
vt.update()
# Background work to clean up previous builds.
if self.context.options.for_global_scope().workdir_max_build_entries is not None:
self._launch_background_workdir_cleanup(invalidation_check.all_vts)
class InvalidationCacheManagerTest(TestBase):
@staticmethod
def is_empty(dir_name):
return not os.listdir(dir_name)
@staticmethod
def has_symlinked_result_dir(vt):
return os.path.realpath(vt.results_dir) == os.path.realpath(vt.current_results_dir)
@staticmethod
def clobber_symlink(vt):
# Munge the state to mimic a common error found before we added the clean- it accidentally clobbers the symlink!
# Commonly caused by safe_mkdir(vt.results_dir, clean=True), broken up here to keep the tests from being brittle.
safe_rmtree(vt.results_dir)
safe_mkdir(vt.results_dir)
def setUp(self):
super(InvalidationCacheManagerTest, self).setUp()
self._dir = tempfile.mkdtemp()
self.cache_manager = InvalidationCacheManager(
results_dir_root=os.path.join(self._dir, 'results'),
cache_key_generator=CacheKeyGenerator(),
build_invalidator=BuildInvalidator(os.path.join(self._dir, 'build_invalidator')),
invalidate_dependents=True,
)
def tearDown(self):
shutil.rmtree(self._dir, ignore_errors=True)
super(InvalidationCacheManagerTest, self).tearDown()
def make_vt(self):
# Create an arbitrary VT. It will mimic the state of the VT handed back by a task.
a_target = self.make_target(':a', dependencies=[])
ic = self.cache_manager.check([a_target])
vt = ic.all_vts[0]
self.task_execute(vt)
vt.update()
return vt
def task_execute(self, vt):
vt.create_results_dir()
task_output = os.path.join(vt.results_dir, 'a_file')
self.create_file(task_output, 'foo')
def test_creates_stable_result_dir_symlink(self):
vt = self.make_vt()
vt.create_results_dir()
parent, unstable_dir_name = os.path.split(vt._current_results_dir)
self.assertSetEqual({'current', unstable_dir_name}, set(os.listdir(parent)))
symlink = os.path.join(parent, 'current')
self.assertTrue(os.path.islink(symlink))
self.assertEqual(unstable_dir_name, os.readlink(symlink))
self.assertEqual(symlink, vt.results_dir)
# Repoint the symlink, but keep the vt valid (simulates the case where the underlying target's
# version has changed, so the symlink is pointed to that version, but now the version has
# reverted, so we must point it back).
os.unlink(symlink)
os.symlink(unstable_dir_name + '.other', symlink)
vt.create_results_dir()
self.assertEqual(unstable_dir_name, os.readlink(symlink))
def test_creates_stable_results_dir_prefix_symlink(self):
parent, unstable_dir_name = os.path.split(self.cache_manager._results_dir_prefix)
self.assertSetEqual({'current', unstable_dir_name}, set(os.listdir(parent)))
symlink = os.path.join(parent, 'current')
self.assertTrue(os.path.islink(symlink))
self.assertTrue(unstable_dir_name, os.readlink(symlink))
def test_check_marks_all_as_invalid_by_default(self):
a = self.make_target(':a', dependencies=[])
b = self.make_target(':b', dependencies=[a])
c = self.make_target(':c', dependencies=[b])
d = self.make_target(':d', dependencies=[c, a])
e = self.make_target(':e', dependencies=[d])
targets = [a, b, c, d, e]
ic = self.cache_manager.check(targets)
all_vts = ic.all_vts
invalid_vts = ic.invalid_vts
self.assertEqual(5, len(invalid_vts))
self.assertEqual(5, len(all_vts))
vts_targets = [vt.targets[0] for vt in all_vts]
self.assertEqual(set(targets), set(vts_targets))
def test_force_invalidate(self):
vt = self.make_vt()
self.assertTrue(vt.valid)
vt.force_invalidate()
self.assertFalse(vt.valid)
def test_invalid_vts_are_cleaned(self):
# Ensure that calling create_results_dir on an invalid target will wipe any pre-existing output.
vt = self.make_vt()
self.assertFalse(self.is_empty(vt.results_dir))
# Force invalidate should change no state under the results_dir.
vt.force_invalidate()
self.assertFalse(self.is_empty(vt.results_dir))
vt.create_results_dir()
self.assertTrue(self.has_symlinked_result_dir(vt))
self.assertTrue(self.is_empty(vt.results_dir))
def test_valid_vts_are_not_cleaned(self):
# No cleaning of results_dir occurs, since create_results_dir short-circuits if the VT is valid.
vt = self.make_vt()
self.assertFalse(self.is_empty(vt.results_dir))
file_names = os.listdir(vt.results_dir)
vt.create_results_dir()
self.assertFalse(self.is_empty(vt.results_dir))
self.assertTrue(self.has_symlinked_result_dir(vt))
# Show that the files inside the directory have not changed during the create_results_dir noop.
self.assertEqual(file_names, os.listdir(vt.results_dir))
def test_illegal_results_dir_cannot_be_updated_to_valid(self):
# A regression test for a former bug. Calling safe_mkdir(vt.results_dir, clean=True) would silently
# delete the results_dir symlink and yet leave any existing crufty content behind in the vt.current_results_dir.
# https://github.com/pantsbuild/pants/issues/4137
# https://github.com/pantsbuild/pants/issues/4051
vt = self.make_vt()
# Show all is right with the world, there is content from the task run and it's visible from both legal result_dirs.
self.assertFalse(self.is_empty(vt.results_dir))
self.assertTrue(self.has_symlinked_result_dir(vt))
vt.force_invalidate()
self.clobber_symlink(vt)
# Arg, and the resultingly unlinked current_results_dir is uncleaned. The two directories have diverging contents!
self.assertFalse(self.has_symlinked_result_dir(vt))
self.assertFalse(self.has_symlinked_result_dir(vt))
# Big consequences- the files used for Products(vt.results_dir) and the cache(vt.current_results_dir) have diverged!
self.assertNotEqual(os.listdir(vt.results_dir), os.listdir(vt.current_results_dir))
# The main protection for this is the exception raised when the cache_manager attempts to mark the VT valid.
self.assertFalse(vt.valid)
with self.assertRaises(VersionedTargetSet.IllegalResultsDir):
vt.update()
def test_exception_for_invalid_vt_result_dirs(self):
# Show that the create_results_dir will error if a previous operation changed the results_dir from a symlink.
vt = self.make_vt()
self.clobber_symlink(vt)
self.assertFalse(self.has_symlinked_result_dir(vt))
# This only is caught here if the VT is still invalid for some reason, otherwise it's caught by the update() method.
vt.force_invalidate()
with self.assertRaisesRegexp(ValueError, r'Path for link.*overwrite an existing directory*'):
vt.create_results_dir()
def test_raises_for_clobbered_symlink(self):
vt = self.make_vt()
self.clobber_symlink(vt)
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*symlink*'):
vt.ensure_legal()
def test_raises_missing_current_results_dir(self):
vt = self.make_vt()
safe_rmtree(vt.current_results_dir)
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*current_results_dir*'):
vt.ensure_legal()
def test_raises_both_clobbered_symlink_and_missing_current_results_dir(self):
vt = self.make_vt()
self.clobber_symlink(vt)
safe_rmtree(vt.current_results_dir)
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*symlink*'):
vt.ensure_legal()
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*current_results_dir*'):
vt.ensure_legal()
def test_for_illegal_vts(self):
# The update() checks this through vts.ensure_legal, checked here since those checks are on different branches.
vt = self.make_vt()
self.clobber_symlink(vt)
vts = VersionedTargetSet.from_versioned_targets([vt])
with self.assertRaises(VersionedTargetSet.IllegalResultsDir):
vts.update()
def invalidated(self,
targets,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=None,
topological_order=False):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
:API: public
:param targets: The targets to check for changes.
:param invalidate_dependents: If True then any targets depending on changed targets are
invalidated.
:param silent: If true, suppress logging information about target invalidation.
:param fingerprint_strategy: A FingerprintStrategy instance, which can do per task,
finer grained fingerprinting of a given Target.
:param topological_order: Whether to invalidate in dependency order.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
:returns: Yields an InvalidationCheck object reflecting the targets.
:rtype: InvalidationCheck
"""
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(
self.workdir,
cache_key_generator,
self._build_invalidator_dir,
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=type(self).__name__,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
invalidation_check = cache_manager.check(
targets, topological_order=topological_order)
self._maybe_create_results_dirs(invalidation_check.all_vts)
if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled(
):
with self.context.new_workunit('cache'):
cached_vts, uncached_vts, uncached_causes = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
self.context.run_tracker.artifact_cache_stats.add_hits(
cache_manager.task_name, cached_targets)
if not silent:
self._report_targets('Using cached artifacts for ',
cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
self.context.run_tracker.artifact_cache_stats.add_misses(
cache_manager.task_name, uncached_targets, uncached_causes)
if not silent:
self._report_targets('No cached artifacts for ',
uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts)
if not silent:
targets = []
for vt in invalidation_check.invalid_vts:
targets.extend(vt.targets)
if len(targets):
msg_elements = [
'Invalidated ',
items_to_report_element(
[t.address.reference() for t in targets], 'target'),
'.'
]
self.context.log.info(*msg_elements)
invalidation_report = self.context.invalidation_report
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='pre-check')
# Cache has been checked to create the full list of invalid VTs.
# Only copy previous_results for this subset of VTs.
if self.incremental:
for vts in invalidation_check.invalid_vts:
vts.copy_previous_results()
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='post-check')
for vt in invalidation_check.invalid_vts:
vt.update()
# Background work to clean up previous builds.
if self.context.options.for_global_scope(
).workdir_max_build_entries is not None:
self._launch_background_workdir_cleanup(invalidation_check.all_vts)
class InvalidationCacheManagerTest(BaseTest):
@staticmethod
def is_empty(dir_name):
return not os.listdir(dir_name)
@staticmethod
def has_symlinked_result_dir(vt):
return os.path.realpath(vt.results_dir) == os.path.realpath(vt.current_results_dir)
@staticmethod
def clobber_symlink(vt):
# Munge the state to mimic a common error found before we added the clean- it accidentally clobbers the symlink!
# Commonly caused by safe_mkdir(vt.results_dir, clean=True), broken up here to keep the tests from being brittle.
safe_rmtree(vt.results_dir)
safe_mkdir(vt.results_dir)
def setUp(self):
super(InvalidationCacheManagerTest, self).setUp()
self._dir = tempfile.mkdtemp()
self.cache_manager = InvalidationCacheManager(
cache_key_generator=CacheKeyGenerator(),
build_invalidator_dir=self ._dir,
invalidate_dependents=True,
)
def tearDown(self):
shutil.rmtree(self._dir, ignore_errors=True)
super(InvalidationCacheManagerTest, self).tearDown()
def make_vt(self, invalid=False):
# Create an arbitrary VT. If invalid is False, it will mimic the state of the VT handed back by a task.
a_target = self.make_target(':a', dependencies=[])
ic = self.cache_manager.check([a_target])
vt = ic.all_vts[0]
if not invalid:
self.task_execute(vt)
vt.update()
return vt
def task_execute(self, vt):
vt.create_results_dir(self._dir)
task_output = os.path.join(vt.results_dir, 'a_file')
self.create_file(task_output, 'foo')
def test_check_marks_all_as_invalid_by_default(self):
a = self.make_target(':a', dependencies=[])
b = self.make_target(':b', dependencies=[a])
c = self.make_target(':c', dependencies=[b])
d = self.make_target(':d', dependencies=[c, a])
e = self.make_target(':e', dependencies=[d])
targets = [a, b, c, d, e]
ic = self.cache_manager.check(targets)
all_vts = ic.all_vts
invalid_vts = ic.invalid_vts
self.assertEquals(5, len(invalid_vts))
self.assertEquals(5, len(all_vts))
vts_targets = [vt.targets[0] for vt in all_vts]
self.assertEquals(set(targets), set(vts_targets))
def test_force_invalidate(self):
vt = self.make_vt()
self.assertTrue(vt.valid)
vt.force_invalidate()
self.assertFalse(vt.valid)
def test_invalid_vts_are_cleaned(self):
# Ensure that calling create_results_dir on an invalid target will wipe any pre-existing output.
vt = self.make_vt()
self.assertFalse(self.is_empty(vt.results_dir))
# Force invalidate should change no state under the results_dir.
vt.force_invalidate()
self.assertFalse(self.is_empty(vt.results_dir))
vt.create_results_dir(self._dir)
self.assertTrue(self.has_symlinked_result_dir(vt))
self.assertTrue(self.is_empty(vt.results_dir))
def test_valid_vts_are_not_cleaned(self):
# No cleaning of results_dir occurs, since create_results_dir short-circuits if the VT is valid.
vt = self.make_vt()
self.assertFalse(self.is_empty(vt.results_dir))
file_names = os.listdir(vt.results_dir)
vt.create_results_dir(self._dir)
self.assertFalse(self.is_empty(vt.results_dir))
self.assertTrue(self.has_symlinked_result_dir(vt))
# Show that the files inside the directory have not changed during the create_results_dir noop.
self.assertEqual(file_names, os.listdir(vt.results_dir))
def test_illegal_results_dir_cannot_be_updated_to_valid(self):
# A regression test for a former bug. Calling safe_mkdir(vt.results_dir, clean=True) would silently
# delete the results_dir symlink and yet leave any existing crufty content behind in the vt.current_results_dir.
# https://github.com/pantsbuild/pants/issues/4137
# https://github.com/pantsbuild/pants/issues/4051
vt = self.make_vt()
# Show all is right with the world, there is content from the task run and it's visible from both legal result_dirs.
self.assertFalse(self.is_empty(vt.results_dir))
self.assertTrue(self.has_symlinked_result_dir(vt))
vt.force_invalidate()
self.clobber_symlink(vt)
# Arg, and the resultingly unlinked current_results_dir is uncleaned. The two directories have diverging contents!
self.assertFalse(self.has_symlinked_result_dir(vt))
self.assertFalse(self.has_symlinked_result_dir(vt))
# Big consequences- the files used for Products(vt.results_dir) and the cache(vt.current_results_dir) have diverged!
self.assertNotEqual(os.listdir(vt.results_dir), os.listdir(vt.current_results_dir))
# The main protection for this is the exception raised when the cache_manager attempts to mark the VT valid.
self.assertFalse(vt.valid)
with self.assertRaises(VersionedTargetSet.IllegalResultsDir):
vt.update()
def test_exception_for_invalid_vt_result_dirs(self):
# Show that the create_results_dir will error if a previous operation changed the results_dir from a symlink.
vt = self.make_vt()
self.clobber_symlink(vt)
self.assertFalse(self.has_symlinked_result_dir(vt))
# This only is caught here if the VT is still invalid for some reason, otherwise it's caught by the update() method.
vt.force_invalidate()
with self.assertRaisesRegexp(ValueError, r'Path for link.*overwrite an existing directory*'):
vt.create_results_dir(self._dir)
def test_raises_for_clobbered_symlink(self):
vt = self.make_vt()
self.clobber_symlink(vt)
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*symlink*'):
vt.ensure_legal()
def test_raises_missing_current_results_dir(self):
vt = self.make_vt()
safe_rmtree(vt.current_results_dir)
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*current_results_dir*'):
vt.ensure_legal()
def test_raises_both_clobbered_symlink_and_missing_current_results_dir(self):
vt = self.make_vt()
self.clobber_symlink(vt)
safe_rmtree(vt.current_results_dir)
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*symlink*'):
vt.ensure_legal()
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*current_results_dir*'):
vt.ensure_legal()
def test_for_illegal_vts(self):
# The update() checks this through vts.ensure_legal, checked here since those checks are on different branches.
vt = self.make_vt()
self.clobber_symlink(vt)
vts = VersionedTargetSet.from_versioned_targets([vt])
with self.assertRaises(VersionedTargetSet.IllegalResultsDir):
vts.update()
def invalidated(self,
targets,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=None,
topological_order=False):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
:API: public
:param targets: The targets to check for changes.
:param invalidate_dependents: If True then any targets depending on changed targets are
invalidated.
:param silent: If true, suppress logging information about target invalidation.
:param fingerprint_strategy: A FingerprintStrategy instance, which can do per task,
finer grained fingerprinting of a given Target.
:param topological_order: Whether to invalidate in dependency order.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
:returns: Yields an InvalidationCheck object reflecting the targets.
:rtype: InvalidationCheck
"""
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(self.workdir,
cache_key_generator,
self._build_invalidator(),
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=type(self).__name__,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
# If this Task's execution has been forced, invalidate all our target fingerprints.
if self._cache_factory.ignore and not self._force_invalidated:
self.invalidate()
self._force_invalidated = True
invalidation_check = cache_manager.check(targets, topological_order=topological_order)
self._maybe_create_results_dirs(invalidation_check.all_vts)
if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled():
with self.context.new_workunit('cache'):
cached_vts, uncached_vts, uncached_causes = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
self.context.run_tracker.artifact_cache_stats.add_hits(cache_manager.task_name,
cached_targets)
if not silent:
self._report_targets('Using cached artifacts for ', cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
self.context.run_tracker.artifact_cache_stats.add_misses(cache_manager.task_name,
uncached_targets,
uncached_causes)
if not silent:
self._report_targets('No cached artifacts for ', uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts)
if not silent:
targets = []
for vt in invalidation_check.invalid_vts:
targets.extend(vt.targets)
if len(targets):
msg_elements = ['Invalidated ',
items_to_report_element([t.address.reference() for t in targets], 'target'),
'.']
self.context.log.info(*msg_elements)
invalidation_report = self.context.invalidation_report
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager, vts.targets, vts.cache_key, vts.valid,
phase='pre-check')
# Cache has been checked to create the full list of invalid VTs.
# Only copy previous_results for this subset of VTs.
if self.incremental:
for vts in invalidation_check.invalid_vts:
vts.copy_previous_results()
# This may seem odd: why would we need to invalidate a VersionedTargetSet that is already
# invalid? But the name force_invalidate() is slightly misleading in this context - what it
# actually does is delete the key file created at the end of the last successful task run.
# This is necessary to avoid the following scenario:
#
# 1) In state A: Task suceeds and writes some output. Key is recorded by the invalidator.
# 2) In state B: Task fails, but writes some output. Key is not recorded.
# 3) After reverting back to state A: The current key is the same as the one recorded at the
# end of step 1), so it looks like no work needs to be done, but actually the task
# must re-run, to overwrite the output written in step 2.
#
# Deleting the file ensures that if a task fails, there is no key for which we might think
# we're in a valid state.
for vts in invalidation_check.invalid_vts:
vts.force_invalidate()
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager, vts.targets, vts.cache_key, vts.valid,
phase='post-check')
for vt in invalidation_check.invalid_vts:
vt.update()
# Background work to clean up previous builds.
if self.context.options.for_global_scope().workdir_max_build_entries is not None:
self._launch_background_workdir_cleanup(invalidation_check.all_vts)
def invalidated(self,
targets,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=None,
topological_order=False):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
:API: public
:param targets: The targets to check for changes.
:param invalidate_dependents: If True then any targets depending on changed targets are
invalidated.
:param silent: If true, suppress logging information about target invalidation.
:param fingerprint_strategy: A FingerprintStrategy instance, which can do per task,
finer grained fingerprinting of a given Target.
:param topological_order: Whether to invalidate in dependency order.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
:returns: Yields an InvalidationCheck object reflecting the targets.
:rtype: InvalidationCheck
"""
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(
self.workdir,
cache_key_generator,
self._build_invalidator(),
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=type(self).__name__,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
# If this Task's execution has been forced, invalidate all our target fingerprints.
if self._cache_factory.ignore and not self._force_invalidated:
self.invalidate()
self._force_invalidated = True
invalidation_check = cache_manager.check(
targets, topological_order=topological_order)
self._maybe_create_results_dirs(invalidation_check.all_vts)
if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled(
):
with self.context.new_workunit('cache'):
cached_vts, uncached_vts, uncached_causes = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
self.context.run_tracker.artifact_cache_stats.add_hits(
cache_manager.task_name, cached_targets)
if not silent:
self._report_targets('Using cached artifacts for ',
cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
self.context.run_tracker.artifact_cache_stats.add_misses(
cache_manager.task_name, uncached_targets, uncached_causes)
if not silent:
self._report_targets('No cached artifacts for ',
uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts)
if not silent:
targets = []
for vt in invalidation_check.invalid_vts:
targets.extend(vt.targets)
if len(targets):
msg_elements = [
'Invalidated ',
items_to_report_element(
[t.address.reference() for t in targets], 'target'),
'.'
]
self.context.log.info(*msg_elements)
invalidation_report = self.context.invalidation_report
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='pre-check')
# Cache has been checked to create the full list of invalid VTs.
# Only copy previous_results for this subset of VTs.
if self.incremental:
for vts in invalidation_check.invalid_vts:
vts.copy_previous_results()
# This may seem odd: why would we need to invalidate a VersionedTargetSet that is already
# invalid? But the name force_invalidate() is slightly misleading in this context - what it
# actually does is delete the key file created at the end of the last successful task run.
# This is necessary to avoid the following scenario:
#
# 1) In state A: Task suceeds and writes some output. Key is recorded by the invalidator.
# 2) In state B: Task fails, but writes some output. Key is not recorded.
# 3) After reverting back to state A: The current key is the same as the one recorded at the
# end of step 1), so it looks like no work needs to be done, but actually the task
# must re-run, to overwrite the output written in step 2.
#
# Deleting the file ensures that if a task fails, there is no key for which we might think
# we're in a valid state.
for vts in invalidation_check.invalid_vts:
vts.force_invalidate()
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='post-check')
for vt in invalidation_check.invalid_vts:
vt.update()
# Background work to clean up previous builds.
if self.context.options.for_global_scope(
).workdir_max_build_entries is not None:
self._launch_background_workdir_cleanup(invalidation_check.all_vts)
