def _do_invalidation_check(self,
fingerprint_strategy,
invalidate_dependents,
targets,
topological_order):
if self._cache_factory.ignore:
cache_key_generator = UncacheableCacheKeyGenerator()
else:
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(self.workdir,
cache_key_generator,
self._build_invalidator(),
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=self._task_name,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
# If this Task's execution has been forced, invalidate all our target fingerprints.
if self._cache_factory.ignore and not self._force_invalidated:
self.invalidate()
self._force_invalidated = True
return cache_manager.check(targets, topological_order=topological_order)
def _do_invalidation_check(self,
fingerprint_strategy,
invalidate_dependents,
targets,
topological_order):
if self._cache_factory.ignore:
cache_key_generator = UncacheableCacheKeyGenerator()
else:
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(self.workdir,
cache_key_generator,
self._build_invalidator,
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=self._task_name,
task_version_slug=self.implementation_version_slug(),
artifact_write_callback=self.maybe_write_artifact)
# If this Task's execution has been forced, invalidate all our target fingerprints.
if self._cache_factory.ignore and not self._force_invalidated:
self.invalidate()
self._force_invalidated = True
return cache_manager.check(targets, topological_order=topological_order)
def setUp(self):
super(InvalidationCacheManagerTest, self).setUp()
self._dir = tempfile.mkdtemp()
self.cache_manager = InvalidationCacheManager(
results_dir_root=os.path.join(self._dir, 'results'),
cache_key_generator=CacheKeyGenerator(),
build_invalidator=BuildInvalidator(os.path.join(self._dir, 'build_invalidator')),
invalidate_dependents=True,
)
def setUp(self):
super().setUp()
self._dir = tempfile.mkdtemp()
self.cache_manager = InvalidationCacheManager(
results_dir_root=os.path.join(self._dir, "results"),
cache_key_generator=CacheKeyGenerator(),
build_invalidator=BuildInvalidator(
os.path.join(self._dir, "build_invalidator")),
invalidate_dependents=True,
task_version_slug="deadbeef",
)
def setUp(self):
super(InvalidationCacheManagerTest, self).setUp()
self._dir = tempfile.mkdtemp()
self.cache_manager = InvalidationCacheManager(
cache_key_generator=CacheKeyGenerator(),
build_invalidator_dir=self ._dir,
invalidate_dependents=True,
)
def setUp(self):
super(InvalidationCacheManagerTest, self).setUp()
self._dir = tempfile.mkdtemp()
self.cache_manager = InvalidationCacheManager(
results_dir_root=os.path.join(self._dir, 'results'),
cache_key_generator=CacheKeyGenerator(),
build_invalidator=BuildInvalidator(os.path.join(self._dir, 'build_invalidator')),
invalidate_dependents=True,
)
def create_cache_manager(self, invalidate_dependents, fingerprint_strategy=None):
"""Creates a cache manager that can be used to invalidate targets on behalf of this task.
Use this if you need to check for invalid targets but can't use the contextmanager created by
invalidated(), e.g., because you don't want to mark the targets as valid when done.
invalidate_dependents: If True then any targets depending on changed targets are invalidated.
fingerprint_strategy: A FingerprintStrategy instance, which can do per task, finer grained
fingerprinting of a given Target.
"""
return InvalidationCacheManager(self._cache_key_generator,
self._build_invalidator_dir,
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=type(self).__name__)
def invalidated(self,
targets,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=None,
topological_order=False):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
:API: public
:param targets: The targets to check for changes.
:param invalidate_dependents: If True then any targets depending on changed targets are
invalidated.
:param silent: If true, suppress logging information about target invalidation.
:param fingerprint_strategy: A FingerprintStrategy instance, which can do per task,
finer grained fingerprinting of a given Target.
:param topological_order: Whether to invalidate in dependency order.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
:returns: Yields an InvalidationCheck object reflecting the targets.
:rtype: InvalidationCheck
"""
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(self.workdir,
cache_key_generator,
self._build_invalidator_dir,
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=type(self).__name__,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
invalidation_check = cache_manager.check(targets, topological_order=topological_order)
self._maybe_create_results_dirs(invalidation_check.all_vts)
if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled():
with self.context.new_workunit('cache'):
cached_vts, uncached_vts, uncached_causes = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
self.context.run_tracker.artifact_cache_stats.add_hits(cache_manager.task_name,
cached_targets)
if not silent:
self._report_targets('Using cached artifacts for ', cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
self.context.run_tracker.artifact_cache_stats.add_misses(cache_manager.task_name,
uncached_targets,
uncached_causes)
if not silent:
self._report_targets('No cached artifacts for ', uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts)
if not silent:
targets = []
for vt in invalidation_check.invalid_vts:
targets.extend(vt.targets)
if len(targets):
msg_elements = ['Invalidated ',
items_to_report_element([t.address.reference() for t in targets], 'target'),
'.']
self.context.log.info(*msg_elements)
invalidation_report = self.context.invalidation_report
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager, vts.targets, vts.cache_key, vts.valid,
phase='pre-check')
# Cache has been checked to create the full list of invalid VTs.
# Only copy previous_results for this subset of VTs.
for vts in invalidation_check.invalid_vts:
if self.incremental:
vts.copy_previous_results(self.workdir)
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager, vts.targets, vts.cache_key, vts.valid,
phase='post-check')
for vt in invalidation_check.invalid_vts:
vt.update()
# Background work to clean up previous builds.
if self.context.options.for_global_scope().workdir_max_build_entries is not None:
self._launch_background_workdir_cleanup(invalidation_check.all_vts)
class InvalidationCacheManagerTest(TestBase):
@staticmethod
def is_empty(dir_name):
return not os.listdir(dir_name)
@staticmethod
def has_symlinked_result_dir(vt):
return os.path.realpath(vt.results_dir) == os.path.realpath(vt.current_results_dir)
@staticmethod
def clobber_symlink(vt):
# Munge the state to mimic a common error found before we added the clean- it accidentally clobbers the symlink!
# Commonly caused by safe_mkdir(vt.results_dir, clean=True), broken up here to keep the tests from being brittle.
safe_rmtree(vt.results_dir)
safe_mkdir(vt.results_dir)
def setUp(self):
super(InvalidationCacheManagerTest, self).setUp()
self._dir = tempfile.mkdtemp()
self.cache_manager = InvalidationCacheManager(
results_dir_root=os.path.join(self._dir, 'results'),
cache_key_generator=CacheKeyGenerator(),
build_invalidator=BuildInvalidator(os.path.join(self._dir, 'build_invalidator')),
invalidate_dependents=True,
)
def tearDown(self):
shutil.rmtree(self._dir, ignore_errors=True)
super(InvalidationCacheManagerTest, self).tearDown()
def make_vt(self):
# Create an arbitrary VT. It will mimic the state of the VT handed back by a task.
a_target = self.make_target(':a', dependencies=[])
ic = self.cache_manager.check([a_target])
vt = ic.all_vts[0]
self.task_execute(vt)
vt.update()
return vt
def task_execute(self, vt):
vt.create_results_dir()
task_output = os.path.join(vt.results_dir, 'a_file')
self.create_file(task_output, 'foo')
def test_creates_stable_result_dir_symlink(self):
vt = self.make_vt()
vt.create_results_dir()
parent, unstable_dir_name = os.path.split(vt._current_results_dir)
self.assertSetEqual({'current', unstable_dir_name}, set(os.listdir(parent)))
symlink = os.path.join(parent, 'current')
self.assertTrue(os.path.islink(symlink))
self.assertEqual(unstable_dir_name, os.readlink(symlink))
self.assertEqual(symlink, vt.results_dir)
# Repoint the symlink, but keep the vt valid (simulates the case where the underlying target's
# version has changed, so the symlink is pointed to that version, but now the version has
# reverted, so we must point it back).
os.unlink(symlink)
os.symlink(unstable_dir_name + '.other', symlink)
vt.create_results_dir()
self.assertEqual(unstable_dir_name, os.readlink(symlink))
def test_creates_stable_results_dir_prefix_symlink(self):
parent, unstable_dir_name = os.path.split(self.cache_manager._results_dir_prefix)
self.assertSetEqual({'current', unstable_dir_name}, set(os.listdir(parent)))
symlink = os.path.join(parent, 'current')
self.assertTrue(os.path.islink(symlink))
self.assertTrue(unstable_dir_name, os.readlink(symlink))
def test_check_marks_all_as_invalid_by_default(self):
a = self.make_target(':a', dependencies=[])
b = self.make_target(':b', dependencies=[a])
c = self.make_target(':c', dependencies=[b])
d = self.make_target(':d', dependencies=[c, a])
e = self.make_target(':e', dependencies=[d])
targets = [a, b, c, d, e]
ic = self.cache_manager.check(targets)
all_vts = ic.all_vts
invalid_vts = ic.invalid_vts
self.assertEqual(5, len(invalid_vts))
self.assertEqual(5, len(all_vts))
vts_targets = [vt.targets[0] for vt in all_vts]
self.assertEqual(set(targets), set(vts_targets))
def test_force_invalidate(self):
vt = self.make_vt()
self.assertTrue(vt.valid)
vt.force_invalidate()
self.assertFalse(vt.valid)
def test_invalid_vts_are_cleaned(self):
# Ensure that calling create_results_dir on an invalid target will wipe any pre-existing output.
vt = self.make_vt()
self.assertFalse(self.is_empty(vt.results_dir))
# Force invalidate should change no state under the results_dir.
vt.force_invalidate()
self.assertFalse(self.is_empty(vt.results_dir))
vt.create_results_dir()
self.assertTrue(self.has_symlinked_result_dir(vt))
self.assertTrue(self.is_empty(vt.results_dir))
def test_valid_vts_are_not_cleaned(self):
# No cleaning of results_dir occurs, since create_results_dir short-circuits if the VT is valid.
vt = self.make_vt()
self.assertFalse(self.is_empty(vt.results_dir))
file_names = os.listdir(vt.results_dir)
vt.create_results_dir()
self.assertFalse(self.is_empty(vt.results_dir))
self.assertTrue(self.has_symlinked_result_dir(vt))
# Show that the files inside the directory have not changed during the create_results_dir noop.
self.assertEqual(file_names, os.listdir(vt.results_dir))
def test_illegal_results_dir_cannot_be_updated_to_valid(self):
# A regression test for a former bug. Calling safe_mkdir(vt.results_dir, clean=True) would silently
# delete the results_dir symlink and yet leave any existing crufty content behind in the vt.current_results_dir.
# https://github.com/pantsbuild/pants/issues/4137
# https://github.com/pantsbuild/pants/issues/4051
vt = self.make_vt()
# Show all is right with the world, there is content from the task run and it's visible from both legal result_dirs.
self.assertFalse(self.is_empty(vt.results_dir))
self.assertTrue(self.has_symlinked_result_dir(vt))
vt.force_invalidate()
self.clobber_symlink(vt)
# Arg, and the resultingly unlinked current_results_dir is uncleaned. The two directories have diverging contents!
self.assertFalse(self.has_symlinked_result_dir(vt))
self.assertFalse(self.has_symlinked_result_dir(vt))
# Big consequences- the files used for Products(vt.results_dir) and the cache(vt.current_results_dir) have diverged!
self.assertNotEqual(os.listdir(vt.results_dir), os.listdir(vt.current_results_dir))
# The main protection for this is the exception raised when the cache_manager attempts to mark the VT valid.
self.assertFalse(vt.valid)
with self.assertRaises(VersionedTargetSet.IllegalResultsDir):
vt.update()
def test_exception_for_invalid_vt_result_dirs(self):
# Show that the create_results_dir will error if a previous operation changed the results_dir from a symlink.
vt = self.make_vt()
self.clobber_symlink(vt)
self.assertFalse(self.has_symlinked_result_dir(vt))
# This only is caught here if the VT is still invalid for some reason, otherwise it's caught by the update() method.
vt.force_invalidate()
with self.assertRaisesRegexp(ValueError, r'Path for link.*overwrite an existing directory*'):
vt.create_results_dir()
def test_raises_for_clobbered_symlink(self):
vt = self.make_vt()
self.clobber_symlink(vt)
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*symlink*'):
vt.ensure_legal()
def test_raises_missing_current_results_dir(self):
vt = self.make_vt()
safe_rmtree(vt.current_results_dir)
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*current_results_dir*'):
vt.ensure_legal()
def test_raises_both_clobbered_symlink_and_missing_current_results_dir(self):
vt = self.make_vt()
self.clobber_symlink(vt)
safe_rmtree(vt.current_results_dir)
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*symlink*'):
vt.ensure_legal()
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*current_results_dir*'):
vt.ensure_legal()
def test_for_illegal_vts(self):
# The update() checks this through vts.ensure_legal, checked here since those checks are on different branches.
vt = self.make_vt()
self.clobber_symlink(vt)
vts = VersionedTargetSet.from_versioned_targets([vt])
with self.assertRaises(VersionedTargetSet.IllegalResultsDir):
vts.update()
def __init__(self, tmpdir):
InvalidationCacheManager.__init__(self,
AppendingCacheKeyGenerator(),
tmpdir, True)
def __init__(self, tmpdir): InvalidationCacheManager.__init__(self, AppendingCacheKeyGenerator(), tmpdir, True)
def invalidated(self,
targets,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=None,
topological_order=False):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
:API: public
:param targets: The targets to check for changes.
:param invalidate_dependents: If True then any targets depending on changed targets are
invalidated.
:param silent: If true, suppress logging information about target invalidation.
:param fingerprint_strategy: A FingerprintStrategy instance, which can do per task,
finer grained fingerprinting of a given Target.
:param topological_order: Whether to invalidate in dependency order.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
:returns: Yields an InvalidationCheck object reflecting the targets.
:rtype: InvalidationCheck
"""
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(
self.workdir,
cache_key_generator,
self._build_invalidator_dir,
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=type(self).__name__,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
invalidation_check = cache_manager.check(
targets, topological_order=topological_order)
self._maybe_create_results_dirs(invalidation_check.all_vts)
if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled(
):
with self.context.new_workunit('cache'):
cached_vts, uncached_vts, uncached_causes = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
self.context.run_tracker.artifact_cache_stats.add_hits(
cache_manager.task_name, cached_targets)
if not silent:
self._report_targets('Using cached artifacts for ',
cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
self.context.run_tracker.artifact_cache_stats.add_misses(
cache_manager.task_name, uncached_targets, uncached_causes)
if not silent:
self._report_targets('No cached artifacts for ',
uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts)
if not silent:
targets = []
for vt in invalidation_check.invalid_vts:
targets.extend(vt.targets)
if len(targets):
msg_elements = [
'Invalidated ',
items_to_report_element(
[t.address.reference() for t in targets], 'target'),
'.'
]
self.context.log.info(*msg_elements)
invalidation_report = self.context.invalidation_report
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='pre-check')
# Cache has been checked to create the full list of invalid VTs.
# Only copy previous_results for this subset of VTs.
if self.incremental:
for vts in invalidation_check.invalid_vts:
vts.copy_previous_results()
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='post-check')
for vt in invalidation_check.invalid_vts:
vt.update()
# Background work to clean up previous builds.
if self.context.options.for_global_scope(
).workdir_max_build_entries is not None:
self._launch_background_workdir_cleanup(invalidation_check.all_vts)
class InvalidationCacheManagerTest(BaseTest):
@staticmethod
def is_empty(dir_name):
return not os.listdir(dir_name)
@staticmethod
def has_symlinked_result_dir(vt):
return os.path.realpath(vt.results_dir) == os.path.realpath(vt.current_results_dir)
@staticmethod
def clobber_symlink(vt):
# Munge the state to mimic a common error found before we added the clean- it accidentally clobbers the symlink!
# Commonly caused by safe_mkdir(vt.results_dir, clean=True), broken up here to keep the tests from being brittle.
safe_rmtree(vt.results_dir)
safe_mkdir(vt.results_dir)
def setUp(self):
super(InvalidationCacheManagerTest, self).setUp()
self._dir = tempfile.mkdtemp()
self.cache_manager = InvalidationCacheManager(
cache_key_generator=CacheKeyGenerator(),
build_invalidator_dir=self ._dir,
invalidate_dependents=True,
)
def tearDown(self):
shutil.rmtree(self._dir, ignore_errors=True)
super(InvalidationCacheManagerTest, self).tearDown()
def make_vt(self, invalid=False):
# Create an arbitrary VT. If invalid is False, it will mimic the state of the VT handed back by a task.
a_target = self.make_target(':a', dependencies=[])
ic = self.cache_manager.check([a_target])
vt = ic.all_vts[0]
if not invalid:
self.task_execute(vt)
vt.update()
return vt
def task_execute(self, vt):
vt.create_results_dir(self._dir)
task_output = os.path.join(vt.results_dir, 'a_file')
self.create_file(task_output, 'foo')
def test_check_marks_all_as_invalid_by_default(self):
a = self.make_target(':a', dependencies=[])
b = self.make_target(':b', dependencies=[a])
c = self.make_target(':c', dependencies=[b])
d = self.make_target(':d', dependencies=[c, a])
e = self.make_target(':e', dependencies=[d])
targets = [a, b, c, d, e]
ic = self.cache_manager.check(targets)
all_vts = ic.all_vts
invalid_vts = ic.invalid_vts
self.assertEquals(5, len(invalid_vts))
self.assertEquals(5, len(all_vts))
vts_targets = [vt.targets[0] for vt in all_vts]
self.assertEquals(set(targets), set(vts_targets))
def test_force_invalidate(self):
vt = self.make_vt()
self.assertTrue(vt.valid)
vt.force_invalidate()
self.assertFalse(vt.valid)
def test_invalid_vts_are_cleaned(self):
# Ensure that calling create_results_dir on an invalid target will wipe any pre-existing output.
vt = self.make_vt()
self.assertFalse(self.is_empty(vt.results_dir))
# Force invalidate should change no state under the results_dir.
vt.force_invalidate()
self.assertFalse(self.is_empty(vt.results_dir))
vt.create_results_dir(self._dir)
self.assertTrue(self.has_symlinked_result_dir(vt))
self.assertTrue(self.is_empty(vt.results_dir))
def test_valid_vts_are_not_cleaned(self):
# No cleaning of results_dir occurs, since create_results_dir short-circuits if the VT is valid.
vt = self.make_vt()
self.assertFalse(self.is_empty(vt.results_dir))
file_names = os.listdir(vt.results_dir)
vt.create_results_dir(self._dir)
self.assertFalse(self.is_empty(vt.results_dir))
self.assertTrue(self.has_symlinked_result_dir(vt))
# Show that the files inside the directory have not changed during the create_results_dir noop.
self.assertEqual(file_names, os.listdir(vt.results_dir))
def test_illegal_results_dir_cannot_be_updated_to_valid(self):
# A regression test for a former bug. Calling safe_mkdir(vt.results_dir, clean=True) would silently
# delete the results_dir symlink and yet leave any existing crufty content behind in the vt.current_results_dir.
# https://github.com/pantsbuild/pants/issues/4137
# https://github.com/pantsbuild/pants/issues/4051
vt = self.make_vt()
# Show all is right with the world, there is content from the task run and it's visible from both legal result_dirs.
self.assertFalse(self.is_empty(vt.results_dir))
self.assertTrue(self.has_symlinked_result_dir(vt))
vt.force_invalidate()
self.clobber_symlink(vt)
# Arg, and the resultingly unlinked current_results_dir is uncleaned. The two directories have diverging contents!
self.assertFalse(self.has_symlinked_result_dir(vt))
self.assertFalse(self.has_symlinked_result_dir(vt))
# Big consequences- the files used for Products(vt.results_dir) and the cache(vt.current_results_dir) have diverged!
self.assertNotEqual(os.listdir(vt.results_dir), os.listdir(vt.current_results_dir))
# The main protection for this is the exception raised when the cache_manager attempts to mark the VT valid.
self.assertFalse(vt.valid)
with self.assertRaises(VersionedTargetSet.IllegalResultsDir):
vt.update()
def test_exception_for_invalid_vt_result_dirs(self):
# Show that the create_results_dir will error if a previous operation changed the results_dir from a symlink.
vt = self.make_vt()
self.clobber_symlink(vt)
self.assertFalse(self.has_symlinked_result_dir(vt))
# This only is caught here if the VT is still invalid for some reason, otherwise it's caught by the update() method.
vt.force_invalidate()
with self.assertRaisesRegexp(ValueError, r'Path for link.*overwrite an existing directory*'):
vt.create_results_dir(self._dir)
def test_raises_for_clobbered_symlink(self):
vt = self.make_vt()
self.clobber_symlink(vt)
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*symlink*'):
vt.ensure_legal()
def test_raises_missing_current_results_dir(self):
vt = self.make_vt()
safe_rmtree(vt.current_results_dir)
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*current_results_dir*'):
vt.ensure_legal()
def test_raises_both_clobbered_symlink_and_missing_current_results_dir(self):
vt = self.make_vt()
self.clobber_symlink(vt)
safe_rmtree(vt.current_results_dir)
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*symlink*'):
vt.ensure_legal()
with self.assertRaisesRegexp(VersionedTargetSet.IllegalResultsDir, r'The.*current_results_dir*'):
vt.ensure_legal()
def test_for_illegal_vts(self):
# The update() checks this through vts.ensure_legal, checked here since those checks are on different branches.
vt = self.make_vt()
self.clobber_symlink(vt)
vts = VersionedTargetSet.from_versioned_targets([vt])
with self.assertRaises(VersionedTargetSet.IllegalResultsDir):
vts.update()
def invalidated(self,
targets,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=None,
topological_order=False):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
:API: public
:param targets: The targets to check for changes.
:param invalidate_dependents: If True then any targets depending on changed targets are
invalidated.
:param silent: If true, suppress logging information about target invalidation.
:param fingerprint_strategy: A FingerprintStrategy instance, which can do per task,
finer grained fingerprinting of a given Target.
:param topological_order: Whether to invalidate in dependency order.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
:returns: Yields an InvalidationCheck object reflecting the targets.
:rtype: InvalidationCheck
"""
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(self.workdir,
cache_key_generator,
self._build_invalidator(),
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=type(self).__name__,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
# If this Task's execution has been forced, invalidate all our target fingerprints.
if self._cache_factory.ignore and not self._force_invalidated:
self.invalidate()
self._force_invalidated = True
invalidation_check = cache_manager.check(targets, topological_order=topological_order)
self._maybe_create_results_dirs(invalidation_check.all_vts)
if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled():
with self.context.new_workunit('cache'):
cached_vts, uncached_vts, uncached_causes = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
self.context.run_tracker.artifact_cache_stats.add_hits(cache_manager.task_name,
cached_targets)
if not silent:
self._report_targets('Using cached artifacts for ', cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
self.context.run_tracker.artifact_cache_stats.add_misses(cache_manager.task_name,
uncached_targets,
uncached_causes)
if not silent:
self._report_targets('No cached artifacts for ', uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts)
if not silent:
targets = []
for vt in invalidation_check.invalid_vts:
targets.extend(vt.targets)
if len(targets):
msg_elements = ['Invalidated ',
items_to_report_element([t.address.reference() for t in targets], 'target'),
'.']
self.context.log.info(*msg_elements)
invalidation_report = self.context.invalidation_report
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager, vts.targets, vts.cache_key, vts.valid,
phase='pre-check')
# Cache has been checked to create the full list of invalid VTs.
# Only copy previous_results for this subset of VTs.
if self.incremental:
for vts in invalidation_check.invalid_vts:
vts.copy_previous_results()
# This may seem odd: why would we need to invalidate a VersionedTargetSet that is already
# invalid? But the name force_invalidate() is slightly misleading in this context - what it
# actually does is delete the key file created at the end of the last successful task run.
# This is necessary to avoid the following scenario:
#
# 1) In state A: Task suceeds and writes some output. Key is recorded by the invalidator.
# 2) In state B: Task fails, but writes some output. Key is not recorded.
# 3) After reverting back to state A: The current key is the same as the one recorded at the
# end of step 1), so it looks like no work needs to be done, but actually the task
# must re-run, to overwrite the output written in step 2.
#
# Deleting the file ensures that if a task fails, there is no key for which we might think
# we're in a valid state.
for vts in invalidation_check.invalid_vts:
vts.force_invalidate()
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager, vts.targets, vts.cache_key, vts.valid,
phase='post-check')
for vt in invalidation_check.invalid_vts:
vt.update()
# Background work to clean up previous builds.
if self.context.options.for_global_scope().workdir_max_build_entries is not None:
self._launch_background_workdir_cleanup(invalidation_check.all_vts)
def invalidated(self,
targets,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=None,
topological_order=False):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
:API: public
:param targets: The targets to check for changes.
:param invalidate_dependents: If True then any targets depending on changed targets are
invalidated.
:param silent: If true, suppress logging information about target invalidation.
:param fingerprint_strategy: A FingerprintStrategy instance, which can do per task,
finer grained fingerprinting of a given Target.
:param topological_order: Whether to invalidate in dependency order.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
:returns: Yields an InvalidationCheck object reflecting the targets.
:rtype: InvalidationCheck
"""
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(
self.workdir,
cache_key_generator,
self._build_invalidator(),
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=type(self).__name__,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
# If this Task's execution has been forced, invalidate all our target fingerprints.
if self._cache_factory.ignore and not self._force_invalidated:
self.invalidate()
self._force_invalidated = True
invalidation_check = cache_manager.check(
targets, topological_order=topological_order)
self._maybe_create_results_dirs(invalidation_check.all_vts)
if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled(
):
with self.context.new_workunit('cache'):
cached_vts, uncached_vts, uncached_causes = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
self.context.run_tracker.artifact_cache_stats.add_hits(
cache_manager.task_name, cached_targets)
if not silent:
self._report_targets('Using cached artifacts for ',
cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
self.context.run_tracker.artifact_cache_stats.add_misses(
cache_manager.task_name, uncached_targets, uncached_causes)
if not silent:
self._report_targets('No cached artifacts for ',
uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts)
if not silent:
targets = []
for vt in invalidation_check.invalid_vts:
targets.extend(vt.targets)
if len(targets):
msg_elements = [
'Invalidated ',
items_to_report_element(
[t.address.reference() for t in targets], 'target'),
'.'
]
self.context.log.info(*msg_elements)
invalidation_report = self.context.invalidation_report
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='pre-check')
# Cache has been checked to create the full list of invalid VTs.
# Only copy previous_results for this subset of VTs.
if self.incremental:
for vts in invalidation_check.invalid_vts:
vts.copy_previous_results()
# This may seem odd: why would we need to invalidate a VersionedTargetSet that is already
# invalid? But the name force_invalidate() is slightly misleading in this context - what it
# actually does is delete the key file created at the end of the last successful task run.
# This is necessary to avoid the following scenario:
#
# 1) In state A: Task suceeds and writes some output. Key is recorded by the invalidator.
# 2) In state B: Task fails, but writes some output. Key is not recorded.
# 3) After reverting back to state A: The current key is the same as the one recorded at the
# end of step 1), so it looks like no work needs to be done, but actually the task
# must re-run, to overwrite the output written in step 2.
#
# Deleting the file ensures that if a task fails, there is no key for which we might think
# we're in a valid state.
for vts in invalidation_check.invalid_vts:
vts.force_invalidate()
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='post-check')
for vt in invalidation_check.invalid_vts:
vt.update()
# Background work to clean up previous builds.
if self.context.options.for_global_scope(
).workdir_max_build_entries is not None:
self._launch_background_workdir_cleanup(invalidation_check.all_vts)
