def __init__(self, context, workdir):
"""Subclass __init__ methods, if defined, *must* follow this idiom:
class MyTask(Task):
def __init__(self, *args, **kwargs):
super(MyTask, self).__init__(*args, **kwargs)
...
This allows us to change Task.__init__()'s arguments without
changing every subclass. If the subclass does not need its own
initialization, this method can (and should) be omitted entirely.
"""
super(TaskBase, self).__init__()
self.context = context
self._workdir = workdir
# TODO: It would be nice to use self.get_options().cache_key_gen_version here, because then
# we could have a separate value for each scope if we really wanted to. However we can't
# access per-task options in Task.__init__ because GroupTask.__init__ calls it with the
# group task's scope, which isn't currently in the known scopes we generate options for.
self._cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version)
self._cache_key_errors = set()
self._build_invalidator_dir = os.path.join(
self.context.options.for_global_scope().pants_workdir,
'build_invalidator', self.stable_name())
self._cache_factory = CacheSetup.create_cache_factory_for_task(self)
self._options_fingerprinter = OptionsFingerprinter(
self.context.build_graph)
self._fingerprint = None
def __init__(self,
python_setup,
python_repos,
ivy_bootstrapper,
thrift_binary_factory,
interpreter,
builder,
targets,
platforms,
extra_requirements=None,
log=None):
self._python_setup = python_setup
self._python_repos = python_repos
self._ivy_bootstrapper = ivy_bootstrapper
self._thrift_binary_factory = thrift_binary_factory
self._interpreter = interpreter
self._builder = builder
self._targets = targets
self._platforms = platforms
self._extra_requirements = list(
extra_requirements) if extra_requirements else []
self._logger = log or logger
# Note: unrelated to the general pants artifact cache.
self._artifact_cache_root = os.path.join(
self._python_setup.artifact_cache_dir,
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._artifact_cache_root)
def _do_invalidation_check(self,
fingerprint_strategy,
invalidate_dependents,
targets,
topological_order):
if self._cache_factory.ignore:
cache_key_generator = UncacheableCacheKeyGenerator()
else:
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(self.workdir,
cache_key_generator,
self._build_invalidator(),
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=self._task_name,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
# If this Task's execution has been forced, invalidate all our target fingerprints.
if self._cache_factory.ignore and not self._force_invalidated:
self.invalidate()
self._force_invalidated = True
return cache_manager.check(targets, topological_order=topological_order)
def setUp(self):
super(InvalidationCacheManagerTest, self).setUp()
self._dir = tempfile.mkdtemp()
self.cache_manager = InvalidationCacheManager(
results_dir_root=os.path.join(self._dir, 'results'),
cache_key_generator=CacheKeyGenerator(),
build_invalidator=BuildInvalidator(os.path.join(self._dir, 'build_invalidator')),
invalidate_dependents=True,
)
def setUp(self):
super().setUp()
self._dir = tempfile.mkdtemp()
self.cache_manager = InvalidationCacheManager(
results_dir_root=os.path.join(self._dir, "results"),
cache_key_generator=CacheKeyGenerator(),
build_invalidator=BuildInvalidator(
os.path.join(self._dir, "build_invalidator")),
invalidate_dependents=True,
task_version_slug="deadbeef",
)
def invalidated(self,
targets,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=None,
topological_order=False):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
:API: public
:param targets: The targets to check for changes.
:param invalidate_dependents: If True then any targets depending on changed targets are
invalidated.
:param silent: If true, suppress logging information about target invalidation.
:param fingerprint_strategy: A FingerprintStrategy instance, which can do per task,
finer grained fingerprinting of a given Target.
:param topological_order: Whether to invalidate in dependency order.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
:returns: Yields an InvalidationCheck object reflecting the targets.
:rtype: InvalidationCheck
"""
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(
self.workdir,
cache_key_generator,
self._build_invalidator_dir,
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=type(self).__name__,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
invalidation_check = cache_manager.check(
targets, topological_order=topological_order)
self._maybe_create_results_dirs(invalidation_check.all_vts)
if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled(
):
with self.context.new_workunit('cache'):
cached_vts, uncached_vts, uncached_causes = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
self.context.run_tracker.artifact_cache_stats.add_hits(
cache_manager.task_name, cached_targets)
if not silent:
self._report_targets('Using cached artifacts for ',
cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
self.context.run_tracker.artifact_cache_stats.add_misses(
cache_manager.task_name, uncached_targets, uncached_causes)
if not silent:
self._report_targets('No cached artifacts for ',
uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts)
if not silent:
targets = []
for vt in invalidation_check.invalid_vts:
targets.extend(vt.targets)
if len(targets):
msg_elements = [
'Invalidated ',
items_to_report_element(
[t.address.reference() for t in targets], 'target'),
'.'
]
self.context.log.info(*msg_elements)
invalidation_report = self.context.invalidation_report
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='pre-check')
# Cache has been checked to create the full list of invalid VTs.
# Only copy previous_results for this subset of VTs.
if self.incremental:
for vts in invalidation_check.invalid_vts:
vts.copy_previous_results()
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='post-check')
for vt in invalidation_check.invalid_vts:
vt.update()
# Background work to clean up previous builds.
if self.context.options.for_global_scope(
).workdir_max_build_entries is not None:
self._launch_background_workdir_cleanup(invalidation_check.all_vts)
def test_env(content=TEST_CONTENT):
with temporary_dir() as d:
with tempfile.NamedTemporaryFile() as f:
f.write(content)
f.flush()
yield f, CacheKeyGenerator(), BuildInvalidator(d)
def invalidated(self,
targets,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=None,
topological_order=False):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
:API: public
:param targets: The targets to check for changes.
:param invalidate_dependents: If True then any targets depending on changed targets are
invalidated.
:param silent: If true, suppress logging information about target invalidation.
:param fingerprint_strategy: A FingerprintStrategy instance, which can do per task,
finer grained fingerprinting of a given Target.
:param topological_order: Whether to invalidate in dependency order.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
:returns: Yields an InvalidationCheck object reflecting the targets.
:rtype: InvalidationCheck
"""
cache_key_generator = CacheKeyGenerator(
self.context.options.for_global_scope().cache_key_gen_version,
self.fingerprint)
cache_manager = InvalidationCacheManager(
self.workdir,
cache_key_generator,
self._build_invalidator(),
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=type(self).__name__,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
# If this Task's execution has been forced, invalidate all our target fingerprints.
if self._cache_factory.ignore and not self._force_invalidated:
self.invalidate()
self._force_invalidated = True
invalidation_check = cache_manager.check(
targets, topological_order=topological_order)
self._maybe_create_results_dirs(invalidation_check.all_vts)
if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled(
):
with self.context.new_workunit('cache'):
cached_vts, uncached_vts, uncached_causes = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
self.context.run_tracker.artifact_cache_stats.add_hits(
cache_manager.task_name, cached_targets)
if not silent:
self._report_targets('Using cached artifacts for ',
cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
self.context.run_tracker.artifact_cache_stats.add_misses(
cache_manager.task_name, uncached_targets, uncached_causes)
if not silent:
self._report_targets('No cached artifacts for ',
uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts)
if not silent:
targets = []
for vt in invalidation_check.invalid_vts:
targets.extend(vt.targets)
if len(targets):
msg_elements = [
'Invalidated ',
items_to_report_element(
[t.address.reference() for t in targets], 'target'),
'.'
]
self.context.log.info(*msg_elements)
invalidation_report = self.context.invalidation_report
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='pre-check')
# Cache has been checked to create the full list of invalid VTs.
# Only copy previous_results for this subset of VTs.
if self.incremental:
for vts in invalidation_check.invalid_vts:
vts.copy_previous_results()
# This may seem odd: why would we need to invalidate a VersionedTargetSet that is already
# invalid? But the name force_invalidate() is slightly misleading in this context - what it
# actually does is delete the key file created at the end of the last successful task run.
# This is necessary to avoid the following scenario:
#
# 1) In state A: Task suceeds and writes some output. Key is recorded by the invalidator.
# 2) In state B: Task fails, but writes some output. Key is not recorded.
# 3) After reverting back to state A: The current key is the same as the one recorded at the
# end of step 1), so it looks like no work needs to be done, but actually the task
# must re-run, to overwrite the output written in step 2.
#
# Deleting the file ensures that if a task fails, there is no key for which we might think
# we're in a valid state.
for vts in invalidation_check.invalid_vts:
vts.force_invalidate()
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='post-check')
for vt in invalidation_check.invalid_vts:
vt.update()
# Background work to clean up previous builds.
if self.context.options.for_global_scope(
).workdir_max_build_entries is not None:
self._launch_background_workdir_cleanup(invalidation_check.all_vts)
