def __init__(self, context, workdir): """Subclass __init__ methods, if defined, *must* follow this idiom: class MyTask(Task): def __init__(self, *args, **kwargs): super(MyTask, self).__init__(*args, **kwargs) ... This allows us to change Task.__init__()'s arguments without changing every subclass. If the subclass does not need its own initialization, this method can (and should) be omitted entirely. """ super(TaskBase, self).__init__() self.context = context self._workdir = workdir # TODO: It would be nice to use self.get_options().cache_key_gen_version here, because then # we could have a separate value for each scope if we really wanted to. However we can't # access per-task options in Task.__init__ because GroupTask.__init__ calls it with the # group task's scope, which isn't currently in the known scopes we generate options for. self._cache_key_generator = CacheKeyGenerator( self.context.options.for_global_scope().cache_key_gen_version) self._cache_key_errors = set() self._build_invalidator_dir = os.path.join( self.context.options.for_global_scope().pants_workdir, 'build_invalidator', self.stable_name()) self._cache_factory = CacheSetup.create_cache_factory_for_task(self) self._options_fingerprinter = OptionsFingerprinter( self.context.build_graph) self._fingerprint = None
def __init__(self, python_setup, python_repos, ivy_bootstrapper, thrift_binary_factory, interpreter, builder, targets, platforms, extra_requirements=None, log=None): self._python_setup = python_setup self._python_repos = python_repos self._ivy_bootstrapper = ivy_bootstrapper self._thrift_binary_factory = thrift_binary_factory self._interpreter = interpreter self._builder = builder self._targets = targets self._platforms = platforms self._extra_requirements = list( extra_requirements) if extra_requirements else [] self._logger = log or logger # Note: unrelated to the general pants artifact cache. self._artifact_cache_root = os.path.join( self._python_setup.artifact_cache_dir, str(self._interpreter.identity)) self._key_generator = CacheKeyGenerator() self._build_invalidator = BuildInvalidator(self._artifact_cache_root)
def _do_invalidation_check(self, fingerprint_strategy, invalidate_dependents, targets, topological_order): if self._cache_factory.ignore: cache_key_generator = UncacheableCacheKeyGenerator() else: cache_key_generator = CacheKeyGenerator( self.context.options.for_global_scope().cache_key_gen_version, self.fingerprint) cache_manager = InvalidationCacheManager(self.workdir, cache_key_generator, self._build_invalidator(), invalidate_dependents, fingerprint_strategy=fingerprint_strategy, invalidation_report=self.context.invalidation_report, task_name=self._task_name, task_version=self.implementation_version_str(), artifact_write_callback=self.maybe_write_artifact) # If this Task's execution has been forced, invalidate all our target fingerprints. if self._cache_factory.ignore and not self._force_invalidated: self.invalidate() self._force_invalidated = True return cache_manager.check(targets, topological_order=topological_order)
def setUp(self): super(InvalidationCacheManagerTest, self).setUp() self._dir = tempfile.mkdtemp() self.cache_manager = InvalidationCacheManager( results_dir_root=os.path.join(self._dir, 'results'), cache_key_generator=CacheKeyGenerator(), build_invalidator=BuildInvalidator(os.path.join(self._dir, 'build_invalidator')), invalidate_dependents=True, )
def setUp(self): super().setUp() self._dir = tempfile.mkdtemp() self.cache_manager = InvalidationCacheManager( results_dir_root=os.path.join(self._dir, "results"), cache_key_generator=CacheKeyGenerator(), build_invalidator=BuildInvalidator( os.path.join(self._dir, "build_invalidator")), invalidate_dependents=True, task_version_slug="deadbeef", )
def invalidated(self, targets, invalidate_dependents=False, silent=False, fingerprint_strategy=None, topological_order=False): """Checks targets for invalidation, first checking the artifact cache. Subclasses call this to figure out what to work on. :API: public :param targets: The targets to check for changes. :param invalidate_dependents: If True then any targets depending on changed targets are invalidated. :param silent: If true, suppress logging information about target invalidation. :param fingerprint_strategy: A FingerprintStrategy instance, which can do per task, finer grained fingerprinting of a given Target. :param topological_order: Whether to invalidate in dependency order. If no exceptions are thrown by work in the block, the build cache is updated for the targets. Note: the artifact cache is not updated. That must be done manually. :returns: Yields an InvalidationCheck object reflecting the targets. :rtype: InvalidationCheck """ cache_key_generator = CacheKeyGenerator( self.context.options.for_global_scope().cache_key_gen_version, self.fingerprint) cache_manager = InvalidationCacheManager( self.workdir, cache_key_generator, self._build_invalidator_dir, invalidate_dependents, fingerprint_strategy=fingerprint_strategy, invalidation_report=self.context.invalidation_report, task_name=type(self).__name__, task_version=self.implementation_version_str(), artifact_write_callback=self.maybe_write_artifact) invalidation_check = cache_manager.check( targets, topological_order=topological_order) self._maybe_create_results_dirs(invalidation_check.all_vts) if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled( ): with self.context.new_workunit('cache'): cached_vts, uncached_vts, uncached_causes = \ self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check)) if cached_vts: cached_targets = [vt.target for vt in cached_vts] self.context.run_tracker.artifact_cache_stats.add_hits( cache_manager.task_name, cached_targets) if not silent: self._report_targets('Using cached artifacts for ', cached_targets, '.') if uncached_vts: uncached_targets = [vt.target for vt in uncached_vts] self.context.run_tracker.artifact_cache_stats.add_misses( cache_manager.task_name, uncached_targets, uncached_causes) if not silent: self._report_targets('No cached artifacts for ', uncached_targets, '.') # Now that we've checked the cache, re-partition whatever is still invalid. invalidation_check = \ InvalidationCheck(invalidation_check.all_vts, uncached_vts) if not silent: targets = [] for vt in invalidation_check.invalid_vts: targets.extend(vt.targets) if len(targets): msg_elements = [ 'Invalidated ', items_to_report_element( [t.address.reference() for t in targets], 'target'), '.' ] self.context.log.info(*msg_elements) invalidation_report = self.context.invalidation_report if invalidation_report: for vts in invalidation_check.all_vts: invalidation_report.add_vts(cache_manager, vts.targets, vts.cache_key, vts.valid, phase='pre-check') # Cache has been checked to create the full list of invalid VTs. # Only copy previous_results for this subset of VTs. if self.incremental: for vts in invalidation_check.invalid_vts: vts.copy_previous_results() # Yield the result, and then mark the targets as up to date. yield invalidation_check if invalidation_report: for vts in invalidation_check.all_vts: invalidation_report.add_vts(cache_manager, vts.targets, vts.cache_key, vts.valid, phase='post-check') for vt in invalidation_check.invalid_vts: vt.update() # Background work to clean up previous builds. if self.context.options.for_global_scope( ).workdir_max_build_entries is not None: self._launch_background_workdir_cleanup(invalidation_check.all_vts)
def test_env(content=TEST_CONTENT): with temporary_dir() as d: with tempfile.NamedTemporaryFile() as f: f.write(content) f.flush() yield f, CacheKeyGenerator(), BuildInvalidator(d)
def invalidated(self, targets, invalidate_dependents=False, silent=False, fingerprint_strategy=None, topological_order=False): """Checks targets for invalidation, first checking the artifact cache. Subclasses call this to figure out what to work on. :API: public :param targets: The targets to check for changes. :param invalidate_dependents: If True then any targets depending on changed targets are invalidated. :param silent: If true, suppress logging information about target invalidation. :param fingerprint_strategy: A FingerprintStrategy instance, which can do per task, finer grained fingerprinting of a given Target. :param topological_order: Whether to invalidate in dependency order. If no exceptions are thrown by work in the block, the build cache is updated for the targets. Note: the artifact cache is not updated. That must be done manually. :returns: Yields an InvalidationCheck object reflecting the targets. :rtype: InvalidationCheck """ cache_key_generator = CacheKeyGenerator( self.context.options.for_global_scope().cache_key_gen_version, self.fingerprint) cache_manager = InvalidationCacheManager( self.workdir, cache_key_generator, self._build_invalidator(), invalidate_dependents, fingerprint_strategy=fingerprint_strategy, invalidation_report=self.context.invalidation_report, task_name=type(self).__name__, task_version=self.implementation_version_str(), artifact_write_callback=self.maybe_write_artifact) # If this Task's execution has been forced, invalidate all our target fingerprints. if self._cache_factory.ignore and not self._force_invalidated: self.invalidate() self._force_invalidated = True invalidation_check = cache_manager.check( targets, topological_order=topological_order) self._maybe_create_results_dirs(invalidation_check.all_vts) if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled( ): with self.context.new_workunit('cache'): cached_vts, uncached_vts, uncached_causes = \ self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check)) if cached_vts: cached_targets = [vt.target for vt in cached_vts] self.context.run_tracker.artifact_cache_stats.add_hits( cache_manager.task_name, cached_targets) if not silent: self._report_targets('Using cached artifacts for ', cached_targets, '.') if uncached_vts: uncached_targets = [vt.target for vt in uncached_vts] self.context.run_tracker.artifact_cache_stats.add_misses( cache_manager.task_name, uncached_targets, uncached_causes) if not silent: self._report_targets('No cached artifacts for ', uncached_targets, '.') # Now that we've checked the cache, re-partition whatever is still invalid. invalidation_check = \ InvalidationCheck(invalidation_check.all_vts, uncached_vts) if not silent: targets = [] for vt in invalidation_check.invalid_vts: targets.extend(vt.targets) if len(targets): msg_elements = [ 'Invalidated ', items_to_report_element( [t.address.reference() for t in targets], 'target'), '.' ] self.context.log.info(*msg_elements) invalidation_report = self.context.invalidation_report if invalidation_report: for vts in invalidation_check.all_vts: invalidation_report.add_vts(cache_manager, vts.targets, vts.cache_key, vts.valid, phase='pre-check') # Cache has been checked to create the full list of invalid VTs. # Only copy previous_results for this subset of VTs. if self.incremental: for vts in invalidation_check.invalid_vts: vts.copy_previous_results() # This may seem odd: why would we need to invalidate a VersionedTargetSet that is already # invalid? But the name force_invalidate() is slightly misleading in this context - what it # actually does is delete the key file created at the end of the last successful task run. # This is necessary to avoid the following scenario: # # 1) In state A: Task suceeds and writes some output. Key is recorded by the invalidator. # 2) In state B: Task fails, but writes some output. Key is not recorded. # 3) After reverting back to state A: The current key is the same as the one recorded at the # end of step 1), so it looks like no work needs to be done, but actually the task # must re-run, to overwrite the output written in step 2. # # Deleting the file ensures that if a task fails, there is no key for which we might think # we're in a valid state. for vts in invalidation_check.invalid_vts: vts.force_invalidate() # Yield the result, and then mark the targets as up to date. yield invalidation_check if invalidation_report: for vts in invalidation_check.all_vts: invalidation_report.add_vts(cache_manager, vts.targets, vts.cache_key, vts.valid, phase='post-check') for vt in invalidation_check.invalid_vts: vt.update() # Background work to clean up previous builds. if self.context.options.for_global_scope( ).workdir_max_build_entries is not None: self._launch_background_workdir_cleanup(invalidation_check.all_vts)