class CacheManager(object):
"""Manages cache checks, updates and invalidation keeping track of basic change
and invalidation statistics.
Note that this is distinct from the ArtifactCache concept, and should probably be renamed.
"""
def __init__(self, cache_key_generator, build_invalidator_dir,
invalidate_dependents, extra_data, only_externaldeps):
self._cache_key_generator = cache_key_generator
self._invalidate_dependents = invalidate_dependents
self._extra_data = pickle.dumps(extra_data) # extra_data may be None.
self._sources = NO_SOURCES if only_externaldeps else TARGET_SOURCES
self._invalidator = BuildInvalidator(build_invalidator_dir)
def update(self, vts):
"""Mark a changed or invalidated VersionedTargetSet as successfully processed."""
for vt in vts.versioned_targets:
self._invalidator.update(vt.cache_key)
vt.valid = True
self._invalidator.update(vts.cache_key)
vts.valid = True
def force_invalidate(self, vts):
"""Force invalidation of a VersionedTargetSet."""
for vt in vts.versioned_targets:
self._invalidator.force_invalidate(vt.cache_key)
vt.valid = False
self._invalidator.force_invalidate(vts.cache_key)
vts.valid = False
def check(self, targets, partition_size_hint=None):
"""Checks whether each of the targets has changed and invalidates it if so.
Returns a list of VersionedTargetSet objects (either valid or invalid). The returned sets 'cover'
the input targets, possibly partitioning them, and are in topological order.
The caller can inspect these in order and, e.g., rebuild the invalid ones.
"""
all_vts = self._sort_and_validate_targets(targets)
invalid_vts = filter(lambda vt: not vt.valid, all_vts)
return InvalidationCheck(all_vts, invalid_vts, partition_size_hint)
def _sort_and_validate_targets(self, targets):
"""Validate each target.
Returns a topologically ordered set of VersionedTargets, each representing one input target.
"""
# We must check the targets in this order, to ensure correctness if invalidate_dependents=True, since
# we use earlier cache keys to compute later cache keys in this case.
ordered_targets = self._order_target_list(targets)
# This will be a list of VersionedTargets that correspond to @targets.
versioned_targets = []
# This will be a mapping from each target to its corresponding VersionedTarget.
versioned_targets_by_target = {}
# Map from id to current fingerprint of the target with that id. We update this as we iterate, in
# topological order, so when handling a target, this will already contain all its deps (in this round).
id_to_hash = {}
for target in ordered_targets:
dependency_keys = set()
if self._invalidate_dependents and hasattr(target, 'dependencies'):
# Note that we only need to do this for the immediate deps, because those will already
# reflect changes in their own deps.
for dep in target.dependencies:
# We rely on the fact that any deps have already been processed, either in an earlier round or
# because they came first in ordered_targets.
if isinstance(dep, Target):
hash = id_to_hash.get(dep.id, None)
if hash is None:
# It may have been processed in a prior round, and therefore the hash should
# have been written out by the invalidator.
hash = self._invalidator.existing_hash(dep.id)
# Note that hash may be None here, indicating that the dependency will not be processed
# until a later phase. For example, if a codegen target depends on a library target (because
# the generated code needs that library).
if hash is not None:
dependency_keys.add(hash)
elif isinstance(dep, JarDependency):
jarid = ''
for key in CacheManager._JAR_HASH_KEYS:
jarid += str(getattr(dep, key))
dependency_keys.add(jarid)
else:
dependency_keys.add(str(dep))
# TODO(John Sirois): Hashing on external targets should have a formal api - we happen to
# know jars are special and python requirements __str__ works for this purpose.
cache_key = self._key_for(target, dependency_keys)
id_to_hash[target.id] = cache_key.hash
# Create a VersionedTarget corresponding to @target.
versioned_target = VersionedTarget(self, target, cache_key)
# Add the new VersionedTarget to the list of computed VersionedTargets.
versioned_targets.append(versioned_target)
# Add to the mapping from Targets to VersionedTargets, for use in hooking up VersionedTarget dependencies below.
versioned_targets_by_target[target] = versioned_target
# Having created all applicable VersionedTargets, now we build the VersionedTarget dependency graph, looking
# through targets that don't correspond to VersionedTargets themselves.
versioned_target_deps_by_target = {}
def get_versioned_target_deps_for_target(target):
# For every dependency of @target, we will store its corresponding VersionedTarget here. For dependencies that
# don't correspond to a VersionedTarget (e.g. pass-through dependency wrappers), we will resolve their actual
# dependencies and find VersionedTargets for them.
versioned_target_deps = set([])
if hasattr(target, 'dependencies'):
for dep in target.dependencies:
for dependency in dep.resolve():
if dependency in versioned_targets_by_target:
# If there exists a VersionedTarget corresponding to this Target, store it and continue.
versioned_target_deps.add(
versioned_targets_by_target[dependency])
elif dependency in versioned_target_deps_by_target:
# Otherwise, see if we've already resolved this dependency to the VersionedTargets it depends on, and use
# those.
versioned_target_deps.update(
versioned_target_deps_by_target[dependency])
else:
# Otherwise, compute the VersionedTargets that correspond to this dependency's dependencies, cache and
# use the computed result.
versioned_target_deps_by_target[
dependency] = get_versioned_target_deps_for_target(
dependency)
versioned_target_deps.update(
versioned_target_deps_by_target[dependency])
# Return the VersionedTarget dependencies that this target's VersionedTarget should depend on.
return versioned_target_deps
# Initialize all VersionedTargets to point to the VersionedTargets they depend on.
for versioned_target in versioned_targets:
versioned_target.dependencies = get_versioned_target_deps_for_target(
versioned_target.target)
return versioned_targets
def needs_update(self, cache_key):
return self._invalidator.needs_update(cache_key)
def _order_target_list(self, targets):
"""Orders the targets topologically, from least to most dependent."""
target_ids = set([x.id for x in targets])
# Most to least dependent.
reverse_ordered_targets_and_deps = InternalTarget.sort_targets(targets)
# Least to most dependent. We must build in this order.
ordered_targets_and_deps = reversed(reverse_ordered_targets_and_deps)
# Return just the ones that were originally in targets.
return filter(lambda x: x.id in target_ids, ordered_targets_and_deps)
def _key_for(self, target, dependency_keys):
def fingerprint_extra(sha):
sha.update(self._extra_data)
for key in sorted(
dependency_keys
): # Sort to ensure hashing in a consistent order.
sha.update(key)
return self._cache_key_generator.key_for_target(
target, sources=self._sources, fingerprint_extra=fingerprint_extra)
_JAR_HASH_KEYS = ('org', 'name', 'rev', 'force', 'excludes', 'transitive',
'_configurations', 'artifacts')
class CacheManager(object):
"""Manages cache checks, updates and invalidation keeping track of basic change
and invalidation statistics.
Note that this is distinct from the ArtifactCache concept, and should probably be renamed.
"""
def __init__(self, cache_key_generator, build_invalidator_dir,
invalidate_dependents, extra_data, only_externaldeps):
self._cache_key_generator = cache_key_generator
self._invalidate_dependents = invalidate_dependents
self._extra_data = pickle.dumps(extra_data) # extra_data may be None.
self._sources = NO_SOURCES if only_externaldeps else TARGET_SOURCES
self._invalidator = BuildInvalidator(build_invalidator_dir)
def update(self, vts):
"""Mark a changed or invalidated VersionedTargetSet as successfully processed."""
for vt in vts.versioned_targets:
self._invalidator.update(vt.cache_key)
vt.valid = True
self._invalidator.update(vts.cache_key)
vts.valid = True
def force_invalidate(self, vts):
"""Force invalidation of a VersionedTargetSet."""
for vt in vts.versioned_targets:
self._invalidator.force_invalidate(vt.cache_key)
vt.valid = False
self._invalidator.force_invalidate(vts.cache_key)
vts.valid = False
def check(self, targets, partition_size_hint=None):
"""Checks whether each of the targets has changed and invalidates it if so.
Returns a list of VersionedTargetSet objects (either valid or invalid). The returned sets
'cover' the input targets, possibly partitioning them, and are in topological order.
The caller can inspect these in order and, e.g., rebuild the invalid ones.
"""
all_vts = self._sort_and_validate_targets(targets)
invalid_vts = filter(lambda vt: not vt.valid, all_vts)
return InvalidationCheck(all_vts, invalid_vts, partition_size_hint)
def _sort_and_validate_targets(self, targets):
"""Validate each target.
Returns a topologically ordered set of VersionedTargets, each representing one input target.
"""
# We must check the targets in this order, to ensure correctness if invalidate_dependents=True,
# since we use earlier cache keys to compute later cache keys in this case.
ordered_targets = self._order_target_list(targets)
# This will be a list of VersionedTargets that correspond to @targets.
versioned_targets = []
# This will be a mapping from each target to its corresponding VersionedTarget.
versioned_targets_by_target = {}
# Map from id to current fingerprint of the target with that id. We update this as we iterate,
# in topological order, so when handling a target, this will already contain all its deps (in
# this round).
id_to_hash = {}
for target in ordered_targets:
dependency_keys = set()
if self._invalidate_dependents and hasattr(target, 'dependencies'):
# Note that we only need to do this for the immediate deps, because those will already
# reflect changes in their own deps.
for dep in target.dependencies:
# We rely on the fact that any deps have already been processed, either in an earlier
# round or because they came first in ordered_targets.
# Note that only external deps (e.g., JarDependency) or targets with sources can
# affect invalidation. Other targets (JarLibrary, Pants) are just dependency scaffolding.
if isinstance(dep, ExternalDependency):
dependency_keys.add(dep.cache_key())
elif isinstance(dep, TargetWithSources):
fprint = id_to_hash.get(dep.id, None)
if fprint is None:
# It may have been processed in a prior round, and therefore the fprint should
# have been written out by the invalidator.
fprint = self._invalidator.existing_hash(dep.id)
# Note that fprint may still be None here. E.g., a codegen target is in the list
# of deps, but its fprint is not visible to our self._invalidator (that of the
# target synthesized from it is visible, so invalidation will still be correct.)
#
# Another case where this can happen is a dep of a codegen target on, say,
# a java target that hasn't been built yet (again, the synthesized target will
# depend on that same java target, so invalidation will still be correct.)
# TODO(benjy): Make this simpler and more obviously correct.
if fprint is not None:
dependency_keys.add(fprint)
elif isinstance(dep, JarLibrary) or isinstance(dep, Pants):
pass
else:
raise ValueError('Cannot calculate a cache_key for a dependency: %s' % dep)
cache_key = self._key_for(target, dependency_keys)
id_to_hash[target.id] = cache_key.hash
# Create a VersionedTarget corresponding to @target.
versioned_target = VersionedTarget(self, target, cache_key)
# Add the new VersionedTarget to the list of computed VersionedTargets.
versioned_targets.append(versioned_target)
# Add to the mapping from Targets to VersionedTargets, for use in hooking up VersionedTarget
# dependencies below.
versioned_targets_by_target[target] = versioned_target
# Having created all applicable VersionedTargets, now we build the VersionedTarget dependency
# graph, looking through targets that don't correspond to VersionedTargets themselves.
versioned_target_deps_by_target = {}
def get_versioned_target_deps_for_target(target):
# For every dependency of @target, we will store its corresponding VersionedTarget here. For
# dependencies that don't correspond to a VersionedTarget (e.g. pass-through dependency
# wrappers), we will resolve their actual dependencies and find VersionedTargets for them.
versioned_target_deps = set([])
if hasattr(target, 'dependencies'):
for dep in target.dependencies:
for dependency in dep.resolve():
if dependency in versioned_targets_by_target:
# If there exists a VersionedTarget corresponding to this Target, store it and
# continue.
versioned_target_deps.add(versioned_targets_by_target[dependency])
elif dependency in versioned_target_deps_by_target:
# Otherwise, see if we've already resolved this dependency to the VersionedTargets it
# depends on, and use those.
versioned_target_deps.update(versioned_target_deps_by_target[dependency])
else:
# Otherwise, compute the VersionedTargets that correspond to this dependency's
# dependencies, cache and use the computed result.
versioned_target_deps_by_target[dependency] = get_versioned_target_deps_for_target(
dependency)
versioned_target_deps.update(versioned_target_deps_by_target[dependency])
# Return the VersionedTarget dependencies that this target's VersionedTarget should depend on.
return versioned_target_deps
# Initialize all VersionedTargets to point to the VersionedTargets they depend on.
for versioned_target in versioned_targets:
versioned_target.dependencies = get_versioned_target_deps_for_target(versioned_target.target)
return versioned_targets
def needs_update(self, cache_key):
return self._invalidator.needs_update(cache_key)
def _order_target_list(self, targets):
"""Orders the targets topologically, from least to most dependent."""
targets = set(t for t in targets if isinstance(t, Target))
return filter(targets.__contains__, reversed(InternalTarget.sort_targets(targets)))
def _key_for(self, target, dependency_keys):
def fingerprint_extra(sha):
sha.update(self._extra_data)
for key in sorted(dependency_keys): # Sort to ensure hashing in a consistent order.
sha.update(key)
return self._cache_key_generator.key_for_target(
target,
sources=self._sources,
fingerprint_extra=fingerprint_extra
)
class CacheManager(object):
"""Manages cache checks, updates and invalidation keeping track of basic change
and invalidation statistics.
Note that this is distinct from the ArtifactCache concept, and should probably be renamed.
"""
def __init__(
self, cache_key_generator, build_invalidator_dir, invalidate_dependents, extra_data, only_externaldeps
):
self._cache_key_generator = cache_key_generator
self._invalidate_dependents = invalidate_dependents
self._extra_data = pickle.dumps(extra_data) # extra_data may be None.
self._sources = NO_SOURCES if only_externaldeps else TARGET_SOURCES
self._invalidator = BuildInvalidator(build_invalidator_dir)
def update(self, vts):
"""Mark a changed or invalidated VersionedTargetSet as successfully processed."""
for vt in vts.versioned_targets:
self._invalidator.update(vt.cache_key)
vt.valid = True
self._invalidator.update(vts.cache_key)
vts.valid = True
def force_invalidate(self, vts):
"""Force invalidation of a VersionedTargetSet."""
for vt in vts.versioned_targets:
self._invalidator.force_invalidate(vt.cache_key)
vt.valid = False
self._invalidator.force_invalidate(vts.cache_key)
vts.valid = False
def check(self, targets, partition_size_hint=None):
"""Checks whether each of the targets has changed and invalidates it if so.
Returns a list of VersionedTargetSet objects (either valid or invalid). The returned sets 'cover'
the input targets, possibly partitioning them, and are in topological order.
The caller can inspect these in order and, e.g., rebuild the invalid ones.
"""
all_vts = self._sort_and_validate_targets(targets)
invalid_vts = filter(lambda vt: not vt.valid, all_vts)
return InvalidationCheck(all_vts, invalid_vts, partition_size_hint)
def _sort_and_validate_targets(self, targets):
"""Validate each target.
Returns a topologically ordered set of VersionedTargets, each representing one input target.
"""
# We must check the targets in this order, to ensure correctness if invalidate_dependents=True, since
# we use earlier cache keys to compute later cache keys in this case.
ordered_targets = self._order_target_list(targets)
# This will be a list of VersionedTargets that correspond to @targets.
versioned_targets = []
# This will be a mapping from each target to its corresponding VersionedTarget.
versioned_targets_by_target = {}
# Map from id to current fingerprint of the target with that id. We update this as we iterate, in
# topological order, so when handling a target, this will already contain all its deps (in this round).
id_to_hash = {}
for target in ordered_targets:
dependency_keys = set()
if self._invalidate_dependents and hasattr(target, "dependencies"):
# Note that we only need to do this for the immediate deps, because those will already
# reflect changes in their own deps.
for dep in target.dependencies:
# We rely on the fact that any deps have already been processed, either in an earlier round or
# because they came first in ordered_targets.
if isinstance(dep, Target):
hash = id_to_hash.get(dep.id, None)
if hash is None:
# It may have been processed in a prior round, and therefore the hash should
# have been written out by the invalidator.
hash = self._invalidator.existing_hash(dep.id)
# Note that hash may be None here, indicating that the dependency will not be processed
# until a later phase. For example, if a codegen target depends on a library target (because
# the generated code needs that library).
if hash is not None:
dependency_keys.add(hash)
elif isinstance(dep, JarDependency):
jarid = ""
for key in CacheManager._JAR_HASH_KEYS:
jarid += str(getattr(dep, key))
dependency_keys.add(jarid)
else:
dependency_keys.add(str(dep))
# TODO(John Sirois): Hashing on external targets should have a formal api - we happen to
# know jars are special and python requirements __str__ works for this purpose.
cache_key = self._key_for(target, dependency_keys)
id_to_hash[target.id] = cache_key.hash
# Create a VersionedTarget corresponding to @target.
versioned_target = VersionedTarget(self, target, cache_key)
# Add the new VersionedTarget to the list of computed VersionedTargets.
versioned_targets.append(versioned_target)
# Add to the mapping from Targets to VersionedTargets, for use in hooking up VersionedTarget dependencies below.
versioned_targets_by_target[target] = versioned_target
# Having created all applicable VersionedTargets, now we build the VersionedTarget dependency graph, looking
# through targets that don't correspond to VersionedTargets themselves.
versioned_target_deps_by_target = {}
def get_versioned_target_deps_for_target(target):
# For every dependency of @target, we will store its corresponding VersionedTarget here. For dependencies that
# don't correspond to a VersionedTarget (e.g. pass-through dependency wrappers), we will resolve their actual
# dependencies and find VersionedTargets for them.
versioned_target_deps = set([])
if hasattr(target, "dependencies"):
for dep in target.dependencies:
for dependency in dep.resolve():
if dependency in versioned_targets_by_target:
# If there exists a VersionedTarget corresponding to this Target, store it and continue.
versioned_target_deps.add(versioned_targets_by_target[dependency])
elif dependency in versioned_target_deps_by_target:
# Otherwise, see if we've already resolved this dependency to the VersionedTargets it depends on, and use
# those.
versioned_target_deps.update(versioned_target_deps_by_target[dependency])
else:
# Otherwise, compute the VersionedTargets that correspond to this dependency's dependencies, cache and
# use the computed result.
versioned_target_deps_by_target[dependency] = get_versioned_target_deps_for_target(
dependency
)
versioned_target_deps.update(versioned_target_deps_by_target[dependency])
# Return the VersionedTarget dependencies that this target's VersionedTarget should depend on.
return versioned_target_deps
# Initialize all VersionedTargets to point to the VersionedTargets they depend on.
for versioned_target in versioned_targets:
versioned_target.dependencies = get_versioned_target_deps_for_target(versioned_target.target)
return versioned_targets
def needs_update(self, cache_key):
return self._invalidator.needs_update(cache_key)
def _order_target_list(self, targets):
"""Orders the targets topologically, from least to most dependent."""
target_ids = set([x.id for x in targets])
# Most to least dependent.
reverse_ordered_targets_and_deps = InternalTarget.sort_targets(targets)
# Least to most dependent. We must build in this order.
ordered_targets_and_deps = reversed(reverse_ordered_targets_and_deps)
# Return just the ones that were originally in targets.
return filter(lambda x: x.id in target_ids, ordered_targets_and_deps)
def _key_for(self, target, dependency_keys):
def fingerprint_extra(sha):
sha.update(self._extra_data)
for key in sorted(dependency_keys): # Sort to ensure hashing in a consistent order.
sha.update(key)
return self._cache_key_generator.key_for_target(
target, sources=self._sources, fingerprint_extra=fingerprint_extra
)
_JAR_HASH_KEYS = ("org", "name", "rev", "force", "excludes", "transitive", "_configurations", "artifacts")
