def invalidated(self,
targets,
invalidate_dependents=False,
partition_size_hint=sys.maxint,
silent=False,
locally_changed_targets=None,
fingerprint_strategy=None,
topological_order=False):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
:param targets: The targets to check for changes.
:param invalidate_dependents: If True then any targets depending on changed targets are invalidated.
:param partition_size_hint: Each VersionedTargetSet in the yielded list will represent targets
containing roughly this number of source files, if possible. Set to
sys.maxint for a single VersionedTargetSet. Set to 0 for one
VersionedTargetSet per target. It is up to the caller to do the right
thing with whatever partitioning it asks for.
:param locally_changed_targets: Targets that we've edited locally. If specified, and there aren't too
many of them, we keep these in separate partitions from other targets,
as these are more likely to have build errors, and so to be rebuilt over
and over, and partitioning them separately is a performance win.
:param fingerprint_strategy: A FingerprintStrategy instance, which can do per task, finer grained
fingerprinting of a given Target.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
:returns: Yields an InvalidationCheck object reflecting the (partitioned) targets.
:rtype: InvalidationCheck
"""
# TODO(benjy): Compute locally_changed_targets here instead of passing it in? We currently pass
# it in because JvmCompile already has the source->target mapping for other reasons, and also
# to selectively enable this feature.
fingerprint_strategy = fingerprint_strategy or TaskIdentityFingerprintStrategy(
self)
cache_manager = self.create_cache_manager(
invalidate_dependents, fingerprint_strategy=fingerprint_strategy)
# We separate locally-modified targets from others by coloring them differently.
# This can be a performance win, because these targets are more likely to be iterated
# over, and this preserves "chunk stability" for them.
colors = {}
# But we only do so if there aren't too many, or this optimization will backfire.
locally_changed_target_limit = 10
if locally_changed_targets and len(
locally_changed_targets) < locally_changed_target_limit:
for t in targets:
if t in locally_changed_targets:
colors[t] = 'locally_changed'
else:
colors[t] = 'not_locally_changed'
invalidation_check = cache_manager.check(
targets,
partition_size_hint,
colors,
topological_order=topological_order)
if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled(
):
with self.context.new_workunit('cache'):
cached_vts, uncached_vts = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
for t in cached_targets:
self.context.run_tracker.artifact_cache_stats.add_hit(
'default', t)
if not silent:
self._report_targets('Using cached artifacts for ',
cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
for t in uncached_targets:
self.context.run_tracker.artifact_cache_stats.add_miss(
'default', t)
if not silent:
self._report_targets('No cached artifacts for ',
uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts, partition_size_hint, colors)
self._maybe_create_results_dirs(invalidation_check.all_vts)
if not silent:
targets = []
num_invalid_partitions = len(
invalidation_check.invalid_vts_partitioned)
for vt in invalidation_check.invalid_vts_partitioned:
targets.extend(vt.targets)
if len(targets):
msg_elements = [
'Invalidated ',
items_to_report_element(
[t.address.reference() for t in targets], 'target')
]
if num_invalid_partitions > 1:
msg_elements.append(' in {} target partitions'.format(
num_invalid_partitions))
msg_elements.append('.')
self.context.log.info(*msg_elements)
invalidation_report = self.context.invalidation_report
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='pre-check')
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='post-check')
for vt in invalidation_check.invalid_vts:
vt.update() # In case the caller doesn't update.
write_to_cache = (self.cache_target_dirs
and self.artifact_cache_writes_enabled()
and invalidation_check.invalid_vts)
if write_to_cache:
pairs = []
for vt in invalidation_check.invalid_vts:
if self._should_cache(vt):
pairs.append((vt, [vt.results_dir]))
self.update_artifact_cache(pairs)
def invalidated(self,
targets,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=None,
topological_order=False,
use_cache=True):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
:API: public
:param targets: The targets to check for changes.
:param invalidate_dependents: If True then any targets depending on changed targets are invalidated.
:param fingerprint_strategy: A FingerprintStrategy instance, which can do per task, finer grained
fingerprinting of a given Target.
:param use_cache: A boolean to indicate whether to read/write the cache within this
invalidate call. In order for the cache to be used, both the task
settings and this parameter must agree that they should be used.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
:returns: Yields an InvalidationCheck object reflecting the targets.
:rtype: InvalidationCheck
"""
fingerprint_strategy = fingerprint_strategy or TaskIdentityFingerprintStrategy(
self)
cache_manager = self.create_cache_manager(
invalidate_dependents, fingerprint_strategy=fingerprint_strategy)
invalidation_check = cache_manager.check(
targets, topological_order=topological_order)
if invalidation_check.invalid_vts and use_cache and self.artifact_cache_reads_enabled(
):
with self.context.new_workunit('cache'):
cached_vts, uncached_vts, uncached_causes = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
self.context.run_tracker.artifact_cache_stats.add_hits(
cache_manager.task_name, cached_targets)
if not silent:
self._report_targets('Using cached artifacts for ',
cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
self.context.run_tracker.artifact_cache_stats.add_misses(
cache_manager.task_name, uncached_targets, uncached_causes)
if not silent:
self._report_targets('No cached artifacts for ',
uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts)
self._maybe_create_results_dirs(invalidation_check.all_vts)
if not silent:
targets = []
for vt in invalidation_check.invalid_vts:
targets.extend(vt.targets)
if len(targets):
msg_elements = [
'Invalidated ',
items_to_report_element(
[t.address.reference() for t in targets], 'target')
]
msg_elements.append('.')
self.context.log.info(*msg_elements)
invalidation_report = self.context.invalidation_report
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='pre-check')
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='post-check')
for vt in invalidation_check.invalid_vts:
vt.update() # In case the caller doesn't update.
# Background work to clean up previous builds.
if self.context.options.for_global_scope(
).workdir_max_build_entries is not None:
self._launch_background_workdir_cleanup(invalidation_check.all_vts)
write_to_cache = (self.cache_target_dirs and use_cache
and self.artifact_cache_writes_enabled()
and invalidation_check.invalid_vts)
if write_to_cache:
pairs = []
for vt in invalidation_check.invalid_vts:
if self._should_cache(vt):
pairs.append((vt, [vt.current_results_dir]))
self.update_artifact_cache(pairs)
def execute(self):
# Pants does no longer allows options to be tuples or sets. So we use lists of dicts and then convert into
# hashable structures here.
# Pins converted to { (org, name): rev, ... }
if self.get_options().level == 'debug':
logging.getLogger('fsqio.pants.pom').setLevel(logging.DEBUG)
global_pinned_tuples = {}
for pin in self.get_options().global_pinned_versions:
artifact_tuple = (pin['org'], pin['name'])
if artifact_tuple in global_pinned_tuples:
raise Exception(
'An artifact has conflicting overrides!:\n{}:{} and\n'
'{}'.format(artifact_tuple, pin['rev'],
global_pinned_tuples[artifact_tuple]))
global_pinned_tuples[artifact_tuple] = pin['rev']
# Overrrides converted to { (org, name, rev): /path/to/artifact, ... }
override_tuples = {}
for override in self.get_options().local_override_versions:
override_tuples[(override['org'],
override['name'])] = override['artifact_path']
# Exclusions converted to [(org, name), ...]
global_exclusion_tuples = []
for exclusion in self.get_options().global_exclusions:
global_exclusion_tuples.append(
(exclusion['org'], exclusion['name']))
global_exclusions = frozenset(global_exclusion_tuples)
global_pinned_versions = dict(global_pinned_tuples)
local_override_versions = override_tuples
fetchers = ChainedFetcher(self.get_options().maven_repos)
invalidation_context_manager = self.invalidated(
self.all_jar_libs,
invalidate_dependents=False,
fingerprint_strategy=TaskIdentityFingerprintStrategy(self),
)
with invalidation_context_manager as invalidation_check:
# NOTE: In terms of caching this models IvyResolve in pants quite closely. We always
# operate over and cache in terms of the global set of jar dependencies. Note that we override
# `check_artifact_cache_for` in order to get the artifact cache to respect this.
global_vts = VersionedTargetSet.from_versioned_targets(
invalidation_check.all_vts)
vts_workdir = os.path.join(self.workdir, global_vts.cache_key.hash)
analysis_path = os.path.join(vts_workdir, 'analysis.pickle')
if invalidation_check.invalid_vts or not os.path.exists(
analysis_path):
with self.context.new_workunit('traverse-pom-graph'):
global_dep_graph, target_to_dep_graph = self.resolve_dependency_graphs(
self.all_jar_libs,
fetchers,
global_exclusions,
global_pinned_versions,
)
self.report_unused_pins_and_exclusions(
global_dep_graph,
global_pinned_versions,
global_exclusions,
)
# TODO: Not super happy about using target.id really anywhere, since it's just a name.
# But for now this is all completely invalidated whenever any part of 3rdparty:: changes.
# It might however be possible that just renaming a JarLib (and doing nothing else) will
# break this.
for target, dep_graph in target_to_dep_graph.items():
self.target_to_maven_coordinate_closure[target.id] = list(
dep_graph.artifact_closure())
copied_coord_to_artifacts = deepcopy(
global_dep_graph._coord_to_provided_artifacts)
self.maven_coordinate_to_provided_artifacts.update(
copied_coord_to_artifacts)
safe_mkdir(vts_workdir)
analysis = {
'target_to_maven_coordinate_closure':
self.target_to_maven_coordinate_closure,
'maven_coordinate_to_provided_artifacts':
self.maven_coordinate_to_provided_artifacts,
'global_dep_graph': global_dep_graph,
}
with open(analysis_path, 'wb') as f:
pickle.dump(analysis, f)
if self.artifact_cache_writes_enabled():
self.update_artifact_cache([(global_vts, [analysis_path])])
else:
with open(analysis_path, 'rb') as f:
analysis = pickle.load(f)
self.target_to_maven_coordinate_closure.update(
analysis['target_to_maven_coordinate_closure'], )
self.maven_coordinate_to_provided_artifacts.update(
analysis['maven_coordinate_to_provided_artifacts'], )
global_dep_graph = analysis['global_dep_graph']
self.report_for_artifacts(global_dep_graph)
conflicted_deps = global_dep_graph.conflicted_dependencies()
if conflicted_deps:
self.report_conflicted_deps(
conflicted_deps,
global_dep_graph.reverse_unversioned_dep_graph(),
global_dep_graph,
)
raise Exception(
'PomResolve found {} conflicting dependencies. These must be explicitly'
' pinned or excluded in order to generate a consistent global classpath.'
' See the output above for details, and try `./pants pom-resolve --help`'
' for information on flags to get more detailed reporting.'.
format(len(conflicted_deps)))
all_artifacts = set()
for coord_closure in self.target_to_maven_coordinate_closure.values():
for coord in coord_closure:
for artifact in self.maven_coordinate_to_provided_artifacts[
coord]:
all_artifacts.add(artifact)
with self.context.new_workunit('fetch-artifacts'):
coord_to_artifact_symlinks = self._fetch_artifacts(
local_override_versions)
# TODO(mateo): Inline source jar fetching into the invalidation block. Modify `fetch_artifacts` to work with
# either jar type and delete the fetch_source_jars special casing. Very very low priority.
# My naive idea would be to not resolve a fetcher for a source jar, but to just attempt and download an accompanying
# source jar for every successful jar fetch, if the bit is set, skipping misses.
if self.get_options().fetch_source_jars:
with self.context.new_workunit('fetch-source-jars'):
symlink_dir = os.path.join(
self.pom_cache_dir,
'source-jars-symlink-farms',
global_vts.cache_key.hash,
)
if not os.path.exists(symlink_dir):
self._fetch_source_jars(fetchers, symlink_dir)
stderr('\nFetched source jars to {}'.format(symlink_dir))
classpath_dump_file = self.get_options().dump_classpath_file
if classpath_dump_file:
with open(classpath_dump_file, 'wb') as f:
f.write('FINGERPRINT: {}\n'.format(global_vts.cache_key.hash))
for artifact in sorted(all_artifacts):
f.write('{}\n'.format(artifact))
logger.info(
'Dumped classpath file to {}'.format(classpath_dump_file))
classpath_info_filename = self.get_options(
).write_classpath_info_to_file
if classpath_info_filename:
classpath_info = {
'fingerprint':
global_vts.cache_key.hash,
'classpath': [{
'path':
os.path.join(self.pom_cache_dir, artifact.artifact_path),
'groupId':
artifact.groupId,
'artifactId':
artifact.artifactId,
'version':
artifact.version,
'packaging':
artifact.packaging,
'classifier':
artifact.classifier,
} for artifact in all_artifacts],
}
with open(classpath_info_filename, 'w') as classpath_info_file:
classpath_info_file.write(json.dumps(classpath_info))
logger.info('Wrote classpath info JSON to {}.'.format(
classpath_info_filename))
with self.context.new_workunit('populate-compile-classpath'):
self._populate_compile_classpath()
def _fingerprint_strategy(self):
return TaskIdentityFingerprintStrategy(self)
def invalidated(self,
targets,
invalidate_dependents=False,
silent=False,
fingerprint_strategy=None,
topological_order=False):
"""Checks targets for invalidation, first checking the artifact cache.
Subclasses call this to figure out what to work on.
:API: public
:param targets: The targets to check for changes.
:param invalidate_dependents: If True then any targets depending on changed targets are invalidated.
:param fingerprint_strategy: A FingerprintStrategy instance, which can do per task, finer grained
fingerprinting of a given Target.
If no exceptions are thrown by work in the block, the build cache is updated for the targets.
Note: the artifact cache is not updated. That must be done manually.
:returns: Yields an InvalidationCheck object reflecting the targets.
:rtype: InvalidationCheck
"""
fingerprint_strategy = fingerprint_strategy or TaskIdentityFingerprintStrategy(
self)
cache_manager = InvalidationCacheManager(
self.workdir,
self._cache_key_generator,
self._build_invalidator_dir,
invalidate_dependents,
fingerprint_strategy=fingerprint_strategy,
invalidation_report=self.context.invalidation_report,
task_name=type(self).__name__,
task_version=self.implementation_version_str(),
artifact_write_callback=self.maybe_write_artifact)
invalidation_check = cache_manager.check(
targets, topological_order=topological_order)
self._maybe_create_results_dirs(invalidation_check.all_vts)
if invalidation_check.invalid_vts and self.artifact_cache_reads_enabled(
):
with self.context.new_workunit('cache'):
cached_vts, uncached_vts, uncached_causes = \
self.check_artifact_cache(self.check_artifact_cache_for(invalidation_check))
if cached_vts:
cached_targets = [vt.target for vt in cached_vts]
self.context.run_tracker.artifact_cache_stats.add_hits(
cache_manager.task_name, cached_targets)
if not silent:
self._report_targets('Using cached artifacts for ',
cached_targets, '.')
if uncached_vts:
uncached_targets = [vt.target for vt in uncached_vts]
self.context.run_tracker.artifact_cache_stats.add_misses(
cache_manager.task_name, uncached_targets, uncached_causes)
if not silent:
self._report_targets('No cached artifacts for ',
uncached_targets, '.')
# Now that we've checked the cache, re-partition whatever is still invalid.
invalidation_check = \
InvalidationCheck(invalidation_check.all_vts, uncached_vts)
if not silent:
targets = []
for vt in invalidation_check.invalid_vts:
targets.extend(vt.targets)
if len(targets):
msg_elements = [
'Invalidated ',
items_to_report_element(
[t.address.reference() for t in targets], 'target')
]
msg_elements.append('.')
self.context.log.info(*msg_elements)
invalidation_report = self.context.invalidation_report
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='pre-check')
# Cache has been checked to create the full list of invalid VTs. Only copy previous_results for this subset of VTs.
for vts in invalidation_check.invalid_vts:
if self.incremental:
vts.copy_previous_results(self.workdir)
# Yield the result, and then mark the targets as up to date.
yield invalidation_check
if invalidation_report:
for vts in invalidation_check.all_vts:
invalidation_report.add_vts(cache_manager,
vts.targets,
vts.cache_key,
vts.valid,
phase='post-check')
for vt in invalidation_check.invalid_vts:
vt.update()
# Background work to clean up previous builds.
if self.context.options.for_global_scope(
).workdir_max_build_entries is not None:
self._launch_background_workdir_cleanup(invalidation_check.all_vts)