def register_extra_products_from_contexts(self, targets, compile_contexts):
super().register_extra_products_from_contexts(targets, compile_contexts)
def confify(entries):
return [(conf, e) for e in entries for conf in self._confs]
# Ensure that the jar/rsc jar is on the rsc_mixed_compile_classpath.
for target in targets:
merged_cc = compile_contexts[target]
zinc_cc = merged_cc.zinc_cc
rsc_cc = merged_cc.rsc_cc
# Make sure m.jar is digested if it exists when the target is validated.
if rsc_cc.rsc_jar_file.directory_digest is None and os.path.exists(rsc_cc.rsc_jar_file.path):
relpath = fast_relpath(rsc_cc.rsc_jar_file.path, get_buildroot())
classes_dir_snapshot, = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([relpath]),
get_buildroot(),
Digest.load(relpath),
),
])
rsc_cc.rsc_jar_file.hydrate_missing_directory_digest(classes_dir_snapshot.directory_digest)
if rsc_cc.workflow is not None:
cp_entries = match(rsc_cc.workflow, {
self.JvmCompileWorkflowType.zinc_only: lambda: confify([self._classpath_for_context(zinc_cc)]),
self.JvmCompileWorkflowType.zinc_java: lambda: confify([self._classpath_for_context(zinc_cc)]),
self.JvmCompileWorkflowType.rsc_and_zinc: lambda: confify([rsc_cc.rsc_jar_file]),
self.JvmCompileWorkflowType.outline_and_zinc: lambda: confify([rsc_cc.rsc_jar_file]),
})()
self.context.products.get_data('rsc_mixed_compile_classpath').add_for_target(
target,
cp_entries)
def _snapshotted_classpath(self, results_dir):
relpath = fast_relpath(results_dir, get_buildroot())
(classes_dir_snapshot, ) = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(PathGlobs([relpath]), get_buildroot(),
Digest.load(relpath))
])
return ClasspathEntry(results_dir,
classes_dir_snapshot.directory_digest)
def _shaded_jar_as_classpath_entry(self, shaded_jar):
# Capture a Snapshot for the jar.
buildroot = get_buildroot()
snapshot = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([fast_relpath(shaded_jar, buildroot)]),
buildroot,
Digest.load(shaded_jar),
)
])[0]
snapshot.digest.dump(shaded_jar)
return ClasspathEntry(shaded_jar, directory_digest=snapshot.digest)
def _double_check_cache_for_vts(self, vts, zinc_compile_context):
# Double check the cache before beginning compilation
if self.check_cache(vts):
self.context.log.debug(f"Snapshotting results for {vts.target.address.spec}")
classpath_entry = self._classpath_for_context(zinc_compile_context)
relpath = fast_relpath(classpath_entry.path, get_buildroot())
(classes_dir_snapshot,) = self.context._scheduler.capture_snapshots(
[PathGlobsAndRoot(PathGlobs([relpath]), get_buildroot(), Digest.load(relpath),),]
)
classpath_entry.hydrate_missing_directory_digest(classes_dir_snapshot.directory_digest)
# Re-validate the vts!
vts.update()
def add_directory_digests_for_jars(self, targets_and_jars):
"""For each target, get DirectoryDigests for its jars and return them zipped with the jars.
:param targets_and_jars: List of tuples of the form (Target, [pants.java.jar.jar_dependency_utils.ResolveJar])
:return: list[tuple[(Target, list[pants.java.jar.jar_dependency_utils.ResolveJar])]
"""
targets_and_jars = list(targets_and_jars)
if not targets_and_jars:
return targets_and_jars
jar_paths = []
for target, jars_to_snapshot in targets_and_jars:
for jar in jars_to_snapshot:
jar_paths.append(fast_relpath(jar.pants_path, get_buildroot()))
# Capture Snapshots for jars, using an optional adjacent digest. Create the digest afterward
# if it does not exist.
snapshots = self.context._scheduler.capture_snapshots(
tuple(
PathGlobsAndRoot(
PathGlobs([jar]),
get_buildroot(),
Digest.load(jar),
) for jar in jar_paths))
for snapshot, jar_path in zip(snapshots, jar_paths):
snapshot.digest.dump(jar_path)
# We want to map back the list[Snapshot] to targets_and_jars
# We assume that (1) jars_to_snapshot has the same number of ResolveJars as snapshots does Snapshots,
# and that (2) capture_snapshots preserves ordering.
digests = [snapshot.digest for snapshot in snapshots]
digest_iterator = iter(digests)
snapshotted_targets_and_jars = []
for target, jars_to_snapshot in targets_and_jars:
snapshotted_jars = [
ResolvedJar(
coordinate=jar.coordinate,
cache_path=jar.cache_path,
pants_path=jar.pants_path,
directory_digest=next(digest_iterator),
) for jar in jars_to_snapshot
]
snapshotted_targets_and_jars.append((target, snapshotted_jars))
return snapshotted_targets_and_jars
def to_classpath_entries(paths, scheduler):
# list of path ->
# list of (path, optional<digest>) ->
path_and_digests = [(p, Digest.load(os.path.dirname(p))) for p in paths]
# partition: list of path, list of tuples
paths_without_digests = [p for (p, d) in path_and_digests if not d]
if paths_without_digests:
self.context.log.debug('Expected to find digests for {}, capturing them.'
.format(paths_without_digests))
paths_with_digests = [(p, d) for (p, d) in path_and_digests if d]
# list of path -> list path, captured snapshot -> list of path with digest
snapshots = scheduler.capture_snapshots(tuple(pathglob_for(p) for p in paths_without_digests))
captured_paths_and_digests = [(p, s.directory_digest)
for (p, s) in zip(paths_without_digests, snapshots)]
# merge and classpath ify
return [ClasspathEntry(p, d) for (p, d) in paths_with_digests + captured_paths_and_digests]
def _capture_sources(self, vts):
to_capture = []
results_dirs = []
filespecs = []
for vt in vts:
target = vt.target
# Compute the (optional) subdirectory of the results_dir to generate code to. This
# path will end up in the generated FilesetWithSpec and target, and thus needs to be
# located below the stable/symlinked `vt.results_dir`.
synthetic_target_dir = self.synthetic_target_dir(
target, vt.results_dir)
files = self.sources_globs
results_dir_relpath = fast_relpath(synthetic_target_dir,
get_buildroot())
buildroot_relative_globs = tuple(
os.path.join(results_dir_relpath, file) for file in files)
buildroot_relative_excludes = tuple(
os.path.join(results_dir_relpath, file)
for file in self.sources_exclude_globs)
to_capture.append(
PathGlobsAndRoot(
PathGlobs(buildroot_relative_globs,
buildroot_relative_excludes),
text_type(get_buildroot()),
# The digest is stored adjacent to the hash-versioned `vt.current_results_dir`.
Digest.load(vt.current_results_dir),
))
results_dirs.append(results_dir_relpath)
filespecs.append(
FilesetRelPathWrapper.to_filespec(buildroot_relative_globs))
snapshots = self.context._scheduler.capture_snapshots(
tuple(to_capture))
for snapshot, vt in zip(snapshots, vts):
snapshot.directory_digest.dump(vt.current_results_dir)
return tuple(
EagerFilesetWithSpec(
results_dir_relpath,
filespec,
snapshot,
) for (results_dir_relpath, filespec,
snapshot) in zip(results_dirs, filespecs, snapshots))
def _capture_sources(self, vts):
to_capture = []
results_dirs = []
filespecs = []
for vt in vts:
target = vt.target
# Compute the (optional) subdirectory of the results_dir to generate code to. This
# path will end up in the generated FilesetWithSpec and target, and thus needs to be
# located below the stable/symlinked `vt.results_dir`.
synthetic_target_dir = self.synthetic_target_dir(target, vt.results_dir)
files = self.sources_globs
results_dir_relpath = fast_relpath(synthetic_target_dir, get_buildroot())
buildroot_relative_globs = tuple(os.path.join(results_dir_relpath, file) for file in files)
buildroot_relative_excludes = tuple(
os.path.join(results_dir_relpath, file)
for file in self.sources_exclude_globs
)
to_capture.append(
PathGlobsAndRoot(
PathGlobs(buildroot_relative_globs, buildroot_relative_excludes),
text_type(get_buildroot()),
# The digest is stored adjacent to the hash-versioned `vt.current_results_dir`.
Digest.load(vt.current_results_dir),
)
)
results_dirs.append(results_dir_relpath)
filespecs.append(FilesetRelPathWrapper.to_filespec(buildroot_relative_globs))
snapshots = self.context._scheduler.capture_snapshots(tuple(to_capture))
for snapshot, vt in zip(snapshots, vts):
snapshot.directory_digest.dump(vt.current_results_dir)
return tuple(EagerFilesetWithSpec(
results_dir_relpath,
filespec,
snapshot,
) for (results_dir_relpath, filespec, snapshot) in zip(results_dirs, filespecs, snapshots))
def _capture_resources(self, vts):
"""Given a list of VersionedTargets, capture DirectoryDigests for all of them.
:returns: A list of tuples of VersionedTargets and digests for their content.
"""
# Capture Snapshots for each directory, using an optional adjacent digest. Create the digest
# afterward if it does not exist.
buildroot = get_buildroot()
snapshots = self.context._scheduler.capture_snapshots(
tuple(
PathGlobsAndRoot(
PathGlobs([
os.path.join(fast_relpath(vt.results_dir, buildroot),
"**")
]),
buildroot,
Digest.load(vt.current_results_dir),
) for vt in vts))
result = []
for vt, snapshot in zip(vts, snapshots):
snapshot.directory_digest.dump(vt.current_results_dir)
result.append((vt, snapshot.directory_digest))
return result
