def test_merge_directories(self):
with temporary_dir() as temp_dir:
with open(os.path.join(temp_dir, "roland"), "w") as f:
f.write("European Burmese")
with open(os.path.join(temp_dir, "susannah"), "w") as f:
f.write("Not sure actually")
scheduler = self.mk_scheduler(rules=create_fs_rules())
(empty_snapshot, roland_snapshot, susannah_snapshot,
both_snapshot) = (scheduler.capture_snapshots((
PathGlobsAndRoot(PathGlobs(("doesnotmatch", ), ()), temp_dir),
PathGlobsAndRoot(PathGlobs(("roland", ), ()), temp_dir),
PathGlobsAndRoot(PathGlobs(("susannah", ), ()), temp_dir),
PathGlobsAndRoot(PathGlobs(("*", ), ()), temp_dir),
)))
empty_merged = scheduler.merge_directories(
(empty_snapshot.directory_digest))
self.assertEquals(
empty_snapshot.directory_digest,
empty_merged,
)
roland_merged = scheduler.merge_directories((
roland_snapshot.directory_digest,
empty_snapshot.directory_digest,
))
self.assertEquals(
roland_snapshot.directory_digest,
roland_merged,
)
both_merged = scheduler.merge_directories((
roland_snapshot.directory_digest,
susannah_snapshot.directory_digest,
))
self.assertEquals(both_snapshot.directory_digest, both_merged)
def test_synchronously_merge_directories(self):
with temporary_dir() as temp_dir:
with open(os.path.join(temp_dir, "roland"), "w") as f:
f.write("European Burmese")
with open(os.path.join(temp_dir, "susannah"), "w") as f:
f.write("Not sure actually")
(
empty_snapshot,
roland_snapshot,
susannah_snapshot,
both_snapshot,
) = self.scheduler.capture_snapshots(
(
PathGlobsAndRoot(PathGlobs(["doesnotmatch"]), temp_dir),
PathGlobsAndRoot(PathGlobs(["roland"]), temp_dir),
PathGlobsAndRoot(PathGlobs(["susannah"]), temp_dir),
PathGlobsAndRoot(PathGlobs(["*"]), temp_dir),
)
)
empty_merged = self.scheduler.merge_directories((empty_snapshot.directory_digest,))
self.assertEqual(
empty_snapshot.directory_digest, empty_merged,
)
roland_merged = self.scheduler.merge_directories(
(roland_snapshot.directory_digest, empty_snapshot.directory_digest,)
)
self.assertEqual(
roland_snapshot.directory_digest, roland_merged,
)
both_merged = self.scheduler.merge_directories(
(roland_snapshot.directory_digest, susannah_snapshot.directory_digest,)
)
self.assertEqual(both_snapshot.directory_digest, both_merged)
def test_asynchronously_merge_directories(self):
with temporary_dir() as temp_dir:
with open(os.path.join(temp_dir, "roland"), "w") as f:
f.write("European Burmese")
with open(os.path.join(temp_dir, "susannah"), "w") as f:
f.write("Not sure actually")
(empty_snapshot, roland_snapshot, susannah_snapshot,
both_snapshot) = (self.scheduler.capture_snapshots((
PathGlobsAndRoot(PathGlobs(("doesnotmatch", ), ()), temp_dir),
PathGlobsAndRoot(PathGlobs(("roland", ), ()), temp_dir),
PathGlobsAndRoot(PathGlobs(("susannah", ), ()), temp_dir),
PathGlobsAndRoot(PathGlobs(("*", ), ()), temp_dir),
)))
empty_merged = self.request_single_product(
Digest,
DirectoriesToMerge((empty_snapshot.directory_digest, )),
)
self.assertEqual(empty_snapshot.directory_digest, empty_merged)
roland_merged = self.request_single_product(
Digest,
DirectoriesToMerge((roland_snapshot.directory_digest,
empty_snapshot.directory_digest)),
)
self.assertEqual(
roland_snapshot.directory_digest,
roland_merged,
)
both_merged = self.request_single_product(
Digest,
DirectoriesToMerge((roland_snapshot.directory_digest,
susannah_snapshot.directory_digest)),
)
self.assertEqual(both_snapshot.directory_digest, both_merged)
def test_asynchronously_merge_digests(self) -> None:
with temporary_dir() as temp_dir:
Path(temp_dir, "roland").write_text("European Burmese")
Path(temp_dir, "susannah").write_text("Not sure actually")
(
empty_snapshot,
roland_snapshot,
susannah_snapshot,
both_snapshot,
) = self.scheduler.capture_snapshots((
PathGlobsAndRoot(PathGlobs(["doesnotmatch"]), temp_dir),
PathGlobsAndRoot(PathGlobs(["roland"]), temp_dir),
PathGlobsAndRoot(PathGlobs(["susannah"]), temp_dir),
PathGlobsAndRoot(PathGlobs(["*"]), temp_dir),
))
empty_merged = self.request_product(
Digest, [MergeDigests((empty_snapshot.digest, ))])
assert empty_snapshot.digest == empty_merged
roland_merged = self.request_product(
Digest,
[
MergeDigests(
(roland_snapshot.digest, empty_snapshot.digest))
],
)
assert roland_snapshot.digest == roland_merged
both_merged = self.request_product(
Digest,
[
MergeDigests(
(roland_snapshot.digest, susannah_snapshot.digest))
],
)
assert both_snapshot.digest == both_merged
def test_multiple_snapshots_from_outside_buildroot(self) -> None:
with temporary_dir() as temp_dir:
Path(temp_dir, "roland").write_text("European Burmese")
Path(temp_dir, "susannah").write_text("I don't know")
scheduler = self.mk_scheduler(rules=fs_rules())
snapshots = scheduler.capture_snapshots(
(
PathGlobsAndRoot(PathGlobs(["roland"]), temp_dir),
PathGlobsAndRoot(PathGlobs(["susannah"]), temp_dir),
PathGlobsAndRoot(PathGlobs(["doesnotexist"]), temp_dir),
)
)
assert 3 == len(snapshots)
self.assert_snapshot_equals(
snapshots[0],
["roland"],
Digest("63949aa823baf765eff07b946050d76ec0033144c785a94d3ebd82baa931cd16", 80),
)
self.assert_snapshot_equals(
snapshots[1],
["susannah"],
Digest("d3539cfc21eb4bab328ca9173144a8e932c515b1b9e26695454eeedbc5a95f6f", 82),
)
self.assert_snapshot_equals(snapshots[2], [], EMPTY_DIGEST)
def test_snapshot_from_outside_buildroot(self):
with temporary_dir() as temp_dir:
with open(os.path.join(temp_dir, "roland"), "w") as f:
f.write("European Burmese")
scheduler = self.mk_scheduler(rules=create_fs_rules())
globs = PathGlobs(["*"])
snapshot = scheduler.capture_snapshots(
(PathGlobsAndRoot(globs, temp_dir), ))[0]
self.assert_snapshot_equals(
snapshot,
["roland"],
Digest(
"63949aa823baf765eff07b946050d76ec0033144c785a94d3ebd82baa931cd16",
80),
)
def prime_store_with_roland_digest(self):
"""This method primes the store with a directory of a file named 'roland' and contents 'European Burmese'."""
with temporary_dir() as temp_dir:
with open(os.path.join(temp_dir, "roland"), "w") as f:
f.write("European Burmese")
scheduler = self.mk_scheduler(rules=create_fs_rules())
globs = PathGlobs(("*", ), ())
snapshot = scheduler.capture_snapshots(
(PathGlobsAndRoot(globs, text_type(temp_dir)), ))[0]
self.assert_snapshot_equals(
snapshot, ["roland"],
DirectoryDigest(
text_type(
"63949aa823baf765eff07b946050d76ec0033144c785a94d3ebd82baa931cd16"
), 80))
def prime_store_with_roland_digest(self):
"""This method primes the store with a directory of a file named 'roland' and contents
'European Burmese'."""
with temporary_dir() as temp_dir:
with open(os.path.join(temp_dir, "roland"), "w") as f:
f.write("European Burmese")
globs = PathGlobs(["*"])
snapshot = self.scheduler.capture_snapshots(
(PathGlobsAndRoot(globs, temp_dir), ))[0]
expected_digest = Digest(
"63949aa823baf765eff07b946050d76ec0033144c785a94d3ebd82baa931cd16",
80)
self.assert_snapshot_equals(snapshot, ["roland"], expected_digest)
return expected_digest
def _set_directory_digests_for_valid_target_classpath_directories(self, valid_targets, compile_contexts):
def _get_relative_classpath_for_target(target):
cc = self.select_runtime_context(compile_contexts[target])
if self.get_options().use_classpath_jars:
return fast_relpath(cc.jar_file.path, get_buildroot())
else:
return fast_relpath(cc.classes_dir.path, get_buildroot()) + '/**'
snapshots = self.context._scheduler.capture_snapshots(
tuple(PathGlobsAndRoot(PathGlobs(
[_get_relative_classpath_for_target(target)]
), get_buildroot()) for target in valid_targets))
for target, snapshot in list(zip(valid_targets, snapshots)):
cc = self.select_runtime_context(compile_contexts[target])
self._set_directory_digest_for_compile_context(cc, snapshot.directory_digest)
def register_extra_products_from_contexts(self, targets, compile_contexts):
super().register_extra_products_from_contexts(targets, compile_contexts)
def confify(entries):
return [(conf, e) for e in entries for conf in self._confs]
# Ensure that the jar/rsc jar is on the rsc_mixed_compile_classpath.
for target in targets:
merged_cc = compile_contexts[target]
zinc_cc = merged_cc.zinc_cc
rsc_cc = merged_cc.rsc_cc
# Make sure m.jar is digested if it exists when the target is validated.
if rsc_cc.rsc_jar_file.directory_digest is None and os.path.exists(
rsc_cc.rsc_jar_file.path
):
relpath = fast_relpath(rsc_cc.rsc_jar_file.path, get_buildroot())
(classes_dir_snapshot,) = self.context._scheduler.capture_snapshots(
[
PathGlobsAndRoot(
PathGlobs([relpath]), get_buildroot(), Digest.load(relpath),
),
]
)
rsc_cc.rsc_jar_file.hydrate_missing_directory_digest(
classes_dir_snapshot.directory_digest
)
if rsc_cc.workflow is not None:
cp_entries = match(
rsc_cc.workflow,
{
self.JvmCompileWorkflowType.zinc_only: lambda: confify(
[self._classpath_for_context(zinc_cc)]
),
self.JvmCompileWorkflowType.zinc_java: lambda: confify(
[self._classpath_for_context(zinc_cc)]
),
self.JvmCompileWorkflowType.rsc_and_zinc: lambda: confify(
[rsc_cc.rsc_jar_file]
),
self.JvmCompileWorkflowType.outline_and_zinc: lambda: confify(
[rsc_cc.rsc_jar_file]
),
},
)()
self.context.products.get_data("rsc_mixed_compile_classpath").add_for_target(
target, cp_entries
)
def add_directory_digests_for_jars(self, targets_and_jars):
"""For each target, get DirectoryDigests for its jars and return them zipped with the jars.
:param targets_and_jars: List of tuples of the form (Target, [pants.java.jar.jar_dependency_utils.ResolveJar])
:return: list[tuple[(Target, list[pants.java.jar.jar_dependency_utils.ResolveJar])]
"""
targets_and_jars = list(targets_and_jars)
if not targets_and_jars:
return targets_and_jars
jar_paths = []
for target, jars_to_snapshot in targets_and_jars:
for jar in jars_to_snapshot:
jar_paths.append(fast_relpath(jar.pants_path, get_buildroot()))
# Capture Snapshots for jars, using an optional adjacent digest. Create the digest afterward
# if it does not exist.
snapshots = self.context._scheduler.capture_snapshots(
tuple(
PathGlobsAndRoot(
PathGlobs([jar]),
get_buildroot(),
Digest.load(jar),
) for jar in jar_paths))
for snapshot, jar_path in zip(snapshots, jar_paths):
snapshot.digest.dump(jar_path)
# We want to map back the list[Snapshot] to targets_and_jars
# We assume that (1) jars_to_snapshot has the same number of ResolveJars as snapshots does Snapshots,
# and that (2) capture_snapshots preserves ordering.
digests = [snapshot.digest for snapshot in snapshots]
digest_iterator = iter(digests)
snapshotted_targets_and_jars = []
for target, jars_to_snapshot in targets_and_jars:
snapshotted_jars = [
ResolvedJar(
coordinate=jar.coordinate,
cache_path=jar.cache_path,
pants_path=jar.pants_path,
directory_digest=next(digest_iterator),
) for jar in jars_to_snapshot
]
snapshotted_targets_and_jars.append((target, snapshotted_jars))
return snapshotted_targets_and_jars
def _double_check_cache_for_vts(self, vts, zinc_compile_context):
# Double check the cache before beginning compilation
if self.check_cache(vts):
self.context.log.debug(
f"Snapshotting results for {vts.target.address.spec}")
classpath_entry = self._classpath_for_context(zinc_compile_context)
relpath = fast_relpath(classpath_entry.path, get_buildroot())
(classes_dir_snapshot,
) = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(PathGlobs([relpath]), get_buildroot(),
Digest.load(relpath))
])
classpath_entry.hydrate_missing_directory_digest(
classes_dir_snapshot.digest)
# Re-validate the vts!
vts.update()
def _compile_compiler_bridge(self, context):
"""Compile the compiler bridge to be used by zinc, using our scala bootstrapper. It will
compile and cache the jar, and materialize it if not already there.
:param context: The context of the task trying to compile the bridge.
This is mostly needed to use its scheduler to create digests of the relevant jars.
:return: The absolute path to the compiled scala-compiler-bridge jar.
"""
bridge_jar_name = "scala-compiler-bridge.jar"
bridge_jar = os.path.join(self._compiler_bridge_cache_dir, bridge_jar_name)
global_bridge_cache_dir = os.path.join(
self._zinc_factory.get_options().pants_bootstrapdir,
fast_relpath(self._compiler_bridge_cache_dir, self._workdir()),
)
globally_cached_bridge_jar = os.path.join(global_bridge_cache_dir, bridge_jar_name)
# Workaround to avoid recompiling the bridge for every integration test
# We check the bootstrapdir (.cache) for the bridge.
# If it exists, we make a copy to the buildroot.
#
# TODO Remove when action caches are implemented.
if os.path.exists(globally_cached_bridge_jar):
# Cache the bridge jar under buildroot, to allow snapshotting
safe_mkdir(self._relative_to_buildroot(self._compiler_bridge_cache_dir))
safe_hardlink_or_copy(globally_cached_bridge_jar, bridge_jar)
if not os.path.exists(bridge_jar):
res = self._run_bootstrapper(bridge_jar, context)
context._scheduler.materialize_directory(
DirectoryToMaterialize(res.output_directory_digest)
)
# For the workaround above to work, we need to store a copy of the bridge in
# the bootstrapdir cache (.cache).
safe_mkdir(global_bridge_cache_dir)
safe_hardlink_or_copy(bridge_jar, globally_cached_bridge_jar)
return ClasspathEntry(bridge_jar, res.output_directory_digest)
else:
bridge_jar_snapshot = context._scheduler.capture_snapshots(
(
PathGlobsAndRoot(
PathGlobs((self._relative_to_buildroot(bridge_jar),)), get_buildroot()
),
)
)[0]
bridge_jar_digest = bridge_jar_snapshot.directory_digest
return ClasspathEntry(bridge_jar, bridge_jar_digest)
def _capture_sources(self, vts):
to_capture = []
results_dirs = []
filespecs = []
for vt in vts:
target = vt.target
# Compute the (optional) subdirectory of the results_dir to generate code to. This
# path will end up in the generated FilesetWithSpec and target, and thus needs to be
# located below the stable/symlinked `vt.results_dir`.
synthetic_target_dir = self.synthetic_target_dir(
target, vt.results_dir)
files = self.sources_globs
results_dir_relpath = fast_relpath(synthetic_target_dir,
get_buildroot())
buildroot_relative_globs = tuple(
os.path.join(results_dir_relpath, file) for file in files)
buildroot_relative_excludes = tuple(
os.path.join(results_dir_relpath, file)
for file in self.sources_exclude_globs)
to_capture.append(
PathGlobsAndRoot(
PathGlobs(buildroot_relative_globs,
buildroot_relative_excludes),
text_type(get_buildroot()),
# The digest is stored adjacent to the hash-versioned `vt.current_results_dir`.
Digest.load(vt.current_results_dir),
))
results_dirs.append(results_dir_relpath)
filespecs.append(
FilesetRelPathWrapper.to_filespec(buildroot_relative_globs))
snapshots = self.context._scheduler.capture_snapshots(
tuple(to_capture))
for snapshot, vt in zip(snapshots, vts):
snapshot.directory_digest.dump(vt.current_results_dir)
return tuple(
EagerFilesetWithSpec(
results_dir_relpath,
filespec,
snapshot,
) for (results_dir_relpath, filespec,
snapshot) in zip(results_dirs, filespecs, snapshots))
def process_remote_sources(self):
"""Create synthetic targets with populated sources from remote_sources targets."""
unpacked_sources = self.context.products.get_data(UnpackedArchives)
remote_sources_targets = self.context.targets(
predicate=lambda t: isinstance(t, RemoteSources))
if not remote_sources_targets:
return
snapshot_specs = []
filespecs = []
unpack_dirs = []
for target in remote_sources_targets:
unpacked_archive = unpacked_sources[target.sources_target]
sources = unpacked_archive.found_files
rel_unpack_dir = unpacked_archive.rel_unpack_dir
self.context.log.debug(
'target: {}, rel_unpack_dir: {}, sources: {}'.format(
target, rel_unpack_dir, sources))
sources_in_dir = tuple(
os.path.join(rel_unpack_dir, source) for source in sources)
snapshot_specs.append(
PathGlobsAndRoot(
PathGlobs(sources_in_dir),
get_buildroot(),
))
filespecs.append({'globs': sources_in_dir})
unpack_dirs.append(rel_unpack_dir)
snapshots = self.context._scheduler.capture_snapshots(
tuple(snapshot_specs))
for target, snapshot, filespec, rel_unpack_dir in \
zip(remote_sources_targets, snapshots, filespecs, unpack_dirs):
synthetic_target = self.context.add_new_target(
address=Address(os.path.relpath(self.workdir, get_buildroot()),
target.id),
target_type=target.destination_target_type,
dependencies=target.dependencies,
sources=EagerFilesetWithSpec(rel_unpack_dir, filespec,
snapshot),
derived_from=target,
**target.destination_target_args)
self.context.log.debug(
'synthetic_target: {}'.format(synthetic_target))
for dependent in self.context.build_graph.dependents_of(
target.address):
self.context.build_graph.inject_dependency(
dependent, synthetic_target.address)
def _snapshot_resources(resources, prefix='.'):
with temporary_dir() as root_dir:
for filename, filecontent in resources.items():
safe_file_dump(
os.path.join(os.path.join(root_dir, prefix), filename),
filecontent)
extra_resources_relative_to_rootdir = {
os.path.join(prefix, k): v
for k, v in resources.items()
}
snapshot, = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs(extra_resources_relative_to_rootdir),
root_dir)
])
return snapshot.directory_digest
def add_directory_digests_for_jars(self, jars):
"""Get DirectoryDigests for jars and return them zipped with the jars.
:param jars: List of pants.java.jar.jar_dependency_utils.ResolveJar
:return: List of ResolveJars.
"""
snapshots = self.context._scheduler.capture_snapshots(
tuple(
PathGlobsAndRoot(
PathGlobs([fast_relpath(jar.pants_path, get_buildroot())]),
get_buildroot()) for jar in jars))
return [
ResolvedJar(coordinate=jar.coordinate,
cache_path=jar.cache_path,
pants_path=jar.pants_path,
directory_digest=directory_digest)
for jar, directory_digest in list(
zip(jars,
[snapshot.directory_digest for snapshot in snapshots]))
]
def _capture_sources(self, targets_and_dirs):
to_capture = []
results_dirs = []
filespecs = []
for target, synthetic_target_dir in targets_and_dirs:
if self.sources_globs is None:
files = list(self.find_sources(target, synthetic_target_dir))
else:
files = self.sources_globs
results_dir_relpath = os.path.relpath(synthetic_target_dir,
get_buildroot())
buildroot_relative_globs = tuple(
os.path.join(results_dir_relpath, file) for file in files)
buildroot_relative_excludes = tuple(
os.path.join(results_dir_relpath, file)
for file in self.sources_exclude_globs)
to_capture.append(
PathGlobsAndRoot(
PathGlobs(buildroot_relative_globs,
buildroot_relative_excludes),
str(get_buildroot()),
))
results_dirs.append(results_dir_relpath)
filespecs.append(
FilesetRelPathWrapper.to_filespec(buildroot_relative_globs))
snapshots = self.context._scheduler.capture_snapshots(
tuple(to_capture))
return tuple(
EagerFilesetWithSpec(
results_dir_relpath,
filespec,
snapshot,
) for (results_dir_relpath, filespec,
snapshot) in zip(results_dirs, filespecs, snapshots))
def _capture_resources(self, vts):
"""Given a list of VersionedTargets, capture DirectoryDigests for all of them.
:returns: A list of tuples of VersionedTargets and digests for their content.
"""
# Capture Snapshots for each directory, using an optional adjacent digest. Create the digest
# afterward if it does not exist.
buildroot = get_buildroot()
snapshots = self.context._scheduler.capture_snapshots(
tuple(
PathGlobsAndRoot(
PathGlobs([
os.path.join(fast_relpath(vt.results_dir, buildroot),
"**")
]),
buildroot,
Digest.load(vt.current_results_dir),
) for vt in vts))
result = []
for vt, snapshot in zip(vts, snapshots):
snapshot.directory_digest.dump(vt.current_results_dir)
result.append((vt, snapshot.directory_digest))
return result
def find_libs_path_globs(self, names):
libs_abs = self._underlying.find_libs(names)
libs_unrooted = [self._unroot_lib_path(l) for l in libs_abs]
path_globs = PathGlobsAndRoot(PathGlobs(tuple(libs_unrooted)),
text_type(self._underlying.home))
return (libs_unrooted, path_globs)
def _runtool_hermetic(self, main, tool_name, distribution, input_digest,
ctx):
tool_classpath_abs = self._rsc_classpath
tool_classpath = fast_relpath_collection(tool_classpath_abs)
jvm_options = self._jvm_options
if self._rsc.use_native_image:
#jvm_options = []
if jvm_options:
raise ValueError(
"`{}` got non-empty jvm_options when running with a graal native-image, but this is "
"unsupported. jvm_options received: {}".format(
self.options_scope, safe_shlex_join(jvm_options)))
native_image_path, native_image_snapshot = self._rsc.native_image(
self.context)
additional_snapshots = [native_image_snapshot]
initial_args = [native_image_path]
else:
additional_snapshots = []
initial_args = [
distribution.java,
] + self.get_options().jvm_options + [
'-cp',
os.pathsep.join(tool_classpath),
main,
]
argfile_snapshot, = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([fast_relpath(ctx.args_file, get_buildroot())]),
get_buildroot(),
),
])
cmd = initial_args + ['@{}'.format(argfile_snapshot.files[0])]
pathglobs = list(tool_classpath)
if pathglobs:
root = PathGlobsAndRoot(PathGlobs(tuple(pathglobs)),
get_buildroot())
# dont capture snapshot, if pathglobs is empty
path_globs_input_digest = self.context._scheduler.capture_snapshots(
(root, ))[0].directory_digest
epr_input_files = self.context._scheduler.merge_directories(
((path_globs_input_digest, ) if path_globs_input_digest else ()) +
((input_digest, ) if input_digest else ()) +
tuple(s.directory_digest for s in additional_snapshots) +
(argfile_snapshot.directory_digest, ))
epr = ExecuteProcessRequest(
argv=tuple(cmd),
input_files=epr_input_files,
output_files=(fast_relpath(ctx.rsc_jar_file.path,
get_buildroot()), ),
output_directories=tuple(),
timeout_seconds=15 * 60,
description='run {} for {}'.format(tool_name, ctx.target),
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying.home` because
# ExecuteProcessRequest requires an existing, local jdk location.
jdk_home=distribution.underlying_home,
)
res = self.context.execute_process_synchronously_without_raising(
epr, self.name(), [WorkUnitLabel.COMPILER])
if res.exit_code != 0:
raise TaskError(res.stderr, exit_code=res.exit_code)
# TODO: parse the output of -Xprint:timings for rsc and write it to self._record_target_stats()!
res.output_directory_digest.dump(ctx.rsc_jar_file.path)
ctx.rsc_jar_file = ClasspathEntry(ctx.rsc_jar_file.path,
res.output_directory_digest)
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(
# NB the first element here is the root to materialize into, not the dir to snapshot
get_buildroot(),
res.output_directory_digest), ))
return res
def _runtool(
self, main, tool_name, args, distribution, tgt=None, input_files=tuple(), input_digest=None, output_dir=None):
if self.execution_strategy == self.HERMETIC:
with self.context.new_workunit(tool_name) as wu:
tool_classpath_abs = self.tool_classpath(tool_name)
tool_classpath = fast_relpath_collection(tool_classpath_abs)
classpath_for_cmd = os.pathsep.join(tool_classpath)
cmd = [
distribution.java,
]
cmd.extend(self.get_options().jvm_options)
cmd.extend(['-cp', classpath_for_cmd])
cmd.extend([main])
cmd.extend(args)
pathglobs = list(tool_classpath)
pathglobs.extend(f if os.path.isfile(f) else '{}/**'.format(f) for f in input_files)
if pathglobs:
root = PathGlobsAndRoot(
PathGlobs(tuple(pathglobs)),
text_type(get_buildroot()))
# dont capture snapshot, if pathglobs is empty
path_globs_input_digest = self.context._scheduler.capture_snapshots((root,))[0].directory_digest
if path_globs_input_digest and input_digest:
epr_input_files = self.context._scheduler.merge_directories(
(path_globs_input_digest, input_digest))
else:
epr_input_files = path_globs_input_digest or input_digest
epr = ExecuteProcessRequest(
argv=tuple(cmd),
input_files=epr_input_files,
output_files=tuple(),
output_directories=(output_dir,),
timeout_seconds=15*60,
description='run {} for {}'.format(tool_name, tgt),
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying_dist`
jdk_home=text_type(self._zinc.underlying_dist.home),
)
res = self.context.execute_process_synchronously_without_raising(
epr,
self.name(),
[WorkUnitLabel.TOOL])
if res.exit_code != 0:
raise TaskError(res.stderr)
if output_dir:
dump_digest(output_dir, res.output_directory_digest)
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(
# NB the first element here is the root to materialize into, not the dir to snapshot
text_type(get_buildroot()),
res.output_directory_digest),
))
# TODO drop a file containing the digest, named maybe output_dir.digest
return res
else:
with self.context.new_workunit(tool_name) as wu:
result = self.runjava(classpath=self.tool_classpath(tool_name),
main=main,
jvm_options=self.get_options().jvm_options,
args=args,
workunit_name=tool_name,
workunit_labels=[WorkUnitLabel.TOOL],
dist=distribution
)
if result != 0:
raise TaskError('Running {} failed'.format(tool_name))
runjava_wu = None
for c in wu.children:
if c.name is tool_name:
runjava_wu = c
break
if runjava_wu is None:
raise Exception('couldnt find work unit for underlying execution')
return runjava_wu
def _runtool_hermetic(self, main, tool_name, distribution, input_digest,
ctx):
use_youtline = tool_name == "scalac-outliner"
tool_classpath_abs = self._scalac_classpath if use_youtline else self._rsc_classpath
tool_classpath = fast_relpath_collection(tool_classpath_abs)
rsc_jvm_options = Rsc.global_instance().get_options().jvm_options
if not use_youtline and self._rsc.use_native_image:
if rsc_jvm_options:
raise ValueError(
"`{}` got non-empty jvm_options when running with a graal native-image, but this is "
"unsupported. jvm_options received: {}".format(
self.options_scope, safe_shlex_join(rsc_jvm_options)))
native_image_path, native_image_snapshot = self._rsc.native_image(
self.context)
additional_snapshots = [native_image_snapshot]
initial_args = [native_image_path]
else:
additional_snapshots = []
initial_args = (
[distribution.java] + rsc_jvm_options +
["-cp", os.pathsep.join(tool_classpath), main])
(argfile_snapshot, ) = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([fast_relpath(ctx.args_file, get_buildroot())]),
get_buildroot(),
),
])
cmd = initial_args + [f"@{argfile_snapshot.files[0]}"]
pathglobs = list(tool_classpath)
if pathglobs:
root = PathGlobsAndRoot(PathGlobs(tuple(pathglobs)),
get_buildroot())
# dont capture snapshot, if pathglobs is empty
path_globs_input_digest = self.context._scheduler.capture_snapshots(
(root, ))[0].digest
epr_input_files = self.context._scheduler.merge_directories(
((path_globs_input_digest, ) if path_globs_input_digest else ()) +
((input_digest, ) if input_digest else ()) +
tuple(s.digest
for s in additional_snapshots) + (argfile_snapshot.digest, ))
epr = Process(
argv=tuple(cmd),
input_digest=epr_input_files,
output_files=(fast_relpath(ctx.rsc_jar_file.path,
get_buildroot()), ),
output_directories=tuple(),
timeout_seconds=15 * 60,
description=f"run {tool_name} for {ctx.target}",
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying.home` because
# Process requires an existing, local jdk location.
jdk_home=distribution.underlying_home,
is_nailgunnable=True,
)
res = self.context.execute_process_synchronously_without_raising(
epr, self.name(), [WorkUnitLabel.COMPILER])
if res.exit_code != 0:
raise TaskError(res.stderr, exit_code=res.exit_code)
# TODO: parse the output of -Xprint:timings for rsc and write it to self._record_target_stats()!
res.output_digest.dump(ctx.rsc_jar_file.path)
self.context._scheduler.materialize_directory(
DirectoryToMaterialize(res.output_digest), )
ctx.rsc_jar_file.hydrate_missing_directory_digest(res.output_digest)
return res
def test_remove_prefix(rule_runner: RuleRunner) -> None:
relevant_files = (
"characters/dark_tower/roland",
"characters/dark_tower/susannah",
)
all_files = (
"books/dark_tower/gunslinger",
"characters/altered_carbon/kovacs",
*relevant_files,
"index",
)
with temporary_dir() as temp_dir:
safe_file_dump(os.path.join(temp_dir, "index"), "books\ncharacters\n")
safe_file_dump(
os.path.join(temp_dir, "characters", "altered_carbon", "kovacs"),
"Envoy",
makedirs=True,
)
tower_dir = os.path.join(temp_dir, "characters", "dark_tower")
safe_file_dump(os.path.join(tower_dir, "roland"), "European Burmese", makedirs=True)
safe_file_dump(os.path.join(tower_dir, "susannah"), "Not sure actually", makedirs=True)
safe_file_dump(
os.path.join(temp_dir, "books", "dark_tower", "gunslinger"),
"1982",
makedirs=True,
)
snapshot, snapshot_with_extra_files = rule_runner.scheduler.capture_snapshots(
[
PathGlobsAndRoot(PathGlobs(["characters/dark_tower/*"]), temp_dir),
PathGlobsAndRoot(PathGlobs(["**"]), temp_dir),
]
)
# Check that we got the full snapshots that we expect
assert snapshot.files == relevant_files
assert snapshot_with_extra_files.files == all_files
# Strip empty prefix:
zero_prefix_stripped_digest = rule_runner.request(
Digest, [RemovePrefix(snapshot.digest, "")]
)
assert snapshot.digest == zero_prefix_stripped_digest
# Strip a non-empty prefix shared by all files:
stripped_digest = rule_runner.request(
Digest, [RemovePrefix(snapshot.digest, "characters/dark_tower")]
)
assert stripped_digest == Digest(
fingerprint="71e788fc25783c424db555477071f5e476d942fc958a5d06ffc1ed223f779a8c",
serialized_bytes_length=162,
)
expected_snapshot = assert_single_element(
rule_runner.scheduler.capture_snapshots([PathGlobsAndRoot(PathGlobs(["*"]), tower_dir)])
)
assert expected_snapshot.files == ("roland", "susannah")
assert stripped_digest == expected_snapshot.digest
# Try to strip a prefix which isn't shared by all files:
with pytest.raises(Exception) as exc:
rule_runner.request(
Digest,
[RemovePrefix(snapshot_with_extra_files.digest, "characters/dark_tower")],
)
assert (
"Cannot strip prefix characters/dark_tower from root directory (Digest "
"with hash Fingerprint<28c47f77867f0c8d577d2ada2f06b03fc8e5ef2d780e8942713b26c5e3f434b8>)"
" - root directory contained non-matching directory named: books and file named: index"
) in str(exc.value)
def pathglob_for(filename):
return PathGlobsAndRoot(
PathGlobs(
(fast_relpath_optional(filename, get_buildroot()),)),
text_type(get_buildroot()))
def console_output(self, targets):
if not self.get_options().transitive:
targets = self.context.target_roots
input_snapshots = tuple(
target.sources_snapshot(scheduler=self.context._scheduler) for target in targets
)
input_files = {f for snapshot in input_snapshots for f in snapshot.files}
# TODO: Work out a nice library-like utility for writing an argfile, as this will be common.
with temporary_dir() as tmpdir:
list_file = os.path.join(tmpdir, 'input_files_list')
with open(list_file, 'w') as list_file_out:
for input_file in sorted(input_files):
list_file_out.write(input_file)
list_file_out.write('\n')
list_file_snapshot = self.context._scheduler.capture_snapshots((
PathGlobsAndRoot(
PathGlobs(('input_files_list',)),
tmpdir,
),
))[0]
cloc_path, cloc_snapshot = ClocBinary.global_instance().hackily_snapshot(self.context)
directory_digest = self.context._scheduler.merge_directories(tuple(s.directory_digest for s in
input_snapshots + (
cloc_snapshot,
list_file_snapshot,
)))
cmd = (
'/usr/bin/perl',
cloc_path,
'--skip-uniqueness',
'--ignored=ignored',
'--list-file=input_files_list',
'--report-file=report',
)
# The cloc script reaches into $PATH to look up perl. Let's assume it's in /usr/bin.
req = ExecuteProcessRequest(
argv=cmd,
input_files=directory_digest,
output_files=('ignored', 'report'),
description='cloc',
)
exec_result = self.context.execute_process_synchronously_without_raising(req, 'cloc', (WorkUnitLabel.TOOL,))
files_content_tuple = self.context._scheduler.product_request(
FilesContent,
[exec_result.output_directory_digest]
)[0].dependencies
files_content = {fc.path: fc.content.decode() for fc in files_content_tuple}
for line in files_content['report'].split('\n'):
yield line
if self.get_options().ignored:
yield 'Ignored the following files:'
for line in files_content['ignored'].split('\n'):
yield line
def _compile_hermetic(self, jvm_options, ctx, classes_dir, jar_file,
compiler_bridge_classpath_entry,
dependency_classpath, scalac_classpath_entries):
zinc_relpath = fast_relpath(self._zinc.zinc.path, get_buildroot())
snapshots = [
ctx.target.sources_snapshot(self.context._scheduler),
]
# scala_library() targets with java_sources have circular dependencies on those java source
# files, and we provide them to the same zinc command line that compiles the scala, so we need
# to make sure those source files are available in the hermetic execution sandbox.
java_sources_targets = getattr(ctx.target, 'java_sources', [])
java_sources_snapshots = [
tgt.sources_snapshot(self.context._scheduler)
for tgt in java_sources_targets
]
snapshots.extend(java_sources_snapshots)
# Ensure the dependencies and compiler bridge jars are available in the execution sandbox.
relevant_classpath_entries = (
dependency_classpath + [
compiler_bridge_classpath_entry,
self._nailgun_server_classpath_entry(
), # We include nailgun-server, to use it to start servers when needed from the hermetic execution case.
])
directory_digests = [
entry.directory_digest for entry in relevant_classpath_entries
if entry.directory_digest
]
if len(directory_digests) != len(relevant_classpath_entries):
for dep in relevant_classpath_entries:
if not dep.directory_digest:
raise AssertionError(
"ClasspathEntry {} didn't have a Digest, so won't be present for hermetic "
"execution of zinc".format(dep))
directory_digests.extend(
classpath_entry.directory_digest
for classpath_entry in scalac_classpath_entries)
if self._zinc.use_native_image:
if jvm_options:
raise ValueError(
"`{}` got non-empty jvm_options when running with a graal native-image, but this is "
"unsupported. jvm_options received: {}".format(
self.options_scope, safe_shlex_join(jvm_options)))
native_image_path, native_image_snapshot = self._zinc.native_image(
self.context)
native_image_snapshots = [
native_image_snapshot.directory_digest,
]
scala_boot_classpath = [
classpath_entry.path
for classpath_entry in scalac_classpath_entries
] + [
# We include rt.jar on the scala boot classpath because the compiler usually gets its
# contents from the VM it is executing in, but not in the case of a native image. This
# resolves a `object java.lang.Object in compiler mirror not found.` error.
'.jdk/jre/lib/rt.jar',
# The same goes for the jce.jar, which provides javax.crypto.
'.jdk/jre/lib/jce.jar',
]
image_specific_argv = [
native_image_path,
'-java-home',
'.jdk',
'-Dscala.boot.class.path={}'.format(
os.pathsep.join(scala_boot_classpath)),
'-Dscala.usejavacp=true',
]
else:
native_image_snapshots = []
# TODO: Lean on distribution for the bin/java appending here
image_specific_argv = ['.jdk/bin/java'] + jvm_options + [
'-cp', zinc_relpath, Zinc.ZINC_COMPILE_MAIN
]
argfile_snapshot, = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([fast_relpath(ctx.args_file, get_buildroot())]),
get_buildroot(),
),
])
relpath_to_analysis = fast_relpath(ctx.analysis_file, get_buildroot())
merged_local_only_scratch_inputs = self._compute_local_only_inputs(
classes_dir, relpath_to_analysis, jar_file)
# TODO: Extract something common from Executor._create_command to make the command line
argv = image_specific_argv + ['@{}'.format(argfile_snapshot.files[0])]
merged_input_digest = self.context._scheduler.merge_directories(
[self._zinc.zinc.directory_digest] +
[s.directory_digest for s in snapshots] + directory_digests +
native_image_snapshots + [
self.post_compile_extra_resources_digest(ctx),
argfile_snapshot.directory_digest
])
# NB: We always capture the output jar, but if classpath jars are not used, we additionally
# capture loose classes from the workspace. This is because we need to both:
# 1) allow loose classes as an input to dependent compiles
# 2) allow jars to be materialized at the end of the run.
output_directories = () if self.get_options().use_classpath_jars else (
classes_dir, )
req = ExecuteProcessRequest(
argv=tuple(argv),
input_files=merged_input_digest,
output_files=(jar_file, relpath_to_analysis),
output_directories=output_directories,
description="zinc compile for {}".format(ctx.target.address.spec),
unsafe_local_only_files_because_we_favor_speed_over_correctness_for_this_rule
=merged_local_only_scratch_inputs,
jdk_home=self._zinc.underlying_dist.home,
is_nailgunnable=True,
)
res = self.context.execute_process_synchronously_or_raise(
req, self.name(), [WorkUnitLabel.COMPILER])
# TODO: Materialize as a batch in do_compile or somewhere
self.context._scheduler.materialize_directory(
DirectoryToMaterialize(res.output_directory_digest))
# TODO: This should probably return a ClasspathEntry rather than a Digest
return res.output_directory_digest
def _runtool_hermetic(self,
main,
tool_name,
args,
distribution,
tgt=None,
input_files=tuple(),
input_digest=None,
output_dir=None):
tool_classpath_abs = self.tool_classpath(tool_name)
tool_classpath = fast_relpath_collection(tool_classpath_abs)
# TODO(#6071): Our ExecuteProcessRequest expects a specific string type for arguments,
# which py2 doesn't default to. This can be removed when we drop python 2.
str_jvm_options = [
text_type(opt) for opt in self.get_options().jvm_options
]
cmd = [
distribution.java,
] + str_jvm_options + [
'-cp',
os.pathsep.join(tool_classpath),
main,
] + args
pathglobs = list(tool_classpath)
pathglobs.extend(f if os.path.isfile(f) else '{}/**'.format(f)
for f in input_files)
if pathglobs:
root = PathGlobsAndRoot(PathGlobs(tuple(pathglobs)),
text_type(get_buildroot()))
# dont capture snapshot, if pathglobs is empty
path_globs_input_digest = self.context._scheduler.capture_snapshots(
(root, ))[0].directory_digest
epr_input_files = self.context._scheduler.merge_directories((
(path_globs_input_digest, ) if path_globs_input_digest else ()) + (
(input_digest, ) if input_digest else ()))
epr = ExecuteProcessRequest(
argv=tuple(cmd),
input_files=epr_input_files,
output_files=tuple(),
output_directories=(output_dir, ),
timeout_seconds=15 * 60,
description='run {} for {}'.format(tool_name, tgt),
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying.home` because
# ExecuteProcessRequest requires an existing, local jdk location.
jdk_home=text_type(distribution.underlying_home),
)
res = self.context.execute_process_synchronously_without_raising(
epr, self.name(), [WorkUnitLabel.TOOL])
if res.exit_code != 0:
raise TaskError(res.stderr, exit_code=res.exit_code)
if output_dir:
res.output_directory_digest.dump(output_dir)
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(
# NB the first element here is the root to materialize into, not the dir to snapshot
text_type(get_buildroot()),
res.output_directory_digest), ))
# TODO drop a file containing the digest, named maybe output_dir.digest
return res
def console_output(self, targets):
if not self.get_options().transitive:
targets = self.context.target_roots
# TODO: Work out a nice library-like utility for writing an argfile, as this will be common.
with temporary_dir() as tmpdir:
list_file = os.path.join(tmpdir, 'input_files_list')
input_files = set()
with open(list_file, 'w') as list_file_out:
for target in targets:
for source in target.sources_relative_to_buildroot():
input_files.add(source)
list_file_out.write(source)
list_file_out.write(b'\n')
list_file_snapshot = self.context._scheduler.capture_snapshots(
(PathGlobsAndRoot(
PathGlobs(('input_files_list', )),
str(tmpdir),
), ))[0]
cloc_path, cloc_snapshot = ClocBinary.global_instance(
).hackily_snapshot(self.context)
# TODO: This should use an input file snapshot which should be provided on the Target object,
# rather than hackily re-snapshotting each of the input files.
# See https://github.com/pantsbuild/pants/issues/5762
input_pathglobs = PathGlobs(tuple(input_files))
input_snapshot = self.context._scheduler.product_request(
Snapshot, [input_pathglobs])[0]
directory_digest = self.context._scheduler.merge_directories((
cloc_snapshot.directory_digest,
input_snapshot.directory_digest,
list_file_snapshot.directory_digest,
))
cmd = (
'/usr/bin/perl',
cloc_path,
'--skip-uniqueness',
'--ignored=ignored',
'--list-file=input_files_list',
'--report-file=report',
)
# The cloc script reaches into $PATH to look up perl. Let's assume it's in /usr/bin.
req = ExecuteProcessRequest(cmd, (), directory_digest,
('ignored', 'report'), (), 15 * 60, 'cloc')
exec_result = self.context.execute_process_synchronously(
req, 'cloc', (WorkUnitLabel.TOOL, ))
# TODO: Remove this check when https://github.com/pantsbuild/pants/issues/5719 is resolved.
if exec_result.exit_code != 0:
raise TaskError('{} ... exited non-zero ({}).'.format(
' '.join(cmd), exec_result.exit_code))
files_content_tuple = self.context._scheduler.product_request(
FilesContent,
[exec_result.output_directory_digest])[0].dependencies
files_content = {fc.path: fc.content for fc in files_content_tuple}
for line in files_content['report'].split('\n'):
yield line
if self.get_options().ignored:
yield 'Ignored the following files:'
for line in files_content['ignored'].split('\n'):
yield line
def test_strip_prefix(self):
# Set up files:
relevant_files = (
'characters/dark_tower/roland',
'characters/dark_tower/susannah',
)
all_files = (
'books/dark_tower/gunslinger',
'characters/altered_carbon/kovacs',
) + relevant_files + (
'index',
)
with temporary_dir() as temp_dir:
safe_file_dump(os.path.join(temp_dir, 'index'), 'books\ncharacters\n')
safe_file_dump(
os.path.join(temp_dir, "characters", "altered_carbon", "kovacs"),
"Envoy",
makedirs=True,
)
tower_dir = os.path.join(temp_dir, "characters", "dark_tower")
safe_file_dump(os.path.join(tower_dir, "roland"), "European Burmese", makedirs=True)
safe_file_dump(os.path.join(tower_dir, "susannah"), "Not sure actually", makedirs=True)
safe_file_dump(
os.path.join(temp_dir, "books", "dark_tower", "gunslinger"),
"1982",
makedirs=True,
)
snapshot, snapshot_with_extra_files = self.scheduler.capture_snapshots((
PathGlobsAndRoot(PathGlobs(("characters/dark_tower/*",)), temp_dir),
PathGlobsAndRoot(PathGlobs(("**",)), temp_dir),
))
# Check that we got the full snapshots that we expect
self.assertEquals(snapshot.files, relevant_files)
self.assertEquals(snapshot_with_extra_files.files, all_files)
# Strip empty prefix:
zero_prefix_stripped_digest = assert_single_element(self.scheduler.product_request(
Digest,
[DirectoryWithPrefixToStrip(snapshot.directory_digest, "")],
))
self.assertEquals(snapshot.directory_digest, zero_prefix_stripped_digest)
# Strip a non-empty prefix shared by all files:
stripped_digest = assert_single_element(self.scheduler.product_request(
Digest,
[DirectoryWithPrefixToStrip(snapshot.directory_digest, "characters/dark_tower")],
))
self.assertEquals(
stripped_digest,
Digest(
fingerprint='71e788fc25783c424db555477071f5e476d942fc958a5d06ffc1ed223f779a8c',
serialized_bytes_length=162,
)
)
expected_snapshot = assert_single_element(self.scheduler.capture_snapshots((
PathGlobsAndRoot(PathGlobs(("*",)), tower_dir),
)))
self.assertEquals(expected_snapshot.files, ('roland', 'susannah'))
self.assertEquals(stripped_digest, expected_snapshot.directory_digest)
# Try to strip a prefix which isn't shared by all files:
with self.assertRaisesWithMessageContaining(Exception, "Cannot strip prefix characters/dark_tower from root directory Digest(Fingerprint<28c47f77867f0c8d577d2ada2f06b03fc8e5ef2d780e8942713b26c5e3f434b8>, 243) - root directory contained non-matching directory named: books and file named: index"):
self.scheduler.product_request(
Digest,
[DirectoryWithPrefixToStrip(snapshot_with_extra_files.directory_digest, "characters/dark_tower")]
)
