def test_classpath_by_targets(self):
b = self.make_target('b', JvmTarget)
a = self.make_target('a',
JvmTarget,
dependencies=[b],
excludes=[Exclude('com.example', 'lib')])
classpath_products = ClasspathProducts(self.pants_workdir)
path1 = self._path('jar/path1')
path2 = self._path('jar/path2')
path3 = os.path.join(self.pants_workdir, 'jar/path3')
resolved_jar = ResolvedJar(M2Coordinate(org='com.example',
name='lib',
rev='1.0'),
cache_path='somewhere',
pants_path=path3)
classpath_products.add_for_target(a, [('default', path1)])
classpath_products.add_for_target(a, [('non-default', path2)])
classpath_products.add_for_target(b, [('default', path2)])
classpath_products.add_jars_for_targets([b], 'default', [resolved_jar])
classpath_products.add_excludes_for_targets([a])
# (a, path2) filtered because of conf
# (b, path3) filtered because of excludes
self.assertEqual(
OrderedDict([(a, [ClasspathEntry(path1)]),
(b, [ClasspathEntry(path2)])]),
ClasspathUtil.classpath_by_targets(a.closure(bfs=True),
classpath_products))
def test_get_product_target_mappings_for_targets_intransitive(self):
b = self.make_target('b',
JvmTarget,
excludes=[Exclude('com.example', 'lib')])
a = self.make_target('a', JvmTarget, dependencies=[b])
classpath_product = ClasspathProducts(self.pants_workdir)
example_jar_path = self._example_jar_path()
resolved_jar = self.add_jar_classpath_element_for_path(
classpath_product, a, example_jar_path)
classpath_product.add_for_target(
b, [('default', self.path('b/loose/classes/dir'))])
classpath_product.add_for_target(
a, [('default', self.path('a/loose/classes/dir')),
('default', self.path('an/internally/generated.jar'))])
classpath_target_tuples = classpath_product.get_product_target_mappings_for_targets(
[a])
self.assertEqual([
(('default',
ArtifactClasspathEntry(example_jar_path, resolved_jar.coordinate,
resolved_jar.cache_path)), a),
(('default', ClasspathEntry(self.path('a/loose/classes/dir'))), a),
(('default',
ClasspathEntry(self.path('an/internally/generated.jar'))), a)
], classpath_target_tuples)
def create_compile_context(self, target, target_workdir):
# workdir layout:
# rsc/
# - outline/ -- semanticdbs for the current target as created by rsc
# - m.jar -- reified scala signature jar
# zinc/
# - classes/ -- class files
# - z.analysis -- zinc analysis for the target
# - z.jar -- final jar for the target
# - zinc_args -- file containing the used zinc args
sources = self._compute_sources_for_target(target)
rsc_dir = os.path.join(target_workdir, "rsc")
zinc_dir = os.path.join(target_workdir, "zinc")
return [
RscCompileContext(
target=target,
analysis_file=None,
classes_dir=None,
jar_file=None,
zinc_args_file=None,
rsc_jar_file=os.path.join(rsc_dir, 'm.jar'),
log_dir=os.path.join(rsc_dir, 'logs'),
sources=sources,
workflow=self._classify_target_compile_workflow(target),
),
CompileContext(
target=target,
analysis_file=os.path.join(zinc_dir, 'z.analysis'),
classes_dir=ClasspathEntry(os.path.join(zinc_dir, 'classes'), None),
jar_file=ClasspathEntry(os.path.join(zinc_dir, 'z.jar'), None),
log_dir=os.path.join(zinc_dir, 'logs'),
zinc_args_file=os.path.join(zinc_dir, 'zinc_args'),
sources=sources,
)
]
def test_get_internal_classpath_entries_for_targets(self):
b = self.make_target("b", JvmTarget)
a = self.make_target("a", JvmTarget, dependencies=[b])
classpath_product = ClasspathProducts(self.pants_workdir)
# This artifact classpath entry should be ignored.
example_jar_path = self._example_jar_path()
self.add_jar_classpath_element_for_path(classpath_product, a,
example_jar_path)
classpath_product.add_for_target(
b, [("default", self.path("b/loose/classes/dir"))])
classpath_product.add_for_target(
a,
[
("default", self.path("a/loose/classes/dir")),
("default", self.path("an/internally/generated.jar")),
],
)
classpath = classpath_product.get_internal_classpath_entries_for_targets(
a.closure(bfs=True))
self.assertEqual(
[
("default", ClasspathEntry(self.path("a/loose/classes/dir"))),
("default",
ClasspathEntry(self.path("an/internally/generated.jar"))),
("default", ClasspathEntry(self.path("b/loose/classes/dir"))),
],
classpath,
)
def test_classpath_by_targets(self):
b = self.make_target("b", JvmTarget)
a = self.make_target("a",
JvmTarget,
dependencies=[b],
excludes=[Exclude("com.example", "lib")])
classpath_products = ClasspathProducts(self.pants_workdir)
path1 = self._path("jar/path1")
path2 = self._path("jar/path2")
path3 = os.path.join(self.pants_workdir, "jar/path3")
resolved_jar = ResolvedJar(
M2Coordinate(org="com.example", name="lib", rev="1.0"),
cache_path="somewhere",
pants_path=path3,
)
classpath_products.add_for_target(a, [("default", path1)])
classpath_products.add_for_target(a, [("non-default", path2)])
classpath_products.add_for_target(b, [("default", path2)])
classpath_products.add_jars_for_targets([b], "default", [resolved_jar])
classpath_products.add_excludes_for_targets([a])
# (a, path2) filtered because of conf
# (b, path3) filtered because of excludes
self.assertEqual(
OrderedDict([(a, [ClasspathEntry(path1)]),
(b, [ClasspathEntry(path2)])]),
ClasspathUtil.classpath_by_targets(a.closure(bfs=True),
classpath_products),
)
def create_compile_context(self, target, target_workdir):
return CompileContext(target=target,
analysis_file=os.path.join(target_workdir, 'z.analysis'),
classes_dir=ClasspathEntry(os.path.join(target_workdir, 'classes')),
jar_file=ClasspathEntry(os.path.join(target_workdir, 'z.jar')),
log_dir=os.path.join(target_workdir, 'logs'),
args_file=os.path.join(target_workdir, 'zinc_args'),
post_compile_merge_dir=os.path.join(target_workdir,
'post_compile_merge_dir'),
sources=self._compute_sources_for_target(target))
def execute(self):
# Tracked and returned for use in tests.
# TODO: Rewrite those tests. execute() is not supposed to return anything.
processed_targets = []
compile_classpath = self.context.products.get_data("compile_classpath")
runtime_classpath = self.context.products.get_data(
"runtime_classpath", compile_classpath.copy)
all_relevant_resources_targets = self.find_all_relevant_resources_targets(
)
if not all_relevant_resources_targets:
return processed_targets
with self.invalidated(
targets=all_relevant_resources_targets,
fingerprint_strategy=self.create_invalidation_strategy(),
invalidate_dependents=False,
topological_order=False,
) as invalidation:
for vt in invalidation.invalid_vts:
# Generate resources to the chroot.
self.prepare_resources(vt.target, vt.results_dir)
processed_targets.append(vt.target)
for vt, digest in self._capture_resources(invalidation.all_vts):
# Register the target's chroot in the products.
for conf in self.get_options().confs:
runtime_classpath.add_for_target(
vt.target,
[(conf, ClasspathEntry(vt.results_dir, digest))])
return processed_targets
def _compile_compiler_bridge(self, context):
"""Compile the compiler bridge to be used by zinc, using our scala bootstrapper. It will
compile and cache the jar, and materialize it if not already there.
:param context: The context of the task trying to compile the bridge.
This is mostly needed to use its scheduler to create digests of the relevant jars.
:return: The absolute path to the compiled scala-compiler-bridge jar.
"""
bridge_jar_name = "scala-compiler-bridge.jar"
bridge_jar = os.path.join(self._compiler_bridge_cache_dir, bridge_jar_name)
global_bridge_cache_dir = os.path.join(
self._zinc_factory.get_options().pants_bootstrapdir,
fast_relpath(self._compiler_bridge_cache_dir, self._workdir()),
)
globally_cached_bridge_jar = os.path.join(global_bridge_cache_dir, bridge_jar_name)
# Workaround to avoid recompiling the bridge for every integration test
# We check the bootstrapdir (.cache) for the bridge.
# If it exists, we make a copy to the buildroot.
#
# TODO Remove when action caches are implemented.
if os.path.exists(globally_cached_bridge_jar):
# Cache the bridge jar under buildroot, to allow snapshotting
safe_mkdir(self._relative_to_buildroot(self._compiler_bridge_cache_dir))
safe_hardlink_or_copy(globally_cached_bridge_jar, bridge_jar)
if not os.path.exists(bridge_jar):
res = self._run_bootstrapper(bridge_jar, context)
context._scheduler.materialize_directory(
DirectoryToMaterialize(res.output_directory_digest)
)
# For the workaround above to work, we need to store a copy of the bridge in
# the bootstrapdir cache (.cache).
safe_mkdir(global_bridge_cache_dir)
safe_hardlink_or_copy(bridge_jar, globally_cached_bridge_jar)
return ClasspathEntry(bridge_jar, res.output_directory_digest)
else:
bridge_jar_snapshot = context._scheduler.capture_snapshots(
(
PathGlobsAndRoot(
PathGlobs((self._relative_to_buildroot(bridge_jar),)), get_buildroot()
),
)
)[0]
bridge_jar_digest = bridge_jar_snapshot.directory_digest
return ClasspathEntry(bridge_jar, bridge_jar_digest)
def _nailgun_server_classpath_entry(self):
nailgun_jar = self.tool_jar('nailgun-server')
nailgun_jar_snapshot, = self.context._scheduler.capture_snapshots(
(PathGlobsAndRoot(
PathGlobs((fast_relpath(nailgun_jar, get_buildroot()), )),
get_buildroot()), ))
nailgun_jar_digest = nailgun_jar_snapshot.directory_digest
return ClasspathEntry(nailgun_jar, nailgun_jar_digest)
def _snapshotted_classpath(self, results_dir):
relpath = fast_relpath(results_dir, get_buildroot())
(classes_dir_snapshot, ) = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(PathGlobs([relpath]), get_buildroot(),
Digest.load(relpath))
])
return ClasspathEntry(results_dir,
classes_dir_snapshot.directory_digest)
def create_compile_context(self, target, target_workdir):
# workdir layout:
# rsc/
# - outline/ -- semanticdbs for the current target as created by rsc
# - m.jar -- reified scala signature jar, also used for scalac -Youtline
# zinc/
# - classes/ -- class files
# - z.analysis -- zinc analysis for the target
# - z.jar -- final jar for the target
# - zinc_args -- file containing the used zinc args
sources = self._compute_sources_for_target(target)
rsc_dir = os.path.join(target_workdir, "rsc")
zinc_dir = os.path.join(target_workdir, "zinc")
return self.RscZincMergedCompileContexts(
rsc_cc=RscCompileContext(
target=target,
# The analysis_file and classes_dir are not supposed to be useful
# It's a hacky way of preserving most of the logic in zinc_compile.py
# While allowing it to use RscCompileContexts for outlining.
analysis_file=os.path.join(rsc_dir, "z.analysis.outline"),
classes_dir=ClasspathEntry(
os.path.join(rsc_dir, "zinc_classes"), None),
jar_file=None,
args_file=os.path.join(rsc_dir, "rsc_args"),
rsc_jar_file=ClasspathEntry(os.path.join(rsc_dir, "m.jar")),
log_dir=os.path.join(rsc_dir, "logs"),
post_compile_merge_dir=os.path.join(rsc_dir,
"post_compile_merge_dir"),
sources=sources,
workflow=self._classify_target_compile_workflow(target),
diagnostics_out=None,
),
zinc_cc=CompileContext(
target=target,
analysis_file=os.path.join(zinc_dir, "z.analysis"),
classes_dir=ClasspathEntry(os.path.join(zinc_dir, "classes"),
None),
jar_file=ClasspathEntry(os.path.join(zinc_dir, "z.jar"), None),
log_dir=os.path.join(zinc_dir, "logs"),
args_file=os.path.join(zinc_dir, "zinc_args"),
post_compile_merge_dir=os.path.join(zinc_dir,
"post_compile_merge_dir"),
sources=sources,
diagnostics_out=os.path.join(zinc_dir, "diagnostics.json"),
),
)
def _memoized_scalac_classpath(self, scala_path, scheduler):
snapshots = scheduler.capture_snapshots(
tuple(
PathGlobsAndRoot(PathGlobs([path]), get_buildroot())
for path in scala_path))
return [
ClasspathEntry(path, snapshot)
for path, snapshot in list(zip(scala_path, snapshots))
]
def test_get_product_target_mappings_for_targets_transitive(self):
b = self.make_target("b",
JvmTarget,
excludes=[Exclude("com.example", "lib")])
a = self.make_target("a", JvmTarget, dependencies=[b])
classpath_product = ClasspathProducts(self.pants_workdir)
example_jar_path = self._example_jar_path()
resolved_jar = self.add_jar_classpath_element_for_path(
classpath_product, a, example_jar_path)
classpath_product.add_for_target(
b, [("default", self.path("b/loose/classes/dir"))])
classpath_product.add_for_target(
a,
[
("default", self.path("a/loose/classes/dir")),
("default", self.path("an/internally/generated.jar")),
],
)
classpath_target_tuples = classpath_product.get_product_target_mappings_for_targets(
a.closure(bfs=True))
self.assertEqual(
[
(
(
"default",
ArtifactClasspathEntry(example_jar_path,
resolved_jar.coordinate,
resolved_jar.cache_path),
),
a,
),
(("default", ClasspathEntry(
self.path("a/loose/classes/dir"))), a),
(("default",
ClasspathEntry(
self.path("an/internally/generated.jar"))), a),
(("default", ClasspathEntry(
self.path("b/loose/classes/dir"))), b),
],
classpath_target_tuples,
)
def _memoized_scalac_classpath(self, scala_path, scheduler):
buildroot = get_buildroot()
path_globs_and_roots = tuple(
PathGlobsAndRoot(PathGlobs([fast_relpath(path, buildroot)]),
buildroot) for path in scala_path)
snapshots = scheduler.capture_snapshots(path_globs_and_roots)
return [
ClasspathEntry(path, snapshot)
for path, snapshot in list(zip(scala_path, snapshots))
]
def _wrap_path_elements(self, classpath_elements):
wrapped_path_elements = []
for element in classpath_elements:
if len(element) != 2:
raise ValueError("Input must be a list of tuples containing two elements.")
if isinstance(element[1], ClasspathEntry):
wrapped_path_elements.append(element)
else:
wrapped_path_elements.append((element[0], ClasspathEntry(element[1])))
return wrapped_path_elements
def _shaded_jar_as_classpath_entry(self, shaded_jar):
# Capture a Snapshot for the jar.
buildroot = get_buildroot()
snapshot = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([fast_relpath(shaded_jar, buildroot)]),
buildroot,
Digest.load(shaded_jar),
)
])[0]
snapshot.digest.dump(shaded_jar)
return ClasspathEntry(shaded_jar, directory_digest=snapshot.digest)
def compute_classpath(cls, targets, classpath_products,
extra_classpath_tuples, confs):
"""Return the list of classpath entries for a classpath covering the passed targets.
Filters and adds paths from extra_classpath_tuples to the end of the resulting list.
As compute_classpath_entries but expects and returns strings, not ClasspathEntries.
"""
return list(entry.path for entry in cls.compute_classpath_entries(
targets, classpath_products, (
(scope, ClasspathEntry(path))
for scope, path in extra_classpath_tuples), confs))
def to_classpath_entries(paths, scheduler):
# list of path ->
# list of (path, optional<digest>) ->
path_and_digests = [(p, Digest.load(os.path.dirname(p))) for p in paths]
# partition: list of path, list of tuples
paths_without_digests = [p for (p, d) in path_and_digests if not d]
if paths_without_digests:
self.context.log.debug('Expected to find digests for {}, capturing them.'
.format(paths_without_digests))
paths_with_digests = [(p, d) for (p, d) in path_and_digests if d]
# list of path -> list path, captured snapshot -> list of path with digest
snapshots = scheduler.capture_snapshots(tuple(pathglob_for(p) for p in paths_without_digests))
captured_paths_and_digests = [(p, s.directory_digest)
for (p, s) in zip(paths_without_digests, snapshots)]
# merge and classpath ify
return [ClasspathEntry(p, d) for (p, d) in paths_with_digests + captured_paths_and_digests]
def _runtool_hermetic(self, main, tool_name, distribution, input_digest,
ctx):
tool_classpath_abs = self._rsc_classpath
tool_classpath = fast_relpath_collection(tool_classpath_abs)
jvm_options = self._jvm_options
if self._rsc.use_native_image:
#jvm_options = []
if jvm_options:
raise ValueError(
"`{}` got non-empty jvm_options when running with a graal native-image, but this is "
"unsupported. jvm_options received: {}".format(
self.options_scope, safe_shlex_join(jvm_options)))
native_image_path, native_image_snapshot = self._rsc.native_image(
self.context)
additional_snapshots = [native_image_snapshot]
initial_args = [native_image_path]
else:
additional_snapshots = []
initial_args = [
distribution.java,
] + self.get_options().jvm_options + [
'-cp',
os.pathsep.join(tool_classpath),
main,
]
argfile_snapshot, = self.context._scheduler.capture_snapshots([
PathGlobsAndRoot(
PathGlobs([fast_relpath(ctx.args_file, get_buildroot())]),
get_buildroot(),
),
])
cmd = initial_args + ['@{}'.format(argfile_snapshot.files[0])]
pathglobs = list(tool_classpath)
if pathglobs:
root = PathGlobsAndRoot(PathGlobs(tuple(pathglobs)),
get_buildroot())
# dont capture snapshot, if pathglobs is empty
path_globs_input_digest = self.context._scheduler.capture_snapshots(
(root, ))[0].directory_digest
epr_input_files = self.context._scheduler.merge_directories(
((path_globs_input_digest, ) if path_globs_input_digest else ()) +
((input_digest, ) if input_digest else ()) +
tuple(s.directory_digest for s in additional_snapshots) +
(argfile_snapshot.directory_digest, ))
epr = ExecuteProcessRequest(
argv=tuple(cmd),
input_files=epr_input_files,
output_files=(fast_relpath(ctx.rsc_jar_file.path,
get_buildroot()), ),
output_directories=tuple(),
timeout_seconds=15 * 60,
description='run {} for {}'.format(tool_name, ctx.target),
# TODO: These should always be unicodes
# Since this is always hermetic, we need to use `underlying.home` because
# ExecuteProcessRequest requires an existing, local jdk location.
jdk_home=distribution.underlying_home,
)
res = self.context.execute_process_synchronously_without_raising(
epr, self.name(), [WorkUnitLabel.COMPILER])
if res.exit_code != 0:
raise TaskError(res.stderr, exit_code=res.exit_code)
# TODO: parse the output of -Xprint:timings for rsc and write it to self._record_target_stats()!
res.output_directory_digest.dump(ctx.rsc_jar_file.path)
ctx.rsc_jar_file = ClasspathEntry(ctx.rsc_jar_file.path,
res.output_directory_digest)
self.context._scheduler.materialize_directories((
DirectoryToMaterialize(
# NB the first element here is the root to materialize into, not the dir to snapshot
get_buildroot(),
res.output_directory_digest), ))
return res
def _wrap_path_elements(self, classpath_elements): return [(element[0], ClasspathEntry(element[1])) for element in classpath_elements]
def _set_directory_digest_for_compile_context(self, ctx, directory_digest):
if self.get_options().use_classpath_jars:
ctx.jar_file = ClasspathEntry(ctx.jar_file.path, directory_digest)
else:
ctx.classes_dir = ClasspathEntry(ctx.classes_dir.path, directory_digest)