def classname_for_classfile(self, target, classpath_products):
contents = ClasspathUtil.classpath_contents((target,), classpath_products)
for f in contents:
classname = ClasspathUtil.classname_for_rel_classfile(f)
# None for non `.class` files
if classname:
yield classname
def record_file_ownership(target):
entries = ClasspathUtil.internal_classpath([target],
classpath_products,
transitive=False)
for f in ClasspathUtil.classpath_entries_contents(entries):
if not f.endswith('/'):
artifacts_by_file_name[f].add(target.address.reference())
def add_target(self, target, recursive=False):
"""Adds the classes and resources for a target to an open jar.
:param target: The target to add generated classes and resources for.
:param bool recursive: `True` to add classes and resources for the target's transitive
internal dependency closure.
:returns: `True` if the target contributed any files - manifest entries, classfiles or
resource files - to this jar.
:rtype: bool
"""
products_added = False
classpath_products = self._context.products.get_data('runtime_classpath')
# TODO(John Sirois): Manifest handling is broken. We should be tracking state and failing
# fast if any duplicate entries are added; ie: if we get a second binary or a second agent.
if isinstance(target, JvmBinary):
self._add_manifest_entries(target, self._manifest)
products_added = True
# Ensure that JavaAgent entries are added to the manifest. Either by adding all of the
# transitive JavaAgent deps, if recursive, or by adding the root target, if the root target
# is itself a JavaAgent.
if recursive:
agents = [t for t in target.closure() if isinstance(t, JavaAgent)]
if len(agents) > 1:
raise TaskError('Only 1 agent can be added to a jar, found {} for {}:\n\t{}'
.format(len(agents),
target.address.reference(),
'\n\t'.join(agent.address.reference() for agent in agents)))
elif agents:
self._add_agent_manifest(agents[0], self._manifest)
products_added = True
elif isinstance(target, JavaAgent):
self._add_agent_manifest(target, self._manifest)
products_added = True
# In the transitive case we'll gather internal resources naturally as dependencies, but in the
# non-transitive case we need to manually add these special (in the context of jarring)
# dependencies.
targets = target.closure(bfs=True) if recursive else [target]
if not recursive and target.has_resources:
targets += target.resources
# We only gather internal classpath elements per our contract.
target_classpath = ClasspathUtil.internal_classpath(targets,
classpath_products)
for entry in target_classpath:
if ClasspathUtil.is_jar(entry):
self._jar.writejar(entry)
products_added = True
elif ClasspathUtil.is_dir(entry):
for rel_file in ClasspathUtil.classpath_entries_contents([entry]):
self._jar.write(os.path.join(entry, rel_file), rel_file)
products_added = True
else:
# non-jar and non-directory classpath entries should be ignored
pass
return products_added
def execute(self):
basedir = os.path.join(self.get_options().pants_distdir, self._output_folder)
runtime_classpath = self.context.products.get_data('runtime_classpath')
ClasspathUtil.create_canonical_classpath(runtime_classpath,
self.context.targets(),
basedir,
save_classpath_file=True)
def add_target(self, target, recursive=False):
"""Adds the classes and resources for a target to an open jar.
:param target: The target to add generated classes and resources for.
:param bool recursive: `True` to add classes and resources for the target's transitive
internal dependency closure.
:returns: `True` if the target contributed any files - manifest entries, classfiles or
resource files - to this jar.
:rtype: bool
"""
products_added = False
classpath_products = self._context.products.get_data('runtime_classpath')
# TODO(John Sirois): Manifest handling is broken. We should be tracking state and failing
# fast if any duplicate entries are added; ie: if we get a second binary or a second agent.
if isinstance(target, JvmBinary):
self._add_manifest_entries(target, self._manifest)
products_added = True
# Ensure that JavaAgent entries are added to the manifest. Either by adding all of the
# transitive JavaAgent deps, if recursive, or by adding the root target, if the root target
# is itself a JavaAgent.
if recursive:
agents = [t for t in target.closure() if isinstance(t, JavaAgent)]
if len(agents) > 1:
raise TaskError('Only 1 agent can be added to a jar, found {} for {}:\n\t{}'
.format(len(agents),
target.address.reference(),
'\n\t'.join(agent.address.reference() for agent in agents)))
elif agents:
self._add_agent_manifest(agents[0], self._manifest)
products_added = True
elif isinstance(target, JavaAgent):
self._add_agent_manifest(target, self._manifest)
products_added = True
# In the transitive case we'll gather internal resources naturally as dependencies, but in the
# non-transitive case we need to manually add these special (in the context of jarring)
# dependencies.
targets = target.closure(bfs=True) if recursive else [target]
if not recursive and target.has_resources:
targets += target.resources
# We only gather internal classpath elements per our contract.
target_classpath = ClasspathUtil.internal_classpath(targets,
classpath_products)
for entry in target_classpath:
if ClasspathUtil.is_jar(entry):
self._jar.writejar(entry)
products_added = True
elif ClasspathUtil.is_dir(entry):
for rel_file in ClasspathUtil.classpath_entries_contents([entry]):
self._jar.write(os.path.join(entry, rel_file), rel_file)
products_added = True
else:
# non-jar and non-directory classpath entries should be ignored
pass
return products_added
def execute(self):
basedir = os.path.join(self.get_options().pants_distdir,
self._output_folder)
runtime_classpath = self.context.products.get_data('runtime_classpath')
ClasspathUtil.create_canonical_classpath(runtime_classpath,
self.context.targets(),
basedir,
save_classpath_file=True)
def classname_for_classfile(self, target, classpath_products):
contents = ClasspathUtil.classpath_contents((target, ),
classpath_products)
for f in contents:
classname = ClasspathUtil.classname_for_rel_classfile(f)
# None for non `.class` files
if classname:
yield classname
def _gather(self, context, target):
"""Creates a task for the context, and yields the files that would be placed in the dex."""
gathered = []
for entry in self.create_task(context)._gather_dex_entries(target):
if ClasspathUtil.is_jar(entry) or ClasspathUtil.is_dir(entry):
gathered.extend(ClasspathUtil.classpath_entries_contents([entry]))
else:
gathered.append(entry)
return gathered
def add_classpath_entry(entry):
if ClasspathUtil.is_jar(entry):
self._jar.writejar(entry)
elif ClasspathUtil.is_dir(entry):
for rel_file in ClasspathUtil.classpath_entries_contents([entry]):
self._jar.write(os.path.join(entry, rel_file), rel_file)
else:
# non-jar and non-directory classpath entries should be ignored
pass
def _write_to_artifact_cache(self, vts, compile_context,
get_update_artifact_cache_work):
assert len(vts.targets) == 1
assert vts.targets[0] == compile_context.target
# Noop if the target is uncacheable.
if (compile_context.target.has_label('no_cache')):
return
vt = vts.versioned_targets[0]
# Set up args to relativize analysis in the background.
portable_analysis_file = self._portable_analysis_for_target(
self._analysis_dir, compile_context.target)
relativize_args_tuple = (compile_context.analysis_file,
portable_analysis_file)
# Collect the artifacts for this target.
artifacts = []
# Intransitive classpath entries.
target_classpath = ClasspathUtil.classpath_entries(
(compile_context.target, ),
self.context.products.get_data('runtime_classpath'), ('default', ),
transitive=False)
for entry in target_classpath:
if ClasspathUtil.is_jar(entry):
artifacts.append(entry)
elif ClasspathUtil.is_dir(entry):
for rel_file in ClasspathUtil.classpath_entries_contents(
[entry]):
artifacts.append(os.path.join(entry, rel_file))
else:
# non-jar and non-directory classpath entries should be ignored
pass
# Log file.
log_file = self._capture_log_file(compile_context.target)
if log_file and os.path.exists(log_file):
artifacts.append(log_file)
# Jar.
artifacts.append(compile_context.jar_file)
# Get the 'work' that will publish these artifacts to the cache.
# NB: the portable analysis_file won't exist until we finish.
vts_artifactfiles_pair = (vt, artifacts + [portable_analysis_file])
update_artifact_cache_work = get_update_artifact_cache_work(
[vts_artifactfiles_pair])
# And execute it.
if update_artifact_cache_work:
work_chain = [
Work(self._analysis_tools.relativize, [relativize_args_tuple],
'relativize'), update_artifact_cache_work
]
self.context.submit_background_work_chain(
work_chain, parent_workunit_name='cache')
def add_classpath_entry(entry):
if ClasspathUtil.is_jar(entry):
self._jar.writejar(entry)
elif ClasspathUtil.is_dir(entry):
for rel_file in ClasspathUtil.classpath_entries_contents(
[entry]):
self._jar.write(os.path.join(entry, rel_file),
rel_file)
else:
# non-jar and non-directory classpath entries should be ignored
pass
def test_fails_on_paths_outside_buildroot(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
classpath_product.add_for_target(a, [('default', '/dev/null')])
with self.assertRaises(TaskError) as cm:
ClasspathUtil.compute_classpath([a], classpath_product, [], ['default'])
self.assertEqual(
str('Classpath entry /dev/null for target a:a is located outside the buildroot.'),
str(cm.exception))
def _calculate_tests_from_targets(self, targets):
"""
:param list targets: list of targets to calculate test classes for.
generates tuples (class_name, target).
"""
classpath_products = self.context.products.get_data('runtime_classpath')
for target in targets:
contents = ClasspathUtil.classpath_contents((target,), classpath_products, confs=self.confs)
for f in contents:
classname = ClasspathUtil.classname_for_rel_classfile(f)
if classname:
yield (classname, target)
def _target_classes(self, target):
"""Set of target's provided classes.
Call at the target level is to memoize efficiently.
"""
target_classes = set()
contents = ClasspathUtil.classpath_contents((target,), self.runtime_classpath)
for f in contents:
classname = ClasspathUtil.classname_for_rel_classfile(f)
if classname:
target_classes.add(classname)
return target_classes
def _target_classes(self, target):
"""Set of target's provided classes.
Call at the target level is to memoize efficiently.
"""
target_classes = set()
contents = ClasspathUtil.classpath_contents((target,), self.runtime_classpath)
for f in contents:
classname = ClasspathUtil.classname_for_rel_classfile(f)
if classname:
target_classes.add(classname)
return target_classes
def _write_to_artifact_cache(self, vts, compile_context, get_update_artifact_cache_work):
assert len(vts.targets) == 1
assert vts.targets[0] == compile_context.target
# Noop if the target is uncacheable.
if (compile_context.target.has_label('no_cache')):
return
vt = vts.versioned_targets[0]
# Set up args to relativize analysis in the background.
portable_analysis_file = self._portable_analysis_for_target(
self._analysis_dir, compile_context.target)
relativize_args_tuple = (compile_context.analysis_file, portable_analysis_file)
# Collect the artifacts for this target.
artifacts = []
# Intransitive classpath entries.
target_classpath = ClasspathUtil.classpath_entries(
(compile_context.target,),
self.context.products.get_data('runtime_classpath'),
('default',),
transitive=False)
for entry in target_classpath:
if ClasspathUtil.is_jar(entry):
artifacts.append(entry)
elif ClasspathUtil.is_dir(entry):
for rel_file in ClasspathUtil.classpath_entries_contents([entry]):
artifacts.append(os.path.join(entry, rel_file))
else:
# non-jar and non-directory classpath entries should be ignored
pass
# Log file.
log_file = self._capture_log_file(compile_context.target)
if log_file and os.path.exists(log_file):
artifacts.append(log_file)
# Jar.
artifacts.append(compile_context.jar_file)
# Get the 'work' that will publish these artifacts to the cache.
# NB: the portable analysis_file won't exist until we finish.
vts_artifactfiles_pair = (vt, artifacts + [portable_analysis_file])
update_artifact_cache_work = get_update_artifact_cache_work([vts_artifactfiles_pair])
# And execute it.
if update_artifact_cache_work:
work_chain = [
Work(self._analysis_tools.relativize, [relativize_args_tuple], 'relativize'),
update_artifact_cache_work
]
self.context.submit_background_work_chain(work_chain, parent_workunit_name='cache')
def _calculate_tests_from_targets(self, targets):
"""
:param list targets: list of targets to calculate test classes for.
generates tuples (Test, Target).
"""
classpath_products = self.context.products.get_data('runtime_classpath')
for target in targets:
contents = ClasspathUtil.classpath_contents((target,), classpath_products, confs=self.confs)
for f in contents:
classname = ClasspathUtil.classname_for_rel_classfile(f)
if classname:
yield Test(classname=classname), target
def execute(self):
basedir = os.path.join(self.get_options().pants_distdir, self._output_folder)
runtime_classpath = self.context.products.get_data('runtime_classpath')
targets = self.context.targets()
if self.get_options().manifest_jar_only:
classpath = ClasspathUtil.classpath(targets, runtime_classpath)
# Safely create e.g. dist/export-classpath/manifest.jar
safe_classpath(classpath, basedir, "manifest.jar")
else:
ClasspathUtil.create_canonical_classpath(runtime_classpath,
targets,
basedir,
save_classpath_file=True)
def test_fails_on_paths_outside_buildroot(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
classpath_product.add_for_target(a, [('default', '/dev/null')])
with self.assertRaises(TaskError) as cm:
ClasspathUtil.compute_classpath([a], classpath_product, [],
['default'])
self.assertEqual(
str('Classpath entry /dev/null for target a:a is located outside the buildroot.'
), str(cm.exception))
def execute(self):
basedir = os.path.join(self.get_options().pants_distdir,
self._output_folder)
runtime_classpath = self.context.products.get_data('runtime_classpath')
targets = self.context.targets()
if self.get_options().manifest_jar_only:
classpath = ClasspathUtil.classpath(targets, runtime_classpath)
# Safely create e.g. dist/export-classpath/manifest.jar
safe_classpath(classpath, basedir, "manifest.jar")
else:
ClasspathUtil.create_canonical_classpath(runtime_classpath,
targets,
basedir,
save_classpath_file=True)
def _consolidate_classpath(self, targets, classpath_products):
"""Convert loose directories in classpath_products into jars. """
# TODO: find a way to not process classpath entries for valid VTs.
# NB: It is very expensive to call to get entries for each target one at a time.
# For performance reasons we look them all up at once.
entries_map = defaultdict(list)
for (cp, target) in classpath_products.get_product_target_mappings_for_targets(targets, True):
entries_map[target].append(cp)
with self.invalidated(targets=targets, invalidate_dependents=True) as invalidation:
for vt in invalidation.all_vts:
entries = entries_map.get(vt.target, [])
for index, (conf, entry) in enumerate(entries):
if ClasspathUtil.is_dir(entry.path):
jarpath = os.path.join(vt.results_dir, 'output-{}.jar'.format(index))
# Regenerate artifact for invalid vts.
if not vt.valid:
with self.open_jar(jarpath, overwrite=True, compressed=False) as jar:
jar.write(entry.path)
# Replace directory classpath entry with its jarpath.
classpath_products.remove_for_target(vt.target, [(conf, entry)])
classpath_products.add_for_target(vt.target, [(conf, jarpath)])
def _test_canonical_classpath_helper(self,
classpath_products,
targets,
libs_dir,
expected_canonical_classpath,
expected_classspath_files,
excludes=None):
"""
Helper method to call `create_canonical_classpath` and verify generated canonical classpath.
:param ClasspathProducts classpath_products: Classpath products.
:param list targets: List of targets to generate canonical classpath from.
:param string libs_dir: Directory where canonical classpath are to be generated.
:param list expected_canonical_classpath: List of canonical classpath relative to a base directory.
:param dict expected_classspath_files: A dict of classpath.txt path to its expected content.
"""
canonical_classpath = ClasspathUtil.create_canonical_classpath(classpath_products,
targets,
libs_dir,
save_classpath_file=True,
internal_classpath_only=False,
excludes=excludes)
# check canonical path returned
self.assertEquals(expected_canonical_classpath,
relativize_paths(canonical_classpath, libs_dir))
# check canonical path created contain the exact set of files, no more, no less
self.assertTrue(contains_exact_files(libs_dir,
expected_canonical_classpath +
expected_classspath_files.keys()))
# check the content of classpath.txt
for classpath_file in expected_classspath_files:
self.assertTrue(check_file_content(os.path.join(libs_dir, classpath_file),
expected_classspath_files[classpath_file]))
def _compute_classpath_entries(
self, compile_classpaths, target_closure, compile_context, extra_compile_time_classpath
):
# Generate a classpath specific to this compile and target.
return ClasspathUtil.compute_classpath_for_target(
compile_context.target, compile_classpaths, extra_compile_time_classpath, self._confs, target_closure
)
def _compute_classpath_entries(self,
classpath_products,
compile_context,
extra_compile_time_classpath):
# Generate a classpath specific to this compile and target.
target = compile_context.target
if compile_context.strict_deps:
classpath_targets = list(self._compute_strict_dependencies(target))
pruned = [t.address.spec for t in target.closure(bfs=True, **self._target_closure_kwargs)
if t not in classpath_targets]
self.context.log.debug(
'Using strict classpath for {}, which prunes the following dependencies: {}'.format(
target.address.spec, pruned))
else:
classpath_targets = target.closure(bfs=True, **self._target_closure_kwargs)
classpath_targets = (t for t in classpath_targets if t != target)
cp_entries = [compile_context.classes_dir]
cp_entries.extend(ClasspathUtil.compute_classpath(classpath_targets, classpath_products,
extra_compile_time_classpath, self._confs))
if isinstance(target, JavacPlugin):
# Javac plugins need to compile against our distribution's tools.jar. There's no way to
# express this via traversable_dependency_specs, so we inject it into the classpath here.
cp_entries = self.dist.find_libs(['tools.jar']) + cp_entries
return cp_entries
def bundle(self, app):
"""Create a self-contained application bundle.
The bundle will contain the target classes, dependencies and resources.
"""
assert (isinstance(app, BundleCreate.App))
bundle_dir = os.path.join(self.get_options().pants_distdir,
'{}-bundle'.format(app.basename))
self.context.log.info('creating {}'.format(
os.path.relpath(bundle_dir, get_buildroot())))
safe_mkdir(bundle_dir, clean=True)
classpath = OrderedSet()
# Create symlinks for both internal and external dependencies under `lib_dir`. This is
# only needed when not creating a deployjar
lib_dir = os.path.join(bundle_dir, self.LIBS_DIR)
if not self.get_options().deployjar:
os.mkdir(lib_dir)
runtime_classpath = self.context.products.get_data(
'runtime_classpath')
classpath.update(
ClasspathUtil.create_canonical_classpath(
runtime_classpath,
app.target.closure(bfs=True),
lib_dir,
internal_classpath_only=False,
excludes=app.binary.deploy_excludes))
bundle_jar = os.path.join(bundle_dir,
'{}.jar'.format(app.binary.basename))
with self.monolithic_jar(app.binary,
bundle_jar,
manifest_classpath=classpath) as jar:
self.add_main_manifest_entry(jar, app.binary)
# Make classpath complete by adding the monolithic jar.
classpath.update([jar.path])
if app.binary.shading_rules:
for jar_path in classpath:
# In case `jar_path` is a symlink, this is still safe, shaded jar will overwrite jar_path,
# original file `jar_path` linked to remains untouched.
# TODO run in parallel to speed up
self.shade_jar(shading_rules=app.binary.shading_rules,
jar_path=jar_path)
for bundle in app.bundles:
for path, relpath in bundle.filemap.items():
bundle_path = os.path.join(bundle_dir, relpath)
if not os.path.exists(path):
raise TaskError(
'Given path: {} does not exist in target {}'.format(
path, app.address.spec))
safe_mkdir(os.path.dirname(bundle_path))
os.symlink(path, bundle_path)
return bundle_dir
def _test_canonical_classpath_helper(self, classpath_products, targets,
expected_canonical_classpath,
expected_classspath_files,
use_target_id):
"""
Helper method to call `create_canonical_classpath` and verify generated canonical classpath.
:param ClasspathProducts classpath_products: Classpath products.
:param list targets: List of targets to generate canonical classpath from.
:param list expected_canonical_classpath: List of canonical classpath relative to a base directory.
:param dict expected_classspath_files: A dict of classpath.txt path to its expected content.
"""
with temporary_dir() as base_dir:
canonical_classpath = ClasspathUtil.create_canonical_classpath(classpath_products,
targets,
base_dir,
save_classpath_file=True,
use_target_id=use_target_id)
# check canonical path returned
self.assertEquals(expected_canonical_classpath,
relativize_paths(canonical_classpath, base_dir))
# check canonical path created contain the exact set of files, no more, no less
self.assertTrue(contains_exact_files(base_dir,
expected_canonical_classpath +
expected_classspath_files.keys()))
# check the content of classpath.txt
for classpath_file in expected_classspath_files:
self.assertTrue(check_file_content(os.path.join(base_dir, classpath_file),
expected_classspath_files[classpath_file]))
def test_classpath_by_targets(self):
b = self.make_target("b", JvmTarget)
a = self.make_target("a",
JvmTarget,
dependencies=[b],
excludes=[Exclude("com.example", "lib")])
classpath_products = ClasspathProducts(self.pants_workdir)
path1 = self._path("jar/path1")
path2 = self._path("jar/path2")
path3 = os.path.join(self.pants_workdir, "jar/path3")
resolved_jar = ResolvedJar(
M2Coordinate(org="com.example", name="lib", rev="1.0"),
cache_path="somewhere",
pants_path=path3,
)
classpath_products.add_for_target(a, [("default", path1)])
classpath_products.add_for_target(a, [("non-default", path2)])
classpath_products.add_for_target(b, [("default", path2)])
classpath_products.add_jars_for_targets([b], "default", [resolved_jar])
classpath_products.add_excludes_for_targets([a])
# (a, path2) filtered because of conf
# (b, path3) filtered because of excludes
self.assertEqual(
OrderedDict([(a, [ClasspathEntry(path1)]),
(b, [ClasspathEntry(path2)])]),
ClasspathUtil.classpath_by_targets(a.closure(bfs=True),
classpath_products),
)
def _consolidate_classpath(self, targets, classpath_products):
"""Convert loose directories in classpath_products into jars. """
# TODO: find a way to not process classpath entries for valid VTs.
# NB: It is very expensive to call to get entries for each target one at a time.
# For performance reasons we look them all up at once.
entries_map = defaultdict(list)
for (cp, target) in classpath_products.get_product_target_mappings_for_targets(targets, True):
entries_map[target].append(cp)
with self.invalidated(targets=targets, invalidate_dependents=True) as invalidation:
for vt in invalidation.all_vts:
entries = entries_map.get(vt.target, [])
for index, (conf, entry) in enumerate(entries):
if ClasspathUtil.is_dir(entry.path):
jarpath = os.path.join(vt.results_dir, 'output-{}.jar'.format(index))
# Regenerate artifact for invalid vts.
if not vt.valid:
with self.open_jar(jarpath, overwrite=True, compressed=False) as jar:
jar.write(entry.path)
# Replace directory classpath entry with its jarpath.
classpath_products.remove_for_target(vt.target, [(conf, entry.path)])
classpath_products.add_for_target(vt.target, [(conf, jarpath)])
def consolidate_classpath(self, targets, classpath_products):
"""Convert loose directories in classpath_products into jars. """
with self.invalidated(targets=targets,
invalidate_dependents=True) as invalidation:
for vt in invalidation.all_vts:
entries = classpath_products.get_internal_classpath_entries_for_targets(
[vt.target])
for index, (conf, entry) in enumerate(entries):
if ClasspathUtil.is_dir(entry.path):
jarpath = os.path.join(vt.results_dir,
'output-{}.jar'.format(index))
# regenerate artifact for invalid vts
if not vt.valid:
with self.open_jar(jarpath,
overwrite=True,
compressed=False) as jar:
jar.write(entry.path)
# replace directory classpath entry with its jarpath
classpath_products.remove_for_target(
vt.target, [(conf, entry.path)])
classpath_products.add_for_target(
vt.target, [(conf, jarpath)])
def _get_external_dependencies(self, binary_target):
artifacts_by_file_name = defaultdict(set)
for external_dep, coordinate in self.list_external_jar_dependencies(binary_target):
self.context.log.debug(' scanning {} from {}'.format(coordinate, external_dep))
for qualified_file_name in ClasspathUtil.classpath_entries_contents([external_dep]):
artifacts_by_file_name[qualified_file_name].add(coordinate.artifact_filename)
return artifacts_by_file_name
def _get_external_dependencies(self, binary_target):
artifacts_by_file_name = defaultdict(set)
for external_dep, coordinate in self.list_external_jar_dependencies(binary_target):
self.context.log.debug(' scanning {} from {}'.format(coordinate, external_dep))
for qualified_file_name in ClasspathUtil.classpath_entries_contents([external_dep]):
artifacts_by_file_name[qualified_file_name].add(coordinate.artifact_filename)
return artifacts_by_file_name
def compile_classpath(self,
classpath_product_key,
target,
extra_cp_entries=None):
"""Compute the compile classpath for the given target."""
classpath_product = self._products.get_data(classpath_product_key)
if DependencyContext.global_instance().defaulted_property(
target, lambda x: x.strict_deps):
dependencies = target.strict_dependencies(
DependencyContext.global_instance())
else:
dependencies = DependencyContext.global_instance(
).all_dependencies(target)
all_extra_cp_entries = list(self._compiler_plugins_cp_entries())
if extra_cp_entries:
all_extra_cp_entries.extend(extra_cp_entries)
# TODO: We convert dependencies to an iterator here in order to _preserve_ a bug that will be
# fixed in https://github.com/pantsbuild/pants/issues/4874: `ClasspathUtil.compute_classpath`
# expects to receive a list, but had been receiving an iterator. In the context of an
# iterator, `excludes` are not applied
# in ClasspathProducts.get_product_target_mappings_for_targets.
return ClasspathUtil.compute_classpath(iter(dependencies),
classpath_product,
all_extra_cp_entries,
self.DEFAULT_CONFS)
def classpath(self, targets, classpath_prefix=None, classpath_product=None, exclude_scopes=None,
include_scopes=None):
"""Builds a transitive classpath for the given targets.
Optionally includes a classpath prefix or building from a non-default classpath product.
:param targets: the targets for which to build the transitive classpath.
:param classpath_prefix: optional additional entries to prepend to the classpath.
:param classpath_product: an optional ClasspathProduct from which to build the classpath. if not
specified, the runtime_classpath will be used.
:param :class:`pants.build_graph.target_scopes.Scope` exclude_scopes: Exclude targets which
have at least one of these scopes on the classpath.
:param :class:`pants.build_graph.target_scopes.Scope` include_scopes: Only include targets which
have at least one of these scopes on the classpath. Defaults to Scopes.JVM_RUNTIME_SCOPES.
:return: a list of classpath strings.
"""
include_scopes = Scopes.JVM_RUNTIME_SCOPES if include_scopes is None else include_scopes
classpath_product = classpath_product or self.context.products.get_data('runtime_classpath')
closure = BuildGraph.closure(targets, bfs=True, include_scopes=include_scopes,
exclude_scopes=exclude_scopes, respect_intransitive=True)
classpath_for_targets = ClasspathUtil.classpath(closure, classpath_product, self.confs)
classpath = list(classpath_prefix or ())
classpath.extend(classpath_for_targets)
return classpath
def test_classpath_by_targets(self):
b = self.make_target('b', JvmTarget)
a = self.make_target('a',
JvmTarget,
dependencies=[b],
excludes=[Exclude('com.example', 'lib')])
classpath_products = ClasspathProducts(self.pants_workdir)
path1 = self._path('jar/path1')
path2 = self._path('jar/path2')
path3 = os.path.join(self.pants_workdir, 'jar/path3')
resolved_jar = ResolvedJar(M2Coordinate(org='com.example',
name='lib',
rev='1.0'),
cache_path='somewhere',
pants_path=path3)
classpath_products.add_for_target(a, [('default', path1)])
classpath_products.add_for_target(a, [('non-default', path2)])
classpath_products.add_for_target(b, [('default', path2)])
classpath_products.add_jars_for_targets([b], 'default', [resolved_jar])
classpath_products.add_excludes_for_targets([a])
# (a, path2) filtered because of conf
# (b, path3) filtered because of excludes
self.assertEqual(
OrderedDict([(a, [ClasspathEntry(path1)]),
(b, [ClasspathEntry(path2)])]),
ClasspathUtil.classpath_by_targets(a.closure(bfs=True),
classpath_products))
def _test_canonical_classpath_helper(self, classpath_products, targets,
expected_canonical_classpath,
expected_classspath_files,
use_target_id):
"""
Helper method to call `create_canonical_classpath` and verify generated canonical classpath.
:param ClasspathProducts classpath_products: Classpath products.
:param list targets: List of targets to generate canonical classpath from.
:param list expected_canonical_classpath: List of canonical classpath relative to a base directory.
:param dict expected_classspath_files: A dict of classpath.txt path to its expected content.
"""
with temporary_dir() as base_dir:
canonical_classpath = ClasspathUtil.create_canonical_classpath(
classpath_products,
targets,
base_dir,
save_classpath_file=True,
use_target_id=use_target_id)
# check canonical path returned
self.assertEquals(expected_canonical_classpath,
relativize_paths(canonical_classpath, base_dir))
# check canonical path created contain the exact set of files, no more, no less
self.assertTrue(
contains_exact_files(
base_dir, expected_canonical_classpath +
expected_classspath_files.keys()))
# check the content of classpath.txt
for classpath_file in expected_classspath_files:
self.assertTrue(
check_file_content(
os.path.join(base_dir, classpath_file),
expected_classspath_files[classpath_file]))
def _compute_classpath_entries(self, classpath_products, compile_context,
extra_compile_time_classpath):
# Generate a classpath specific to this compile and target.
target = compile_context.target
if compile_context.strict_deps:
classpath_targets = list(self._compute_strict_dependencies(target))
pruned = [
t.address.spec
for t in target.closure(bfs=True,
**self._target_closure_kwargs)
if t not in classpath_targets
]
self.context.log.debug(
'Using strict classpath for {}, which prunes the following dependencies: {}'
.format(target.address.spec, pruned))
else:
classpath_targets = target.closure(bfs=True,
**self._target_closure_kwargs)
classpath_targets = (t for t in classpath_targets if t != target)
cp_entries = [compile_context.classes_dir]
cp_entries.extend(
ClasspathUtil.compute_classpath(classpath_targets,
classpath_products,
extra_compile_time_classpath,
self._confs))
if isinstance(target, JavacPlugin):
# Javac plugins need to compile against our distribution's tools.jar. There's no way to
# express this via traversable_dependency_specs, so we inject it into the classpath here.
cp_entries = self.dist.find_libs(['tools.jar']) + cp_entries
return cp_entries
def classpath(self, targets, cp=None):
classpath = list(cp) if cp else []
classpath_for_targets = ClasspathUtil.classpath_entries(
targets, self.context.products.get_data('compile_classpath'), self.confs)
classpath.extend(classpath_for_targets)
return classpath
def test_classpath_by_targets(self):
b = self.make_target('b', JvmTarget)
a = self.make_target('a', JvmTarget, dependencies=[b],
excludes=[Exclude('com.example', 'lib')])
classpath_products = ClasspathProducts(self.pants_workdir)
path1 = self._path('jar/path1')
path2 = self._path('jar/path2')
path3 = os.path.join(self.pants_workdir, 'jar/path3')
resolved_jar = ResolvedJar(M2Coordinate(org='com.example', name='lib', rev='1.0'),
cache_path='somewhere',
pants_path=path3)
classpath_products.add_for_target(a, [('default', path1)])
classpath_products.add_for_target(a, [('non-default', path2)])
classpath_products.add_for_target(b, [('default', path2)])
classpath_products.add_jars_for_targets([b], 'default', [resolved_jar])
classpath_products.add_excludes_for_targets([a])
# (a, path2) filtered because of conf
# (b, path3) filtered because of excludes
self.assertEquals(OrderedDict([(a, [ClasspathEntry(path1)]),
(b, [ClasspathEntry(path2)])]),
ClasspathUtil.classpath_by_targets(a.closure(bfs=True),
classpath_products))
def _compute_classpath(runtime_classpath, targets):
closure = BuildGraph.closure(
targets, bfs=True, include_scopes=Scopes.JVM_RUNTIME_SCOPES, respect_intransitive=True
)
classpath_for_targets = ClasspathUtil.classpath(closure, runtime_classpath)
return classpath_for_targets
def _test_canonical_classpath_helper(self,
classpath_products,
targets,
libs_dir,
expected_canonical_classpath,
expected_classspath_files,
excludes=None):
"""
Helper method to call `create_canonical_classpath` and verify generated canonical classpath.
:param ClasspathProducts classpath_products: Classpath products.
:param list targets: List of targets to generate canonical classpath from.
:param string libs_dir: Directory where canonical classpath are to be generated.
:param list expected_canonical_classpath: List of canonical classpath relative to a base directory.
:param dict expected_classspath_files: A dict of classpath.txt path to its expected content.
"""
canonical_classpath = ClasspathUtil.create_canonical_classpath(classpath_products,
targets,
libs_dir,
save_classpath_file=True,
internal_classpath_only=False,
excludes=excludes)
# check canonical path returned
self.assertEquals(expected_canonical_classpath,
relativize_paths(canonical_classpath, libs_dir))
# check canonical path created contain the exact set of files, no more, no less
self.assertTrue(contains_exact_files(libs_dir,
expected_canonical_classpath +
expected_classspath_files.keys()))
# check the content of classpath.txt
for classpath_file in expected_classspath_files:
self.assertTrue(check_file_content(os.path.join(libs_dir, classpath_file),
expected_classspath_files[classpath_file]))
def _compute_classpath_entries(self, compile_classpaths, target_closure,
compile_context,
extra_compile_time_classpath):
# Generate a classpath specific to this compile and target.
return ClasspathUtil.compute_classpath_for_target(
compile_context.target, compile_classpaths,
extra_compile_time_classpath, self._confs, target_closure)
def classpath(self,
targets,
classpath_prefix=None,
classpath_product=None):
"""Builds a transitive classpath for the given targets.
Optionally includes a classpath prefix or building from a non-default classpath product.
:param targets: the targets for which to build the transitive classpath.
:param classpath_prefix: optional additional entries to prepend to the classpath.
:param classpath_product: an optional ClasspathProduct from which to build the classpath. if not
specified, the runtime_classpath will be used.
:return: a list of classpath strings.
"""
classpath = list(classpath_prefix) if classpath_prefix else []
classpath_product = classpath_product or self.context.products.get_data(
'runtime_classpath')
closure = BuildGraph.closure(targets, bfs=True)
classpath_for_targets = ClasspathUtil.classpath(
closure, classpath_product, self.confs)
classpath.extend(classpath_for_targets)
return classpath
def _find_scalac_plugins(self, scalac_plugins, classpath):
"""Returns a map from plugin name to list of plugin classpath entries.
The first entry in each list is the classpath entry containing the plugin metadata.
The rest are the internal transitive deps of the plugin.
This allows us to have in-repo plugins with dependencies (unlike javac, scalac doesn't load
plugins or their deps from the regular classpath, so we have to provide these entries
separately, in the -Xplugin: flag).
Note that we don't currently support external plugins with dependencies, as we can't know which
external classpath elements are required, and we'd have to put the entire external classpath
on each -Xplugin: flag, which seems excessive.
Instead, external plugins should be published as "fat jars" (which appears to be the norm,
since SBT doesn't support plugins with dependencies anyway).
"""
# Allow multiple flags and also comma-separated values in a single flag.
plugin_names = set(
[p for val in scalac_plugins for p in val.split(',')])
if not plugin_names:
return {}
active_plugins = {}
buildroot = get_buildroot()
cp_product = self.context.products.get_data('runtime_classpath')
for classpath_element in classpath:
name = self._maybe_get_plugin_name(classpath_element)
if name in plugin_names:
plugin_target_closure = self._plugin_targets('scalac').get(
name, [])
# It's important to use relative paths, as the compiler flags get embedded in the zinc
# analysis file, and we port those between systems via the artifact cache.
rel_classpath_elements = [
os.path.relpath(cpe, buildroot)
for cpe in ClasspathUtil.internal_classpath(
plugin_target_closure, cp_product, self._confs)
]
# If the plugin is external then rel_classpath_elements will be empty, so we take
# just the external jar itself.
rel_classpath_elements = rel_classpath_elements or [
classpath_element
]
# Some classpath elements may be repeated, so we allow for that here.
if active_plugins.get(
name,
rel_classpath_elements) != rel_classpath_elements:
raise TaskError('Plugin {} defined in {} and in {}'.format(
name, active_plugins[name], classpath_element))
active_plugins[name] = rel_classpath_elements
if len(active_plugins) == len(plugin_names):
# We've found all the plugins, so return now to spare us from processing
# of the rest of the classpath for no reason.
return active_plugins
# If we get here we must have unresolved plugins.
unresolved_plugins = plugin_names - set(active_plugins.keys())
raise TaskError('Could not find requested plugins: {}'.format(
list(unresolved_plugins)))
def _calculate_tests_from_targets(self, targets):
"""
:param list targets: list of targets to calculate test classes for.
generates tuples (class_name, target).
"""
classpath_products = self._context.products.get_data(
'runtime_classpath')
for target in targets:
contents = ClasspathUtil.classpath_contents(
(target, ),
classpath_products,
confs=self._task_exports.confs,
transitive=False)
for f in contents:
classname = ClasspathUtil.classname_for_rel_classfile(f)
if classname:
yield (classname, target)
def classpath(self, targets, cp=None):
classpath = list(cp) if cp else []
classpath_for_targets = ClasspathUtil.classpath_entries(
targets, self.context.products.get_data('compile_classpath'),
self.confs)
classpath.extend(classpath_for_targets)
return classpath
def test_single_classpath_element_no_excludes(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
path = os.path.join(self.build_root, 'jar/path')
classpath_product.add_for_target(a, [('default', path)])
classpath = ClasspathUtil.compute_classpath([a], classpath_product, [], ['default'])
self.assertEqual([path], classpath)
def work_for_vts(vts, ctx):
progress_message = ctx.target.address.spec
# Capture a compilation log if requested.
log_file = ctx.log_file if self._capture_log else None
# Double check the cache before beginning compilation
hit_cache = check_cache(vts)
if not hit_cache:
# Compute the compile classpath for this target.
cp_entries = [compile_context.classes_dir]
cp_entries.extend(ClasspathUtil.compute_classpath(ctx.dependencies(self._dep_context),
classpath_products,
extra_compile_time_classpath,
self._confs))
# TODO: always provide transitive analysis, but not always all classpath entries?
upstream_analysis = dict(self._upstream_analysis(compile_contexts, cp_entries))
# Write analysis to a temporary file, and move it to the final location on success.
tmp_analysis_file = "{}.tmp".format(ctx.analysis_file)
if should_compile_incrementally(vts):
# If this is an incremental compile, rebase the analysis to our new classes directory.
self._analysis_tools.rebase_from_path(ctx.analysis_file,
tmp_analysis_file,
vts.previous_results_dir,
vts.results_dir)
else:
# Otherwise, simply ensure that it is empty.
safe_delete(tmp_analysis_file)
tgt, = vts.targets
fatal_warnings = self._compute_language_property(tgt, lambda x: x.fatal_warnings)
self._compile_vts(vts,
ctx.sources,
tmp_analysis_file,
upstream_analysis,
cp_entries,
ctx.classes_dir,
log_file,
progress_message,
tgt.platform,
fatal_warnings,
counter)
os.rename(tmp_analysis_file, ctx.analysis_file)
self._analysis_tools.relativize(ctx.analysis_file, ctx.portable_analysis_file)
# Write any additional resources for this target to the target workdir.
self.write_extra_resources(ctx)
# Jar the compiled output.
self._create_context_jar(ctx)
# Update the products with the latest classes.
self._register_vts([ctx])
# Once products are registered, check for unused dependencies (if enabled).
if not hit_cache and self._unused_deps_check_enabled:
self._check_unused_deps(ctx)
def work_for_vts(vts, ctx):
progress_message = ctx.target.address.spec
# Capture a compilation log if requested.
log_file = ctx.log_file if self._capture_log else None
# Double check the cache before beginning compilation
hit_cache = check_cache(vts)
if not hit_cache:
# Compute the compile classpath for this target.
cp_entries = [ctx.classes_dir]
cp_entries.extend(
ClasspathUtil.compute_classpath(
ctx.dependencies(self._dep_context),
classpath_products,
extra_compile_time_classpath,
self._confs,
)
)
upstream_analysis = dict(self._upstream_analysis(compile_contexts, cp_entries))
if not should_compile_incrementally(vts, ctx):
# Purge existing analysis file in non-incremental mode.
safe_delete(ctx.analysis_file)
# Work around https://github.com/pantsbuild/pants/issues/3670
safe_rmtree(ctx.classes_dir)
tgt, = vts.targets
fatal_warnings = self._compute_language_property(tgt, lambda x: x.fatal_warnings)
self._compile_vts(
vts,
ctx.sources,
ctx.analysis_file,
upstream_analysis,
cp_entries,
ctx.classes_dir,
log_file,
progress_message,
tgt.platform,
fatal_warnings,
counter,
)
self._analysis_tools.relativize(ctx.analysis_file, ctx.portable_analysis_file)
# Write any additional resources for this target to the target workdir.
self.write_extra_resources(ctx)
# Jar the compiled output.
self._create_context_jar(ctx)
# Update the products with the latest classes.
self._register_vts([ctx])
# Once products are registered, check for unused dependencies (if enabled).
if not hit_cache and self._unused_deps_check_enabled:
self._check_unused_deps(ctx)
def test_relies_on_product_to_validate_paths_outside_buildroot(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
classpath_product.add_for_target(a, [('default', '/dev/null')])
classpath = ClasspathUtil.compute_classpath([a], classpath_product, [], ['default'])
self.assertEqual(['/dev/null'], classpath)
def add_sources_under_test(tgt):
if self.is_coverage_target(tgt):
contents = ClasspathUtil.classpath_contents(
(tgt,), classpath_products, confs=self._task_exports.confs, transitive=False
)
for f in contents:
clsname = _classfile_to_classname(f)
if clsname:
classes_under_test.add(clsname)
def test_single_classpath_element_no_excludes(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
path = os.path.join(self.build_root, 'jar/path')
classpath_product.add_for_target(a, [('default', path)])
classpath = ClasspathUtil.compute_classpath([a], classpath_product, [],
['default'])
self.assertEqual([path], classpath)
def bundle(self, app, results_dir):
"""Create a self-contained application bundle.
The bundle will contain the target classes, dependencies and resources.
"""
assert(isinstance(app, BundleCreate.App))
bundle_dir = self._get_bundle_dir(app, results_dir)
self.context.log.debug('creating {}'.format(os.path.relpath(bundle_dir, get_buildroot())))
safe_mkdir(bundle_dir, clean=True)
classpath = OrderedSet()
# Create symlinks for both internal and external dependencies under `lib_dir`. This is
# only needed when not creating a deployjar
lib_dir = os.path.join(bundle_dir, self.LIBS_DIR)
if not app.deployjar:
os.mkdir(lib_dir)
consolidated_classpath = self.context.products.get_data('consolidated_classpath')
classpath.update(ClasspathUtil.create_canonical_classpath(
consolidated_classpath,
app.target.closure(bfs=True, **self._target_closure_kwargs),
lib_dir,
internal_classpath_only=False,
excludes=app.binary.deploy_excludes,
))
bundle_jar = os.path.join(bundle_dir, '{}.jar'.format(app.binary.basename))
with self.monolithic_jar(app.binary, bundle_jar,
manifest_classpath=classpath) as jar:
self.add_main_manifest_entry(jar, app.binary)
# Make classpath complete by adding the monolithic jar.
classpath.update([jar.path])
if app.binary.shading_rules:
for jar_path in classpath:
# In case `jar_path` is a symlink, this is still safe, shaded jar will overwrite jar_path,
# original file `jar_path` linked to remains untouched.
# TODO run in parallel to speed up
self.shade_jar(shading_rules=app.binary.shading_rules, jar_path=jar_path)
for bundle_counter, bundle in enumerate(app.bundles):
fileset_empty = True
for path, relpath in bundle.filemap.items():
bundle_path = os.path.join(bundle_dir, relpath)
if os.path.exists(path):
safe_mkdir(os.path.dirname(bundle_path))
os.symlink(path, bundle_path)
fileset_empty = False
if fileset_empty:
raise TaskError('In target {}, bundle index {} of "bundles" field does not match any files.'.format(
app.address.spec, bundle_counter))
return bundle_dir
def gen():
# Compute src -> target.
if isinstance(target, JvmTarget):
for src in target.sources_relative_to_buildroot():
yield os.path.join(self.buildroot, src)
# TODO(Tejal Desai): pantsbuild/pants/65: Remove java_sources attribute for ScalaLibrary
if isinstance(target, ScalaLibrary):
for java_source in target.java_sources:
for src in java_source.sources_relative_to_buildroot():
yield os.path.join(self.buildroot, src)
# Compute classfile -> target and jar -> target.
files = ClasspathUtil.classpath_contents((target,), self.runtime_classpath)
# And jars; for binary deps, zinc doesn't emit precise deps (yet).
cp_entries = ClasspathUtil.classpath((target,), self.runtime_classpath)
jars = [cpe for cpe in cp_entries if ClasspathUtil.is_jar(cpe)]
for coll in [files, jars]:
for f in coll:
yield f
def find_consolidate_classpath_candidates(self, classpath_products, targets):
targets_with_directory_in_classpath = []
for target in targets:
entries = classpath_products.get_internal_classpath_entries_for_targets([target])
for conf, entry in entries:
if ClasspathUtil.is_dir(entry.path):
targets_with_directory_in_classpath.append(target)
break
return targets_with_directory_in_classpath
def _classnames_from_source_file(self, context):
source_products = context.products.get_data('classes_by_source').get(self._sourcefile)
if not source_products:
# It's valid - if questionable - to have a source file with no classes when, for
# example, the source file has all its code commented out.
context.log.warn('Source file {0} generated no classes'.format(self._sourcefile))
else:
for _, classes in source_products.rel_paths():
for cls in classes:
yield ClasspathUtil.classname_for_rel_classfile(cls)
def test_excluded_classpath_element(self):
a = self.make_target('a', JvmTarget, excludes=[Exclude('com.example', 'lib')])
classpath_product = UnionProducts()
example_jar_path = os.path.join(self.build_root, 'ivy/jars/com.example/lib/123.4.jar')
classpath_product.add_for_target(a, [('default', example_jar_path)])
classpath = ClasspathUtil.compute_classpath([a], classpath_product, [], ['default'])
self.assertEqual([], classpath)
def _iter_jar_packages(cls, path):
paths = set()
for pathname in ClasspathUtil.classpath_entries_contents([path]):
if cls._potential_package_path(pathname):
package = os.path.dirname(pathname)
if package:
# This check avoids a false positive on things like module-info.class.
# We must never add an empty package, as this will cause every single string
# literal to be rewritten.
paths.add(package)
return cls._iter_packages(paths)
