def execute(self):
requested_compiler = JvmPlatform.global_instance().get_options(
).compiler
if requested_compiler != self.compiler_name:
return
if requested_compiler == self.Compiler.ZINC and self.compiler_name == self.Compiler.RSC:
# Issue a deprecation warning (above) and rewrite zinc to rsc, as zinc is being deprecated.
JvmPlatform.global_instance().get_options().compiler = RankedValue(
0, self.compiler_name)
elif requested_compiler != self.compiler_name:
# If the requested compiler is not the one supported by this task, log and abort
self.context.log.debug(
'Requested an unsupported compiler [{}], aborting'.format(
requested_compiler))
return
# In case we have no relevant targets and return early, create the requested product maps.
self.create_empty_extra_products()
relevant_targets = list(self.context.targets(predicate=self.select))
if not relevant_targets:
return
# Clone the compile_classpath to the runtime_classpath.
classpath_product = self.create_runtime_classpath()
fingerprint_strategy = DependencyContext.global_instance(
).create_fingerprint_strategy(classpath_product)
# Note, JVM targets are validated (`vts.update()`) as they succeed. As a result,
# we begin writing artifacts out to the cache immediately instead of waiting for
# all targets to finish.
with self.invalidated(relevant_targets,
invalidate_dependents=True,
fingerprint_strategy=fingerprint_strategy,
topological_order=True) as invalidation_check:
compile_contexts = {
vt.target: self.create_compile_context(vt.target,
vt.results_dir)
for vt in invalidation_check.all_vts
}
self.do_compile(
invalidation_check,
compile_contexts,
classpath_product,
)
if not self.get_options().use_classpath_jars:
# Once compilation has completed, replace the classpath entry for each target with
# its jar'd representation.
for ccs in compile_contexts.values():
cc = self.select_runtime_context(ccs)
for conf in self._confs:
classpath_product.remove_for_target(
cc.target, [(conf, cc.classes_dir)])
classpath_product.add_for_target(
cc.target, [(conf, cc.jar_file)])
def initialize_graph_info(self):
scala_platform = ScalaPlatform.global_instance()
scala_platform_map = {
"scala_version":
scala_platform.version,
"compiler_classpath": [
cp_entry.path
for cp_entry in scala_platform.compiler_classpath_entries(
self.context.products)
],
}
jvm_platforms_map = {
"default_platform":
JvmPlatform.global_instance().default_platform.name,
"platforms": {
str(platform_name): {
"target_level": str(platform.target_level),
"source_level": str(platform.source_level),
"args": platform.args,
}
for platform_name, platform in
JvmPlatform.global_instance().platforms_by_name.items()
},
}
graph_info = {
"version": DEFAULT_EXPORT_VERSION,
"targets": {},
"jvm_platforms": jvm_platforms_map,
"scala_platform": scala_platform_map,
# `jvm_distributions` are static distribution settings from config,
# `preferred_jvm_distributions` are distributions that pants actually uses for the
# given platform setting.
"preferred_jvm_distributions": {},
}
for platform_name, platform in JvmPlatform.global_instance(
).platforms_by_name.items():
preferred_distributions = {}
for strict, strict_key in [(True, "strict"),
(False, "non_strict")]:
try:
dist = JvmPlatform.preferred_jvm_distribution(
[platform], strict=strict)
preferred_distributions[strict_key] = dist.home
except DistributionLocator.Error:
pass
if preferred_distributions:
graph_info["preferred_jvm_distributions"][
platform_name] = preferred_distributions
return graph_info
def initialize_graph_info(self):
scala_platform = ScalaPlatform.global_instance()
scala_platform_map = {
'scala_version':
scala_platform.version,
'compiler_classpath': [
cp_entry.path
for cp_entry in scala_platform.compiler_classpath_entries(
self.context.products)
],
}
jvm_platforms_map = {
'default_platform':
JvmPlatform.global_instance().default_platform.name,
'platforms': {
str(platform_name): {
'target_level': str(platform.target_level),
'source_level': str(platform.source_level),
'args': platform.args,
}
for platform_name, platform in
JvmPlatform.global_instance().platforms_by_name.items()
},
}
graph_info = {
'version': DEFAULT_EXPORT_VERSION,
'targets': {},
'jvm_platforms': jvm_platforms_map,
'scala_platform': scala_platform_map,
# `jvm_distributions` are static distribution settings from config,
# `preferred_jvm_distributions` are distributions that pants actually uses for the
# given platform setting.
'preferred_jvm_distributions': {}
}
for platform_name, platform in JvmPlatform.global_instance(
).platforms_by_name.items():
preferred_distributions = {}
for strict, strict_key in [(True, 'strict'),
(False, 'non_strict')]:
try:
dist = JvmPlatform.preferred_jvm_distribution(
[platform], strict=strict)
preferred_distributions[strict_key] = dist.home
except DistributionLocator.Error:
pass
if preferred_distributions:
graph_info['preferred_jvm_distributions'][
platform_name] = preferred_distributions
return graph_info
def test_jvm_options(self):
init_subsystem(
JvmPlatform,
options={
"jvm-platform": {
"platforms": {
"platform-with-jvm-options": {
"target": "8",
"jvm_options": ["-Dsomething"],
},
"platform-without-jvm-options": {
"target": "8"
},
},
}
},
)
instance = JvmPlatform.global_instance()
with_options = instance.get_platform_by_name(
"platform-with-jvm-options")
without_options = instance.get_platform_by_name(
"platform-without-jvm-options")
assert ("-Dsomething", ) == with_options.jvm_options
assert tuple() == without_options.jvm_options
def platform(self):
"""Platform associated with this target.
:return: The jvm platform object.
:rtype: JvmPlatformSettings
"""
return JvmPlatform.global_instance().get_platform_for_target(self)
def test_runtime_lookup_no_default_runtime_platform(self):
init_subsystem(
JvmPlatform,
options={
"jvm-platform": {
"platforms": {
"default-platform": {
"target": "8"
},
"default-runtime-platform": {
"target": "8"
},
"target-platform": {
"target": "8"
},
"target-runtime-platform": {
"target": "8"
},
},
"default_platform": "default-platform",
"default_runtime_platform": None,
}
},
)
without_platforms = self.make_target("//:without-platforms",
HasRuntimePlatform)
just_platform = self.make_target("//:with-platform",
HasRuntimePlatform,
platform="target-platform")
just_runtime_platform = self.make_target(
"//:with-runtime-platform",
HasRuntimePlatform,
runtime_platform="target-runtime-platform",
)
both_platforms = self.make_target(
"//:with-platform-and-runtime-platform",
HasRuntimePlatform,
platform="target-platform",
runtime_platform="target-runtime-platform",
)
instance = JvmPlatform.global_instance()
assert (instance.get_runtime_platform_for_target(
without_platforms).name == "default-platform")
assert instance.get_runtime_platform_for_target(
just_platform).name == "default-platform"
assert (instance.get_runtime_platform_for_target(
just_runtime_platform).name == "target-runtime-platform")
assert (instance.get_runtime_platform_for_target(both_platforms).name
== "target-runtime-platform")
def test_jvm_options_from_platform_shlexed(self):
init_subsystem(
JvmPlatform,
options={
"jvm-platform": {
"platforms": {
"platform-with-shlexable-vm-options": {
"target": "8",
"jvm_options": ["-Dsomething -Dsomethingelse"],
},
},
}
},
)
instance = JvmPlatform.global_instance()
need_shlex_options = instance.get_platform_by_name("platform-with-shlexable-vm-options")
assert ("-Dsomething", "-Dsomethingelse") == need_shlex_options.jvm_options
def execute(self):
if JvmPlatform.global_instance().get_options().compiler != self.compiler_name:
# If the requested compiler is not the one supported by this task,
# bail early.
return
# In case we have no relevant targets and return early, create the requested product maps.
self.create_empty_extra_products()
relevant_targets = list(self.context.targets(predicate=self.select))
if not relevant_targets:
return
# Clone the compile_classpath to the runtime_classpath.
classpath_product = self.create_runtime_classpath()
fingerprint_strategy = DependencyContext.global_instance().create_fingerprint_strategy(
classpath_product)
# Note, JVM targets are validated (`vts.update()`) as they succeed. As a result,
# we begin writing artifacts out to the cache immediately instead of waiting for
# all targets to finish.
with self.invalidated(relevant_targets,
invalidate_dependents=True,
fingerprint_strategy=fingerprint_strategy,
topological_order=True) as invalidation_check:
compile_contexts = {vt.target: self.create_compile_context(vt.target, vt.results_dir)
for vt in invalidation_check.all_vts}
self.do_compile(
invalidation_check,
compile_contexts,
classpath_product,
)
if not self.get_options().use_classpath_jars:
# Once compilation has completed, replace the classpath entry for each target with
# its jar'd representation.
for ccs in compile_contexts.values():
cc = self.select_runtime_context(ccs)
for conf in self._confs:
classpath_product.remove_for_target(cc.target, [(conf, cc.classes_dir.path)])
classpath_product.add_for_target(cc.target, [(conf, cc.jar_file.path)])
def test_distribution_from_targets_passes_through_platforms(self):
fake_dist = "a dist"
java8_platform = JvmPlatformSettings('8', '8', [])
targets = [
self.make_target('platformed_target', JvmTarget, platform='java8')
]
with unittest.mock.patch.object(
JvmPlatform, 'preferred_jvm_distribution') as plat_mock:
with unittest.mock.patch.object(JvmPlatform.global_instance(), 'get_platform_for_target') as \
target_plat_mock:
target_plat_mock.return_value = java8_platform
plat_mock.return_value = fake_dist
dist = self.task.preferred_jvm_distribution_for_targets(
targets)
plat_mock.assert_called_once_with([java8_platform],
strict=False,
jdk=False)
self.assertEqual(fake_dist, dist)
def test_distribution_from_targets_passes_through_platforms(self):
fake_dist = "a dist"
java8_platform = self.java8_platform()
targets = [
self.make_target("platformed_target", JvmTarget, platform="java8")
]
with unittest.mock.patch.object(
JvmPlatform, "preferred_jvm_distribution") as plat_mock:
with unittest.mock.patch.object(
JvmPlatform.global_instance(),
"get_platform_for_target") as target_plat_mock:
target_plat_mock.return_value = java8_platform
plat_mock.return_value = fake_dist
dist = self.task.preferred_jvm_distribution_for_targets(
targets)
plat_mock.assert_called_once_with([java8_platform],
strict=None,
jdk=False)
self.assertEqual(fake_dist, dist)
def test_strict_usage(self):
init_subsystem(
JvmPlatform,
options={
"jvm-platform": {
"platforms": {
"default-platform": {"target": "9"},
"8-platform": {"target": "8"},
"9-platform": {"target": "9"},
"strict-8-platform": {"target": "8", "strict": True},
"strict-9-platform": {"target": "9", "strict": True},
},
"default_platform": "default-platform",
"default_runtime_platform": None,
}
},
)
instance = JvmPlatform.global_instance()
strict_8_platform = instance.get_platform_by_name("strict-8-platform")
default_9_platform = instance.default_platform
# TODO maybe this should use the runtime platform
assert instance._preferred_jvm_distribution_args([]) == {
"jdk": False,
}
assert JvmPlatform._preferred_jvm_distribution_args([default_9_platform]) == {
"minimum_version": Revision.lenient("9.0.0"),
"maximum_version": None,
"jdk": False,
}
assert JvmPlatform._preferred_jvm_distribution_args([default_9_platform], strict=True) == {
"minimum_version": Revision.lenient("9.0.0"),
"maximum_version": Revision.lenient("9.0.9999"),
"jdk": False,
}
assert instance._preferred_jvm_distribution_args([strict_8_platform]) == {
"minimum_version": Revision.lenient("1.8.0"),
"maximum_version": Revision.lenient("1.8.9999"),
"jdk": False,
}
assert instance._preferred_jvm_distribution_args([strict_8_platform], strict=False) == {
"minimum_version": Revision.lenient("1.8.0"),
"maximum_version": None,
"jdk": False,
}
with self.assertRaisesRegex(
JvmPlatform.IncompatiblePlatforms,
"lenient platform with higher minimum version, 9, than strict requirement of 1.8",
):
# requested strict 8 & lenient 9.
# fail because 9 is lower bound
JvmPlatform._preferred_jvm_distribution_args(
[
instance.get_platform_by_name("9-platform"),
instance.get_platform_by_name("strict-8-platform"),
]
)
with self.assertRaisesRegex(
JvmPlatform.IncompatiblePlatforms,
"Multiple strict platforms with differing target releases were found: 1.8, 9",
):
# two different strict platforms can't work
JvmPlatform._preferred_jvm_distribution_args(
[
instance.get_platform_by_name("strict-9-platform"),
instance.get_platform_by_name("strict-8-platform"),
]
)
# two of the same strict platform thumbs up
assert JvmPlatform._preferred_jvm_distribution_args(
[
instance.get_platform_by_name("strict-8-platform"),
instance.get_platform_by_name("strict-8-platform"),
]
) == {
"minimum_version": Revision.lenient("1.8.0"),
"maximum_version": Revision.lenient("1.8.9999"),
"jdk": False,
}
# strict highest, matching highest non-strict, other non-strict
assert JvmPlatform._preferred_jvm_distribution_args(
[
instance.get_platform_by_name("strict-9-platform"),
instance.get_platform_by_name("9-platform"),
instance.get_platform_by_name("8-platform"),
]
) == {
"minimum_version": Revision.lenient("9.0.0"),
"maximum_version": Revision.lenient("9.0.9999"),
"jdk": False,
}
def test_synthetic_target_runtime_platform_lookup(self):
init_subsystem(
JvmPlatform,
options={
"jvm-platform": {
"platforms": {
"default-platform": {"target": "8"},
"default-runtime-platform": {"target": "8"},
"target-platform": {"target": "8"},
"target-runtime-platform": {"target": "8"},
"parent-target-platform": {"target": "8"},
"parent-target-runtime-platform": {"target": "8"},
},
"default_platform": "default-platform",
"default_runtime_platform": None,
}
},
)
just_platform = self.make_target(
"//:parent-with-runtime-platform", HasRuntimePlatform, platform="parent-target-platform"
)
just_runtime_platform = self.make_target(
"//:parent-with-platform",
HasRuntimePlatform,
runtime_platform="parent-target-runtime-platform",
)
synth_none = self.make_target(
"//:without-platforms",
HasRuntimePlatform,
synthetic=True,
derived_from=just_runtime_platform,
)
synth_just_platform = self.make_target(
"//:with-platform",
HasRuntimePlatform,
synthetic=True,
derived_from=just_runtime_platform,
platform="target-platform",
)
synth_just_runtime = self.make_target(
"//:with-runtime-platform",
HasRuntimePlatform,
synthetic=True,
derived_from=just_runtime_platform,
runtime_platform="target-runtime-platform",
)
synth_both = self.make_target(
"//:with-platform-and-runtime-platform",
HasRuntimePlatform,
synthetic=True,
derived_from=just_runtime_platform,
platform="target-platform",
runtime_platform="target-runtime-platform",
)
synth_just_platform_with_parent_same = self.make_target(
"//:with-platform-and-platform-parent",
HasRuntimePlatform,
synthetic=True,
derived_from=just_platform,
platform="target-platform",
)
instance = JvmPlatform.global_instance()
assert (
instance.get_runtime_platform_for_target(synth_none).name
== "parent-target-runtime-platform"
)
assert (
instance.get_runtime_platform_for_target(synth_just_platform).name
== "parent-target-runtime-platform"
)
assert (
instance.get_runtime_platform_for_target(synth_just_runtime).name
== "target-runtime-platform"
)
assert (
instance.get_runtime_platform_for_target(synth_both).name == "target-runtime-platform"
)
assert (
instance.get_runtime_platform_for_target(synth_just_platform_with_parent_same).name
== "default-platform"
)
def platform(self):
return JvmPlatform.global_instance().get_platform_for_target(self)
def console_output(self, targets):
targets_map = {}
resource_target_map = {}
classpath_products = (
self.context.products.get_data("compile_classpath") if self.get_options().libraries else None
)
python_interpreter_targets_mapping = defaultdict(list)
def process_target(current_target):
"""
:type current_target:pants.build_graph.target.Target
"""
def get_target_type(target):
if target.is_test:
return Export.SourceRootTypes.TEST
else:
if (
isinstance(target, Resources)
and target in resource_target_map
and resource_target_map[target].is_test
):
return Export.SourceRootTypes.TEST_RESOURCE
elif isinstance(target, Resources):
return Export.SourceRootTypes.RESOURCE
else:
return Export.SourceRootTypes.SOURCE
info = {
"targets": [],
"libraries": [],
"roots": [],
"target_type": get_target_type(current_target),
"is_code_gen": current_target.is_codegen,
"pants_target_type": self._get_pants_target_alias(type(current_target)),
}
if not current_target.is_synthetic:
info["globs"] = current_target.globs_relative_to_buildroot()
if self.get_options().sources:
info["sources"] = list(current_target.sources_relative_to_buildroot())
if isinstance(current_target, PythonRequirementLibrary):
reqs = current_target.payload.get_field_value("requirements", set())
""":type : set[pants.backend.python.python_requirement.PythonRequirement]"""
info["requirements"] = [req.key for req in reqs]
if isinstance(current_target, PythonTarget):
interpreter_for_target = self.select_interpreter_for_targets([current_target])
if interpreter_for_target is None:
raise TaskError("Unable to find suitable interpreter for {}".format(current_target.address))
python_interpreter_targets_mapping[interpreter_for_target].append(current_target)
info["python_interpreter"] = str(interpreter_for_target.identity)
def iter_transitive_jars(jar_lib):
"""
:type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary`
:rtype: :class:`collections.Iterator` of
:class:`pants.backend.jvm.jar_dependency_utils.M2Coordinate`
"""
if classpath_products:
jar_products = classpath_products.get_artifact_classpath_entries_for_targets((jar_lib,))
for _, jar_entry in jar_products:
coordinate = jar_entry.coordinate
# We drop classifier and type_ since those fields are represented in the global
# libraries dict and here we just want the key into that dict (see `_jar_id`).
yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev)
target_libraries = OrderedSet()
if isinstance(current_target, JarLibrary):
target_libraries = OrderedSet(iter_transitive_jars(current_target))
for dep in current_target.dependencies:
info["targets"].append(dep.address.spec)
if isinstance(dep, JarLibrary):
for jar in dep.jar_dependencies:
target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev))
# Add all the jars pulled in by this jar_library
target_libraries.update(iter_transitive_jars(dep))
if isinstance(dep, Resources):
resource_target_map[dep] = current_target
if isinstance(current_target, ScalaLibrary):
for dep in current_target.java_sources:
info["targets"].append(dep.address.spec)
process_target(dep)
if isinstance(current_target, JvmTarget):
info["excludes"] = [self._exclude_id(exclude) for exclude in current_target.excludes]
info["platform"] = current_target.platform.name
info["roots"] = map(
lambda (source_root, package_prefix): {"source_root": source_root, "package_prefix": package_prefix},
self._source_roots_for_target(current_target),
)
if classpath_products:
info["libraries"] = [self._jar_id(lib) for lib in target_libraries]
targets_map[current_target.address.spec] = info
for target in targets:
process_target(target)
jvm_platforms_map = {
"default_platform": JvmPlatform.global_instance().default_platform.name,
"platforms": {
str(platform_name): {
"target_level": str(platform.target_level),
"source_level": str(platform.source_level),
"args": platform.args,
}
for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items()
},
}
graph_info = {
"version": self.DEFAULT_EXPORT_VERSION,
"targets": targets_map,
"jvm_platforms": jvm_platforms_map,
}
jvm_distributions = DistributionLocator.global_instance().all_jdk_paths()
if jvm_distributions:
graph_info["jvm_distributions"] = jvm_distributions
if classpath_products:
graph_info["libraries"] = self._resolve_jars_info(targets, classpath_products)
if python_interpreter_targets_mapping:
interpreters = self.interpreter_cache.select_interpreter(python_interpreter_targets_mapping.keys())
default_interpreter = interpreters[0]
interpreters_info = {}
for interpreter, targets in six.iteritems(python_interpreter_targets_mapping):
chroot = self.cached_chroot(interpreter=interpreter, pex_info=PexInfo.default(), targets=targets)
interpreters_info[str(interpreter.identity)] = {"binary": interpreter.binary, "chroot": chroot.path()}
graph_info["python_setup"] = {
"default_interpreter": str(default_interpreter.identity),
"interpreters": interpreters_info,
}
if self.format:
return json.dumps(graph_info, indent=4, separators=(",", ": ")).splitlines()
else:
return [json.dumps(graph_info)]
def console_output(self, targets):
targets_map = {}
resource_target_map = {}
ivy_info = None
if self.get_options().libraries:
ivy_jar_products = self.context.products.get_data("ivy_jar_products") or {}
# This product is a list for historical reasons (exclusives groups) but in practice should
# have either 0 or 1 entries.
ivy_info_list = ivy_jar_products.get("default")
if ivy_info_list:
assert len(ivy_info_list) == 1, (
"The values in ivy_jar_products should always be length 1,"
" since we no longer have exclusives groups."
)
ivy_info = ivy_info_list[0]
ivy_jar_memo = {}
python_interpreter_targets_mapping = defaultdict(list)
def process_target(current_target):
"""
:type current_target:pants.base.target.Target
"""
def get_target_type(target):
if target.is_test:
return Export.SourceRootTypes.TEST
else:
if (
isinstance(target, Resources)
and target in resource_target_map
and resource_target_map[target].is_test
):
return Export.SourceRootTypes.TEST_RESOURCE
elif isinstance(target, Resources):
return Export.SourceRootTypes.RESOURCE
else:
return Export.SourceRootTypes.SOURCE
def get_transitive_jars(jar_lib):
"""
:type jar_lib: pants.backend.jvm.targets.jar_library.JarLibrary
:rtype: twitter.common.collections.orderedset.OrderedSet
"""
if not ivy_info or not self.get_options().libraries:
return OrderedSet()
transitive_jars = OrderedSet()
for jar in jar_lib.jar_dependencies:
transitive_jars.update(ivy_info.get_jars_for_ivy_module(jar, memo=ivy_jar_memo))
return transitive_jars
info = {
"targets": [],
"libraries": [],
"roots": [],
"target_type": get_target_type(current_target),
"is_code_gen": current_target.is_codegen,
"pants_target_type": self._get_pants_target_alias(type(current_target)),
}
if not current_target.is_synthetic:
info["globs"] = current_target.globs_relative_to_buildroot()
if self.get_options().sources:
info["sources"] = list(current_target.sources_relative_to_buildroot())
if isinstance(current_target, PythonRequirementLibrary):
reqs = current_target.payload.get_field_value("requirements", set())
""":type : set[pants.backend.python.python_requirement.PythonRequirement]"""
info["requirements"] = [req.key for req in reqs]
if isinstance(current_target, PythonTarget):
interpreter_for_target = self.select_interpreter_for_targets([current_target])
if interpreter_for_target is None:
raise TaskError("Unable to find suitable interpreter for {}".format(current_target.address))
python_interpreter_targets_mapping[interpreter_for_target].append(current_target)
info["python_interpreter"] = str(interpreter_for_target.identity)
target_libraries = OrderedSet()
if isinstance(current_target, JarLibrary):
target_libraries = get_transitive_jars(current_target)
for dep in current_target.dependencies:
info["targets"].append(dep.address.spec)
if isinstance(dep, JarLibrary):
for jar in dep.jar_dependencies:
target_libraries.add(IvyModuleRef(jar.org, jar.name, jar.rev))
# Add all the jars pulled in by this jar_library
target_libraries.update(get_transitive_jars(dep))
if isinstance(dep, Resources):
resource_target_map[dep] = current_target
if isinstance(current_target, ScalaLibrary):
for dep in current_target.java_sources:
info["targets"].append(dep.address.spec)
process_target(dep)
if isinstance(current_target, JvmTarget):
info["excludes"] = [self._exclude_id(exclude) for exclude in current_target.excludes]
info["platform"] = current_target.platform.name
info["roots"] = map(
lambda (source_root, package_prefix): {"source_root": source_root, "package_prefix": package_prefix},
self._source_roots_for_target(current_target),
)
if self.get_options().libraries:
info["libraries"] = [self._jar_id(lib) for lib in target_libraries]
targets_map[current_target.address.spec] = info
for target in targets:
process_target(target)
jvm_platforms_map = {
"default_platform": JvmPlatform.global_instance().default_platform.name,
"platforms": {
str(platform_name): {
"target_level": str(platform.target_level),
"source_level": str(platform.source_level),
"args": platform.args,
}
for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items()
},
}
graph_info = {
"version": self.DEFAULT_EXPORT_VERSION,
"targets": targets_map,
"jvm_platforms": jvm_platforms_map,
}
jvm_distributions = DistributionLocator.global_instance().all_jdk_paths()
if jvm_distributions:
graph_info["jvm_distributions"] = jvm_distributions
if self.get_options().libraries:
graph_info["libraries"] = self._resolve_jars_info()
if python_interpreter_targets_mapping:
default_interpreter = self.interpreter_cache.select_interpreter(python_interpreter_targets_mapping.keys())[
0
]
interpreters_info = {}
for interpreter, targets in python_interpreter_targets_mapping.iteritems():
chroot = self.cached_chroot(interpreter=interpreter, pex_info=PexInfo.default(), targets=targets)
interpreters_info[str(interpreter.identity)] = {"binary": interpreter.binary, "chroot": chroot.path()}
graph_info["python_setup"] = {
"default_interpreter": str(default_interpreter.identity),
"interpreters": interpreters_info,
}
if self.format:
return json.dumps(graph_info, indent=4, separators=(",", ": ")).splitlines()
else:
return [json.dumps(graph_info)]
def generate_targets_map(self, targets, classpath_products=None):
"""Generates a dictionary containing all pertinent information about the target graph.
The return dictionary is suitable for serialization by json.dumps.
:param targets: The list of targets to generate the map for.
:param classpath_products: Optional classpath_products. If not provided when the --libraries
option is `True`, this task will perform its own jar resolution.
"""
targets_map = {}
resource_target_map = {}
python_interpreter_targets_mapping = defaultdict(list)
if self.get_options().libraries:
# NB(gmalmquist): This supports mocking the classpath_products in tests.
if classpath_products is None:
classpath_products = self.resolve_jars(targets)
else:
classpath_products = None
def process_target(current_target):
"""
:type current_target:pants.build_graph.target.Target
"""
def get_target_type(target):
if target.is_test:
return ExportTask.SourceRootTypes.TEST
else:
if (isinstance(target, Resources) and
target in resource_target_map and
resource_target_map[target].is_test):
return ExportTask.SourceRootTypes.TEST_RESOURCE
elif isinstance(target, Resources):
return ExportTask.SourceRootTypes.RESOURCE
else:
return ExportTask.SourceRootTypes.SOURCE
info = {
'targets': [],
'libraries': [],
'roots': [],
'target_type': get_target_type(current_target),
'is_code_gen': current_target.is_codegen,
'pants_target_type': self._get_pants_target_alias(type(current_target))
}
if not current_target.is_synthetic:
info['globs'] = current_target.globs_relative_to_buildroot()
if self.get_options().sources:
info['sources'] = list(current_target.sources_relative_to_buildroot())
if isinstance(current_target, PythonRequirementLibrary):
reqs = current_target.payload.get_field_value('requirements', set())
""":type : set[pants.backend.python.python_requirement.PythonRequirement]"""
info['requirements'] = [req.key for req in reqs]
if isinstance(current_target, PythonTarget):
interpreter_for_target = self.select_interpreter_for_targets([current_target])
if interpreter_for_target is None:
raise TaskError('Unable to find suitable interpreter for {}'
.format(current_target.address))
python_interpreter_targets_mapping[interpreter_for_target].append(current_target)
info['python_interpreter'] = str(interpreter_for_target.identity)
def iter_transitive_jars(jar_lib):
"""
:type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary`
:rtype: :class:`collections.Iterator` of
:class:`pants.backend.jvm.jar_dependency_utils.M2Coordinate`
"""
if classpath_products:
jar_products = classpath_products.get_artifact_classpath_entries_for_targets((jar_lib,))
for _, jar_entry in jar_products:
coordinate = jar_entry.coordinate
# We drop classifier and type_ since those fields are represented in the global
# libraries dict and here we just want the key into that dict (see `_jar_id`).
yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev)
target_libraries = OrderedSet()
if isinstance(current_target, JarLibrary):
target_libraries = OrderedSet(iter_transitive_jars(current_target))
for dep in current_target.dependencies:
info['targets'].append(dep.address.spec)
if isinstance(dep, JarLibrary):
for jar in dep.jar_dependencies:
target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev))
# Add all the jars pulled in by this jar_library
target_libraries.update(iter_transitive_jars(dep))
if isinstance(dep, Resources):
resource_target_map[dep] = current_target
if isinstance(current_target, ScalaLibrary):
for dep in current_target.java_sources:
info['targets'].append(dep.address.spec)
process_target(dep)
if isinstance(current_target, JvmTarget):
info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes]
info['platform'] = current_target.platform.name
info['roots'] = map(lambda (source_root, package_prefix): {
'source_root': source_root,
'package_prefix': package_prefix
}, self._source_roots_for_target(current_target))
if classpath_products:
info['libraries'] = [self._jar_id(lib) for lib in target_libraries]
targets_map[current_target.address.spec] = info
for target in targets:
process_target(target)
jvm_platforms_map = {
'default_platform' : JvmPlatform.global_instance().default_platform.name,
'platforms': {
str(platform_name): {
'target_level' : str(platform.target_level),
'source_level' : str(platform.source_level),
'args' : platform.args,
} for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() }
}
graph_info = {
'version': self.DEFAULT_EXPORT_VERSION,
'targets': targets_map,
'jvm_platforms': jvm_platforms_map,
}
jvm_distributions = DistributionLocator.global_instance().all_jdk_paths()
if jvm_distributions:
graph_info['jvm_distributions'] = jvm_distributions
if classpath_products:
graph_info['libraries'] = self._resolve_jars_info(targets, classpath_products)
if python_interpreter_targets_mapping:
interpreters = self.interpreter_cache.select_interpreter(
python_interpreter_targets_mapping.keys())
default_interpreter = interpreters[0]
interpreters_info = {}
for interpreter, targets in six.iteritems(python_interpreter_targets_mapping):
chroot = self.cached_chroot(
interpreter=interpreter,
pex_info=PexInfo.default(),
targets=targets
)
interpreters_info[str(interpreter.identity)] = {
'binary': interpreter.binary,
'chroot': chroot.path()
}
graph_info['python_setup'] = {
'default_interpreter': str(default_interpreter.identity),
'interpreters': interpreters_info
}
return graph_info
def generate_targets_map(self, targets, runtime_classpath):
"""Generates a dictionary containing all pertinent information about the target graph.
The return dictionary is suitable for serialization by json.dumps.
:param targets: The list of targets to generate the map for.
:param classpath_products: Optional classpath_products. If not provided when the --libraries
option is `True`, this task will perform its own jar resolution.
"""
def _get_target_type(tgt):
def is_test(t):
return isinstance(t, JUnitTests)
if is_test(tgt):
return SourceRootTypes.TEST
else:
if (isinstance(tgt, Resources) and
tgt in resource_target_map and
is_test(resource_target_map[tgt])):
return SourceRootTypes.TEST_RESOURCE
elif isinstance(tgt, Resources):
return SourceRootTypes.RESOURCE
else:
return SourceRootTypes.SOURCE
targets_map = {}
resource_target_map = {}
target_roots_set = set(self.context.target_roots)
def process_target(current_target):
"""
:type current_target:pants.build_graph.target.Target
"""
info = {
# this means 'dependencies'
'targets': [],
'libraries': [],
'roots': [],
'id': current_target.id,
'target_type': _get_target_type(current_target),
'is_synthetic': current_target.is_synthetic,
'pants_target_type': self._get_pants_target_alias(type(current_target)),
'is_target_root': current_target in target_roots_set,
'transitive': current_target.transitive,
'scope': str(current_target.scope)
}
if not current_target.is_synthetic:
info['globs'] = current_target.globs_relative_to_buildroot()
def iter_transitive_jars(jar_lib):
"""
:type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary`
:rtype: :class:`collections.Iterator` of
:class:`pants.java.jar.M2Coordinate`
"""
if runtime_classpath:
jar_products = runtime_classpath.get_artifact_classpath_entries_for_targets((jar_lib,))
for _, jar_entry in jar_products:
coordinate = jar_entry.coordinate
# We drop classifier and type_ since those fields are represented in the global
# libraries dict and here we just want the key into that dict (see `_jar_id`).
yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev)
target_libraries = OrderedSet()
if isinstance(current_target, JarLibrary):
target_libraries = OrderedSet(iter_transitive_jars(current_target))
for dep in current_target.dependencies:
info['targets'].append(dep.address.spec)
if isinstance(dep, JarLibrary):
for jar in dep.jar_dependencies:
target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev))
# Add all the jars pulled in by this jar_library
target_libraries.update(iter_transitive_jars(dep))
if isinstance(dep, Resources):
resource_target_map[dep] = current_target
if isinstance(current_target, ScalaLibrary):
for dep in current_target.java_sources:
info['targets'].append(dep.address.spec)
if isinstance(current_target, JvmTarget):
info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes]
info['platform'] = current_target.platform.name
if hasattr(current_target, 'test_platform'):
info['test_platform'] = current_target.test_platform.name
if runtime_classpath:
info['libraries'].extend(self._jar_id(lib) for lib in target_libraries)
if current_target in target_roots_set:
info['roots'] = [{
'source_root': os.path.realpath(source_root_package_prefix[0]),
'package_prefix': source_root_package_prefix[1]
} for source_root_package_prefix in self._source_roots_for_target(current_target)]
targets_map[current_target.address.spec] = info
additional_java_targets = []
for t in targets:
if isinstance(t, ScalaLibrary):
additional_java_targets.extend(t.java_sources)
targets.extend(additional_java_targets)
for target in targets:
process_target(target)
scala_platform = ScalaPlatform.global_instance()
scala_platform_map = {
'scala_version': scala_platform.version,
'compiler_classpath': [
cp_entry.path
for cp_entry in scala_platform.compiler_classpath_entries(self.context.products)
],
}
jvm_platforms_map = {
'default_platform': JvmPlatform.global_instance().default_platform.name,
'platforms': {
str(platform_name): {
'target_level': str(platform.target_level),
'source_level': str(platform.source_level),
'args': platform.args,
} for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items()},
}
graph_info = {
'version': DEFAULT_EXPORT_VERSION,
'targets': targets_map,
'jvm_platforms': jvm_platforms_map,
'scala_platform': scala_platform_map,
# `jvm_distributions` are static distribution settings from config,
# `preferred_jvm_distributions` are distributions that pants actually uses for the
# given platform setting.
'preferred_jvm_distributions': {}
}
for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items():
preferred_distributions = {}
for strict, strict_key in [(True, 'strict'), (False, 'non_strict')]:
try:
dist = JvmPlatform.preferred_jvm_distribution([platform], strict=strict)
preferred_distributions[strict_key] = dist.home
except DistributionLocator.Error:
pass
if preferred_distributions:
graph_info['preferred_jvm_distributions'][platform_name] = preferred_distributions
def zip_sources(target, location, suffix='.jar'):
with temporary_file(root_dir=location, cleanup=False, suffix=suffix) as f:
with zipfile.ZipFile(f, 'a') as zip_file:
for src_from_source_root, src_from_build_root in zip(target.sources_relative_to_source_root(), target.sources_relative_to_buildroot()):
zip_file.write(os.path.join(get_buildroot(), src_from_build_root), src_from_source_root)
return f
if runtime_classpath:
graph_info['libraries'] = self._resolve_jars_info(targets, runtime_classpath)
# Using resolved path in preparation for VCFS.
resource_jar_root = os.path.realpath(self.versioned_workdir)
for t in targets:
target_type = _get_target_type(t)
# If it is a target root or it is already a jar_library target, then no-op.
if t in target_roots_set or targets_map[t.address.spec]['pants_target_type'] == 'jar_library':
continue
if target_type == SourceRootTypes.RESOURCE or target_type == SourceRootTypes.TEST_RESOURCE:
# yic assumed that the cost to fingerprint the target may not be that lower than
# just zipping up the resources anyway.
jarred_resources = zip_sources(t, resource_jar_root)
targets_map[t.address.spec]['pants_target_type'] = 'jar_library'
targets_map[t.address.spec]['libraries'] = [t.id]
graph_info['libraries'][t.id]['default'] = jarred_resources.name
continue
targets_map[t.address.spec]['pants_target_type'] = 'jar_library'
targets_map[t.address.spec]['libraries'] = [t.id]
jar_products = runtime_classpath.get_for_target(t)
for conf, jar_entry in jar_products:
# TODO(yic): check --compile-rsc-use-classpath-jars is enabled.
# If not, zip up the classes/ dir here.
if 'z.jar' in jar_entry:
graph_info['libraries'][t.id][conf] = jar_entry
if self.get_options().sources:
# NB: We create the jar in the same place as we create the resources
# (as opposed to where we store the z.jar), because the path to the z.jar depends
# on tasks outside of this one.
# In addition to that, we may not want to depend on z.jar existing to export source jars.
jarred_sources = zip_sources(t, resource_jar_root, suffix='-sources.jar')
graph_info['libraries'][t.id]['sources'] = jarred_sources.name
return graph_info
def generate_project(self, project):
def linked_folder_id(source_set):
return source_set.source_base.replace(os.path.sep, '.')
def base_path(source_set):
return os.path.join(source_set.root_dir, source_set.source_base,
source_set.path)
def create_source_base_template(source_set):
source_base = base_path(source_set)
return SourceBase(id=linked_folder_id(source_set),
path=source_base)
source_sets = project.sources[:]
if project.has_python:
source_sets.extend(project.py_sources)
source_bases = frozenset(map(create_source_base_template, source_sets))
libs = []
def add_jarlibs(classpath_entries):
for classpath_entry in classpath_entries:
libs.append((classpath_entry.jar, classpath_entry.source_jar))
add_jarlibs(project.internal_jars)
add_jarlibs(project.external_jars)
scala_full_version = scala_platform.scala_build_info[
self.context.options['scala-platform']['version']].full_version
scala = {
'language_level': scala_full_version,
'compiler_classpath': project.scala_compiler_classpath
}
outdir = os.path.abspath(self.ensime_output_dir)
if not os.path.exists(outdir):
os.makedirs(outdir)
java_platform = JvmPlatform.global_instance().default_platform
jdk_home = JvmPlatform.preferred_jvm_distribution([java_platform],
strict=True).home
configured_project = {
'name':
self.project_name,
'java_home':
jdk_home,
'scala':
scala,
'source_bases':
source_bases,
'has_tests':
project.has_tests,
'internal_jars':
[cp_entry.jar for cp_entry in project.internal_jars],
'internal_source_jars': [
cp_entry.source_jar for cp_entry in project.internal_jars
if cp_entry.source_jar
],
'external_jars':
[cp_entry.jar for cp_entry in project.external_jars],
'external_javadoc_jars': [
cp_entry.javadoc_jar for cp_entry in project.external_jars
if cp_entry.javadoc_jar
],
'external_source_jars': [
cp_entry.source_jar for cp_entry in project.external_jars
if cp_entry.source_jar
],
'libs':
libs,
'outdir':
os.path.relpath(outdir, get_buildroot()),
'root_dir':
get_buildroot(),
'cache_dir':
os.path.join(self.cwd, '.ensime_cache')
}
with safe_open(self.project_filename, 'w') as output:
output.write(
self._renderer.render_name(_TEMPLATE,
{'project': configured_project}))
print('\nGenerated ensime project at {}{}'.format(
self.gen_project_workdir, os.sep))
def generate_project(self, project):
def linked_folder_id(source_set):
return source_set.source_base.replace(os.path.sep, '.')
def base_path(source_set):
return os.path.join(source_set.root_dir, source_set.source_base,
source_set.path)
def create_source_base_template(source_set):
source_base = base_path(source_set)
return SourceBase(id=linked_folder_id(source_set),
path=source_base)
source_sets = project.sources[:]
if project.has_python:
source_sets.extend(project.py_sources)
source_bases = frozenset(map(create_source_base_template, source_sets))
libs = []
def add_jarlibs(classpath_entries):
for classpath_entry in classpath_entries:
libs.append((classpath_entry.jar, classpath_entry.source_jar))
add_jarlibs(project.internal_jars)
add_jarlibs(project.external_jars)
scala_full_version = scala_platform.scala_build_info[
self.context.options['scala-platform']['version']].full_version
scala = TemplateData(
language_level=scala_full_version,
compiler_classpath=project.scala_compiler_classpath)
outdir = os.path.abspath(self.ensime_output_dir)
if not os.path.exists(outdir):
os.makedirs(outdir)
java_platform = JvmPlatform.global_instance().default_platform
jdk_home = JvmPlatform.preferred_jvm_distribution([java_platform],
strict=True).home
configured_project = TemplateData(
name=self.project_name,
java_home=jdk_home,
scala=scala,
source_bases=source_bases,
has_tests=project.has_tests,
internal_jars=[cp_entry.jar for cp_entry in project.internal_jars],
internal_source_jars=[
cp_entry.source_jar for cp_entry in project.internal_jars
if cp_entry.source_jar
],
external_jars=[cp_entry.jar for cp_entry in project.external_jars],
external_javadoc_jars=[
cp_entry.javadoc_jar for cp_entry in project.external_jars
if cp_entry.javadoc_jar
],
external_source_jars=[
cp_entry.source_jar for cp_entry in project.external_jars
if cp_entry.source_jar
],
libs=libs,
outdir=os.path.relpath(outdir, get_buildroot()),
root_dir=get_buildroot(),
cache_dir=os.path.join(self.cwd, '.ensime_cache'))
def apply_template(output_path, template_relpath, **template_data):
with safe_open(output_path, 'w') as output:
Generator(pkgutil.get_data(__name__, template_relpath),
**template_data).write(output)
apply_template(self.project_filename,
self.project_template,
project=configured_project)
print('\nGenerated ensime project at {}{}'.format(
self.gen_project_workdir, os.sep))
def platform(self): return JvmPlatform.global_instance().get_platform_for_target(self)
def console_output(self, targets):
targets_map = {}
resource_target_map = {}
ivy_info = None
if self.get_options().libraries:
ivy_jar_products = self.context.products.get_data(
'ivy_jar_products') or {}
# This product is a list for historical reasons (exclusives groups) but in practice should
# have either 0 or 1 entries.
ivy_info_list = ivy_jar_products.get('default')
if ivy_info_list:
assert len(ivy_info_list) == 1, (
'The values in ivy_jar_products should always be length 1,'
' since we no longer have exclusives groups.')
ivy_info = ivy_info_list[0]
ivy_jar_memo = {}
python_interpreter_targets_mapping = defaultdict(list)
def process_target(current_target):
"""
:type current_target:pants.base.target.Target
"""
def get_target_type(target):
if target.is_test:
return Export.SourceRootTypes.TEST
else:
if (isinstance(target, Resources)
and target in resource_target_map
and resource_target_map[target].is_test):
return Export.SourceRootTypes.TEST_RESOURCE
elif isinstance(target, Resources):
return Export.SourceRootTypes.RESOURCE
else:
return Export.SourceRootTypes.SOURCE
def get_transitive_jars(jar_lib):
"""
:type jar_lib: pants.backend.jvm.targets.jar_library.JarLibrary
:rtype: twitter.common.collections.orderedset.OrderedSet
"""
if not ivy_info or not self.get_options().libraries:
return OrderedSet()
transitive_jars = OrderedSet()
for jar in jar_lib.jar_dependencies:
transitive_jars.update(
ivy_info.get_jars_for_ivy_module(jar,
memo=ivy_jar_memo))
return transitive_jars
info = {
'targets': [],
'libraries': [],
'roots': [],
'target_type':
get_target_type(current_target),
'is_code_gen':
current_target.is_codegen,
'pants_target_type':
self._get_pants_target_alias(type(current_target))
}
if not current_target.is_synthetic:
info['globs'] = current_target.globs_relative_to_buildroot()
if self.get_options().sources:
info['sources'] = list(
current_target.sources_relative_to_buildroot())
if isinstance(current_target, PythonRequirementLibrary):
reqs = current_target.payload.get_field_value(
'requirements', set())
""":type : set[pants.backend.python.python_requirement.PythonRequirement]"""
info['requirements'] = [req.key for req in reqs]
if isinstance(current_target, PythonTarget):
interpreter_for_target = self.select_interpreter_for_targets(
[current_target])
if interpreter_for_target is None:
raise TaskError(
'Unable to find suitable interpreter for {}'.format(
current_target.address))
python_interpreter_targets_mapping[
interpreter_for_target].append(current_target)
info['python_interpreter'] = str(
interpreter_for_target.identity)
target_libraries = OrderedSet()
if isinstance(current_target, JarLibrary):
target_libraries = get_transitive_jars(current_target)
for dep in current_target.dependencies:
info['targets'].append(dep.address.spec)
if isinstance(dep, JarLibrary):
for jar in dep.jar_dependencies:
target_libraries.add(
IvyModuleRef(jar.org, jar.name, jar.rev))
# Add all the jars pulled in by this jar_library
target_libraries.update(get_transitive_jars(dep))
if isinstance(dep, Resources):
resource_target_map[dep] = current_target
if isinstance(current_target, ScalaLibrary):
for dep in current_target.java_sources:
info['targets'].append(dep.address.spec)
process_target(dep)
if isinstance(current_target, JvmTarget):
info['excludes'] = [
self._exclude_id(exclude)
for exclude in current_target.excludes
]
info['platform'] = current_target.platform.name
info['roots'] = map(
lambda (source_root, package_prefix): {
'source_root': source_root,
'package_prefix': package_prefix
}, self._source_roots_for_target(current_target))
if self.get_options().libraries:
info['libraries'] = [
self._jar_id(lib) for lib in target_libraries
]
targets_map[current_target.address.spec] = info
for target in targets:
process_target(target)
jvm_platforms_map = {
'default_platform':
JvmPlatform.global_instance().default_platform.name,
'platforms': {
str(platform_name): {
'target_level': str(platform.target_level),
'source_level': str(platform.source_level),
'args': platform.args,
}
for platform_name, platform in
JvmPlatform.global_instance().platforms_by_name.items()
}
}
graph_info = {
'version': self.DEFAULT_EXPORT_VERSION,
'targets': targets_map,
'jvm_platforms': jvm_platforms_map,
}
jvm_distributions = DistributionLocator.global_instance(
).all_jdk_paths()
if jvm_distributions:
graph_info['jvm_distributions'] = jvm_distributions
if self.get_options().libraries:
graph_info['libraries'] = self._resolve_jars_info()
if python_interpreter_targets_mapping:
default_interpreter = self.interpreter_cache.select_interpreter(
python_interpreter_targets_mapping.keys())[0]
interpreters_info = {}
for interpreter, targets in python_interpreter_targets_mapping.iteritems(
):
chroot = self.cached_chroot(interpreter=interpreter,
pex_info=PexInfo.default(),
targets=targets)
interpreters_info[str(interpreter.identity)] = {
'binary': interpreter.binary,
'chroot': chroot.path()
}
graph_info['python_setup'] = {
'default_interpreter': str(default_interpreter.identity),
'interpreters': interpreters_info
}
if self.format:
return json.dumps(graph_info, indent=4,
separators=(',', ': ')).splitlines()
else:
return [json.dumps(graph_info)]
def execute(self):
if JvmPlatform.global_instance().get_options().compiler == 'javac':
return super(JavacCompile, self).execute()
def generate_targets_map(self, targets, classpath_products=None):
"""Generates a dictionary containing all pertinent information about the target graph.
The return dictionary is suitable for serialization by json.dumps.
:param targets: The list of targets to generate the map for.
:param classpath_products: Optional classpath_products. If not provided when the --libraries
option is `True`, this task will perform its own jar resolution.
"""
targets_map = {}
resource_target_map = {}
python_interpreter_targets_mapping = defaultdict(list)
if self.get_options().libraries:
# NB(gmalmquist): This supports mocking the classpath_products in tests.
if classpath_products is None:
classpath_products = self.resolve_jars(targets)
else:
classpath_products = None
target_roots_set = set(self.context.target_roots)
def process_target(current_target):
"""
:type current_target:pants.build_graph.target.Target
"""
def get_target_type(tgt):
def is_test(t):
return isinstance(t, JUnitTests) or isinstance(t, PythonTests)
if is_test(tgt):
return ExportTask.SourceRootTypes.TEST
else:
if (isinstance(tgt, Resources) and
tgt in resource_target_map and
is_test(resource_target_map[tgt])):
return ExportTask.SourceRootTypes.TEST_RESOURCE
elif isinstance(tgt, Resources):
return ExportTask.SourceRootTypes.RESOURCE
else:
return ExportTask.SourceRootTypes.SOURCE
info = {
'targets': [],
'libraries': [],
'roots': [],
'id': current_target.id,
'target_type': get_target_type(current_target),
# NB: is_code_gen should be removed when export format advances to 1.1.0 or higher
'is_code_gen': current_target.is_synthetic,
'is_synthetic': current_target.is_synthetic,
'pants_target_type': self._get_pants_target_alias(type(current_target)),
}
if not current_target.is_synthetic:
info['globs'] = current_target.globs_relative_to_buildroot()
if self.get_options().sources:
info['sources'] = list(current_target.sources_relative_to_buildroot())
info['transitive'] = current_target.transitive
info['scope'] = str(current_target.scope)
info['is_target_root'] = current_target in target_roots_set
if isinstance(current_target, PythonRequirementLibrary):
reqs = current_target.payload.get_field_value('requirements', set())
""":type : set[pants.backend.python.python_requirement.PythonRequirement]"""
info['requirements'] = [req.key for req in reqs]
if isinstance(current_target, PythonTarget):
interpreter_for_target = self._interpreter_cache.select_interpreter_for_targets(
[current_target])
if interpreter_for_target is None:
raise TaskError('Unable to find suitable interpreter for {}'
.format(current_target.address))
python_interpreter_targets_mapping[interpreter_for_target].append(current_target)
info['python_interpreter'] = str(interpreter_for_target.identity)
def iter_transitive_jars(jar_lib):
"""
:type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary`
:rtype: :class:`collections.Iterator` of
:class:`pants.java.jar.M2Coordinate`
"""
if classpath_products:
jar_products = classpath_products.get_artifact_classpath_entries_for_targets((jar_lib,))
for _, jar_entry in jar_products:
coordinate = jar_entry.coordinate
# We drop classifier and type_ since those fields are represented in the global
# libraries dict and here we just want the key into that dict (see `_jar_id`).
yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev)
target_libraries = OrderedSet()
if isinstance(current_target, JarLibrary):
target_libraries = OrderedSet(iter_transitive_jars(current_target))
for dep in current_target.dependencies:
info['targets'].append(dep.address.spec)
if isinstance(dep, JarLibrary):
for jar in dep.jar_dependencies:
target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev))
# Add all the jars pulled in by this jar_library
target_libraries.update(iter_transitive_jars(dep))
if isinstance(dep, Resources):
resource_target_map[dep] = current_target
if isinstance(current_target, ScalaLibrary):
for dep in current_target.java_sources:
info['targets'].append(dep.address.spec)
process_target(dep)
if isinstance(current_target, JvmTarget):
info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes]
info['platform'] = current_target.platform.name
if hasattr(current_target, 'test_platform'):
info['test_platform'] = current_target.test_platform.name
info['roots'] = [{
'source_root': source_root_package_prefix[0],
'package_prefix': source_root_package_prefix[1]
} for source_root_package_prefix in self._source_roots_for_target(current_target)]
if classpath_products:
info['libraries'] = [self._jar_id(lib) for lib in target_libraries]
targets_map[current_target.address.spec] = info
for target in targets:
process_target(target)
jvm_platforms_map = {
'default_platform' : JvmPlatform.global_instance().default_platform.name,
'platforms': {
str(platform_name): {
'target_level' : str(platform.target_level),
'source_level' : str(platform.source_level),
'args' : platform.args,
} for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() },
}
graph_info = {
'version': self.DEFAULT_EXPORT_VERSION,
'targets': targets_map,
'jvm_platforms': jvm_platforms_map,
# `jvm_distributions` are static distribution settings from config,
# `preferred_jvm_distributions` are distributions that pants actually uses for the
# given platform setting.
'preferred_jvm_distributions': {}
}
for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items():
preferred_distributions = {}
for strict, strict_key in [(True, 'strict'), (False, 'non_strict')]:
try:
dist = JvmPlatform.preferred_jvm_distribution([platform], strict=strict)
preferred_distributions[strict_key] = dist.home
except DistributionLocator.Error:
pass
if preferred_distributions:
graph_info['preferred_jvm_distributions'][platform_name] = preferred_distributions
if classpath_products:
graph_info['libraries'] = self._resolve_jars_info(targets, classpath_products)
if python_interpreter_targets_mapping:
# NB: We've selected a python interpreter compatible with each python target individually into
# the `python_interpreter_targets_mapping`. These python targets may not be compatible, ie: we
# could have a python target requiring 'CPython>=2.7<3' (ie: CPython-2.7.x) and another
# requiring 'CPython>=3.6'. To pick a default interpreter then from among these two choices
# is arbitrary and not to be relied on to work as a default interpreter if ever needed by the
# export consumer.
#
# TODO(John Sirois): consider either eliminating the 'default_interpreter' field and pressing
# export consumers to make their own choice of a default (if needed) or else use
# `select.select_interpreter_for_targets` and fail fast if there is no interpreter compatible
# across all the python targets in-play.
#
# For now, make our arbitrary historical choice of a default interpreter explicit and use the
# lowest version.
default_interpreter = min(python_interpreter_targets_mapping.keys())
interpreters_info = {}
for interpreter, targets in six.iteritems(python_interpreter_targets_mapping):
req_libs = [target for target in Target.closure_for_targets(targets)
if has_python_requirements(target)]
chroot = self.resolve_requirements(interpreter, req_libs)
interpreters_info[str(interpreter.identity)] = {
'binary': interpreter.binary,
'chroot': chroot.path()
}
graph_info['python_setup'] = {
'default_interpreter': str(default_interpreter.identity),
'interpreters': interpreters_info
}
return graph_info
def generate_targets_map(self, targets, classpath_products=None):
"""Generates a dictionary containing all pertinent information about the target graph.
The return dictionary is suitable for serialization by json.dumps.
:param targets: The list of targets to generate the map for.
:param classpath_products: Optional classpath_products. If not provided when the --libraries
option is `True`, this task will perform its own jar resolution.
"""
targets_map = {}
resource_target_map = {}
python_interpreter_targets_mapping = defaultdict(list)
if self.get_options().libraries:
# NB(gmalmquist): This supports mocking the classpath_products in tests.
if classpath_products is None:
classpath_products = self.resolve_jars(targets)
else:
classpath_products = None
def process_target(current_target):
"""
:type current_target:pants.build_graph.target.Target
"""
def get_target_type(target):
if target.is_test:
return ExportTask.SourceRootTypes.TEST
else:
if (isinstance(target, Resources) and
target in resource_target_map and
resource_target_map[target].is_test):
return ExportTask.SourceRootTypes.TEST_RESOURCE
elif isinstance(target, Resources):
return ExportTask.SourceRootTypes.RESOURCE
else:
return ExportTask.SourceRootTypes.SOURCE
info = {
'targets': [],
'libraries': [],
'roots': [],
'id': current_target.id,
'target_type': get_target_type(current_target),
# NB: is_code_gen should be removed when export format advances to 1.1.0 or higher
'is_code_gen': current_target.is_codegen,
'is_synthetic': current_target.is_synthetic,
'pants_target_type': self._get_pants_target_alias(type(current_target))
}
if not current_target.is_synthetic:
info['globs'] = current_target.globs_relative_to_buildroot()
if self.get_options().sources:
info['sources'] = list(current_target.sources_relative_to_buildroot())
if isinstance(current_target, PythonRequirementLibrary):
reqs = current_target.payload.get_field_value('requirements', set())
""":type : set[pants.backend.python.python_requirement.PythonRequirement]"""
info['requirements'] = [req.key for req in reqs]
if isinstance(current_target, PythonTarget):
interpreter_for_target = self.select_interpreter_for_targets([current_target])
if interpreter_for_target is None:
raise TaskError('Unable to find suitable interpreter for {}'
.format(current_target.address))
python_interpreter_targets_mapping[interpreter_for_target].append(current_target)
info['python_interpreter'] = str(interpreter_for_target.identity)
def iter_transitive_jars(jar_lib):
"""
:type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary`
:rtype: :class:`collections.Iterator` of
:class:`pants.backend.jvm.jar_dependency_utils.M2Coordinate`
"""
if classpath_products:
jar_products = classpath_products.get_artifact_classpath_entries_for_targets((jar_lib,))
for _, jar_entry in jar_products:
coordinate = jar_entry.coordinate
# We drop classifier and type_ since those fields are represented in the global
# libraries dict and here we just want the key into that dict (see `_jar_id`).
yield M2Coordinate(org=coordinate.org, name=coordinate.name, rev=coordinate.rev)
target_libraries = OrderedSet()
if isinstance(current_target, JarLibrary):
target_libraries = OrderedSet(iter_transitive_jars(current_target))
for dep in current_target.dependencies:
info['targets'].append(dep.address.spec)
if isinstance(dep, JarLibrary):
for jar in dep.jar_dependencies:
target_libraries.add(M2Coordinate(jar.org, jar.name, jar.rev))
# Add all the jars pulled in by this jar_library
target_libraries.update(iter_transitive_jars(dep))
if isinstance(dep, Resources):
resource_target_map[dep] = current_target
if isinstance(current_target, ScalaLibrary):
for dep in current_target.java_sources:
info['targets'].append(dep.address.spec)
process_target(dep)
if isinstance(current_target, JvmTarget):
info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes]
info['platform'] = current_target.platform.name
info['roots'] = map(lambda (source_root, package_prefix): {
'source_root': source_root,
'package_prefix': package_prefix
}, self._source_roots_for_target(current_target))
if classpath_products:
info['libraries'] = [self._jar_id(lib) for lib in target_libraries]
targets_map[current_target.address.spec] = info
for target in targets:
process_target(target)
jvm_platforms_map = {
'default_platform' : JvmPlatform.global_instance().default_platform.name,
'platforms': {
str(platform_name): {
'target_level' : str(platform.target_level),
'source_level' : str(platform.source_level),
'args' : platform.args,
} for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() }
}
graph_info = {
'version': self.DEFAULT_EXPORT_VERSION,
'targets': targets_map,
'jvm_platforms': jvm_platforms_map,
}
jvm_distributions = DistributionLocator.global_instance().all_jdk_paths()
if jvm_distributions:
graph_info['jvm_distributions'] = jvm_distributions
if classpath_products:
graph_info['libraries'] = self._resolve_jars_info(targets, classpath_products)
if python_interpreter_targets_mapping:
interpreters = self.interpreter_cache.select_interpreter(
python_interpreter_targets_mapping.keys())
default_interpreter = interpreters[0]
interpreters_info = {}
for interpreter, targets in six.iteritems(python_interpreter_targets_mapping):
chroot = self.cached_chroot(
interpreter=interpreter,
pex_info=PexInfo.default(),
targets=targets
)
interpreters_info[str(interpreter.identity)] = {
'binary': interpreter.binary,
'chroot': chroot.path()
}
graph_info['python_setup'] = {
'default_interpreter': str(default_interpreter.identity),
'interpreters': interpreters_info
}
return graph_info
def test_platform(self):
if self.payload.test_platform:
return JvmPlatform.global_instance().get_platform_by_name(self.payload.test_platform)
return self.platform
def generate_targets_map(self, targets, classpath_products=None):
"""Generates a dictionary containing all pertinent information about the target graph.
The return dictionary is suitable for serialization by json.dumps.
:param targets: The list of targets to generate the map for.
:param classpath_products: Optional classpath_products. If not provided when the --libraries
option is `True`, this task will perform its own jar resolution.
"""
targets_map = {}
resource_target_map = {}
python_interpreter_targets_mapping = defaultdict(list)
if self.get_options().libraries:
# NB(gmalmquist): This supports mocking the classpath_products in tests.
if classpath_products is None:
classpath_products = self.resolve_jars(targets)
else:
classpath_products = None
target_roots_set = set(self.context.target_roots)
def process_target(current_target):
"""
:type current_target:pants.build_graph.target.Target
"""
def get_target_type(tgt):
def is_test(t):
return isinstance(t, JUnitTests) or isinstance(
t, PythonTests)
if is_test(tgt):
return SourceRootTypes.TEST
else:
if (isinstance(tgt, Resources)
and tgt in resource_target_map
and is_test(resource_target_map[tgt])):
return SourceRootTypes.TEST_RESOURCE
elif isinstance(tgt, Resources):
return SourceRootTypes.RESOURCE
else:
return SourceRootTypes.SOURCE
info = {
'targets': [],
'libraries': [],
'roots': [],
'id':
current_target.id,
'target_type':
get_target_type(current_target),
# NB: is_code_gen should be removed when export format advances to 1.1.0 or higher
'is_code_gen':
current_target.is_synthetic,
'is_synthetic':
current_target.is_synthetic,
'pants_target_type':
self._get_pants_target_alias(type(current_target)),
}
if not current_target.is_synthetic:
info['globs'] = current_target.globs_relative_to_buildroot()
if self.get_options().sources:
info['sources'] = list(
current_target.sources_relative_to_buildroot())
info['transitive'] = current_target.transitive
info['scope'] = str(current_target.scope)
info['is_target_root'] = current_target in target_roots_set
if isinstance(current_target, PythonRequirementLibrary):
reqs = current_target.payload.get_field_value(
'requirements', set())
""":type : set[pants.backend.python.python_requirement.PythonRequirement]"""
info['requirements'] = [req.key for req in reqs]
if isinstance(current_target, PythonTarget):
interpreter_for_target = self._interpreter_cache.select_interpreter_for_targets(
[current_target])
if interpreter_for_target is None:
raise TaskError(
'Unable to find suitable interpreter for {}'.format(
current_target.address))
python_interpreter_targets_mapping[
interpreter_for_target].append(current_target)
info['python_interpreter'] = str(
interpreter_for_target.identity)
def iter_transitive_jars(jar_lib):
"""
:type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary`
:rtype: :class:`collections.Iterator` of
:class:`pants.java.jar.M2Coordinate`
"""
if classpath_products:
jar_products = classpath_products.get_artifact_classpath_entries_for_targets(
(jar_lib, ))
for _, jar_entry in jar_products:
coordinate = jar_entry.coordinate
# We drop classifier and type_ since those fields are represented in the global
# libraries dict and here we just want the key into that dict (see `_jar_id`).
yield M2Coordinate(org=coordinate.org,
name=coordinate.name,
rev=coordinate.rev)
target_libraries = OrderedSet()
if isinstance(current_target, JarLibrary):
target_libraries = OrderedSet(
iter_transitive_jars(current_target))
for dep in current_target.dependencies:
info['targets'].append(dep.address.spec)
if isinstance(dep, JarLibrary):
for jar in dep.jar_dependencies:
target_libraries.add(
M2Coordinate(jar.org, jar.name, jar.rev))
# Add all the jars pulled in by this jar_library
target_libraries.update(iter_transitive_jars(dep))
if isinstance(dep, Resources):
resource_target_map[dep] = current_target
if isinstance(current_target, ScalaLibrary):
for dep in current_target.java_sources:
info['targets'].append(dep.address.spec)
process_target(dep)
if isinstance(current_target, JvmTarget):
info['excludes'] = [
self._exclude_id(exclude)
for exclude in current_target.excludes
]
info['platform'] = current_target.platform.name
if hasattr(current_target, 'test_platform'):
info['test_platform'] = current_target.test_platform.name
info['roots'] = [{
'source_root': source_root_package_prefix[0],
'package_prefix': source_root_package_prefix[1]
} for source_root_package_prefix in self._source_roots_for_target(
current_target)]
if classpath_products:
info['libraries'] = [
self._jar_id(lib) for lib in target_libraries
]
targets_map[current_target.address.spec] = info
for target in targets:
process_target(target)
scala_platform = ScalaPlatform.global_instance()
scala_platform_map = {
'scala_version':
scala_platform.version,
'compiler_classpath': [
cp_entry.path
for cp_entry in scala_platform.compiler_classpath_entries(
self.context.products)
],
}
jvm_platforms_map = {
'default_platform':
JvmPlatform.global_instance().default_platform.name,
'platforms': {
str(platform_name): {
'target_level': str(platform.target_level),
'source_level': str(platform.source_level),
'args': platform.args,
}
for platform_name, platform in
JvmPlatform.global_instance().platforms_by_name.items()
},
}
graph_info = {
'version': DEFAULT_EXPORT_VERSION,
'targets': targets_map,
'jvm_platforms': jvm_platforms_map,
'scala_platform': scala_platform_map,
# `jvm_distributions` are static distribution settings from config,
# `preferred_jvm_distributions` are distributions that pants actually uses for the
# given platform setting.
'preferred_jvm_distributions': {}
}
for platform_name, platform in JvmPlatform.global_instance(
).platforms_by_name.items():
preferred_distributions = {}
for strict, strict_key in [(True, 'strict'),
(False, 'non_strict')]:
try:
dist = JvmPlatform.preferred_jvm_distribution(
[platform], strict=strict)
preferred_distributions[strict_key] = dist.home
except DistributionLocator.Error:
pass
if preferred_distributions:
graph_info['preferred_jvm_distributions'][
platform_name] = preferred_distributions
if classpath_products:
graph_info['libraries'] = self._resolve_jars_info(
targets, classpath_products)
if python_interpreter_targets_mapping:
# NB: We've selected a python interpreter compatible with each python target individually into
# the `python_interpreter_targets_mapping`. These python targets may not be compatible, ie: we
# could have a python target requiring 'CPython>=2.7<3' (ie: CPython-2.7.x) and another
# requiring 'CPython>=3.6'. To pick a default interpreter then from among these two choices
# is arbitrary and not to be relied on to work as a default interpreter if ever needed by the
# export consumer.
#
# TODO(John Sirois): consider either eliminating the 'default_interpreter' field and pressing
# export consumers to make their own choice of a default (if needed) or else use
# `select.select_interpreter_for_targets` and fail fast if there is no interpreter compatible
# across all the python targets in-play.
#
# For now, make our arbitrary historical choice of a default interpreter explicit and use the
# lowest version.
default_interpreter = min(
python_interpreter_targets_mapping.keys())
interpreters_info = {}
for interpreter, targets in python_interpreter_targets_mapping.items(
):
req_libs = [
target for target in Target.closure_for_targets(targets)
if has_python_requirements(target)
]
chroot = self.resolve_requirements(interpreter, req_libs)
interpreters_info[str(interpreter.identity)] = {
'binary': interpreter.binary,
'chroot': chroot.path()
}
graph_info['python_setup'] = {
'default_interpreter': str(default_interpreter.identity),
'interpreters': interpreters_info
}
if self.get_options().available_target_types:
graph_info['available_target_types'] = self._target_types()
return graph_info
def execute(self):
if JvmPlatform.global_instance().get_options().compiler == 'zinc':
return super(ZincCompile, self).execute()
def console_output(self, targets):
targets_map = {}
resource_target_map = {}
ivy_info = None
if self.get_options().libraries:
ivy_jar_products = self.context.products.get_data('ivy_jar_products') or {}
# This product is a list for historical reasons (exclusives groups) but in practice should
# have either 0 or 1 entries.
ivy_info_list = ivy_jar_products.get('default')
if ivy_info_list:
assert len(ivy_info_list) == 1, (
'The values in ivy_jar_products should always be length 1,'
' since we no longer have exclusives groups.'
)
ivy_info = ivy_info_list[0]
ivy_jar_memo = {}
def process_target(current_target):
"""
:type current_target:pants.base.target.Target
"""
def get_target_type(target):
if target.is_test:
return Export.SourceRootTypes.TEST
else:
if (isinstance(target, Resources) and
target in resource_target_map and
resource_target_map[target].is_test):
return Export.SourceRootTypes.TEST_RESOURCE
elif isinstance(target, Resources):
return Export.SourceRootTypes.RESOURCE
else:
return Export.SourceRootTypes.SOURCE
def get_transitive_jars(jar_lib):
"""
:type jar_lib: pants.backend.jvm.targets.jar_library.JarLibrary
:rtype: twitter.common.collections.orderedset.OrderedSet
"""
if not ivy_info or not self.get_options().libraries:
return OrderedSet()
transitive_jars = OrderedSet()
for jar in jar_lib.jar_dependencies:
transitive_jars.update(ivy_info.get_jars_for_ivy_module(jar, memo=ivy_jar_memo))
return transitive_jars
info = {
'targets': [],
'libraries': [],
'roots': [],
'target_type': get_target_type(current_target),
'is_code_gen': current_target.is_codegen,
'pants_target_type': self._get_pants_target_alias(type(current_target))
}
if not current_target.is_synthetic:
info['globs'] = current_target.globs_relative_to_buildroot()
if self.get_options().sources:
info['sources'] = list(current_target.sources_relative_to_buildroot())
target_libraries = OrderedSet()
if isinstance(current_target, JarLibrary):
target_libraries = get_transitive_jars(current_target)
for dep in current_target.dependencies:
info['targets'].append(dep.address.spec)
if isinstance(dep, JarLibrary):
for jar in dep.jar_dependencies:
target_libraries.add(IvyModuleRef(jar.org, jar.name, jar.rev))
# Add all the jars pulled in by this jar_library
target_libraries.update(get_transitive_jars(dep))
if isinstance(dep, Resources):
resource_target_map[dep] = current_target
if isinstance(current_target, ScalaLibrary):
for dep in current_target.java_sources:
info['targets'].append(dep.address.spec)
process_target(dep)
if isinstance(current_target, JvmTarget):
info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes]
info['platform'] = current_target.platform.name
info['roots'] = map(lambda (source_root, package_prefix): {
'source_root': source_root,
'package_prefix': package_prefix
}, self._source_roots_for_target(current_target))
if self.get_options().libraries:
info['libraries'] = [self._jar_id(lib) for lib in target_libraries]
targets_map[current_target.address.spec] = info
for target in targets:
process_target(target)
jvm_platforms_map = {
'default_platform' : JvmPlatform.global_instance().default_platform.name,
'platforms': {
str(platform_name): {
'target_level' : str(platform.target_level),
'source_level' : str(platform.source_level),
'args' : platform.args,
} for platform_name, platform in JvmPlatform.global_instance().platforms_by_name.items() }
}
graph_info = {
'targets': targets_map,
'jvm_platforms' : jvm_platforms_map,
}
jvm_distributions = DistributionLocator.global_instance().all_jdk_paths()
if jvm_distributions:
graph_info['jvm_distributions'] = jvm_distributions
if self.get_options().libraries:
graph_info['libraries'] = self._resolve_jars_info()
graph_info['version'] = self.DEFAULT_EXPORT_VERSION
if self.format:
return json.dumps(graph_info, indent=4, separators=(',', ': ')).splitlines()
else:
return [json.dumps(graph_info)]