def execute(self):
if not self.context.targets(
lambda t: is_python_target(t) or has_python_requirements(t)):
return
interpreter = self.context.products.get_data(PythonInterpreter)
pex = self.resolve_requirements(
interpreter, self.context.targets(has_python_requirements))
self.context.products.register_data(self.REQUIREMENTS_PEX, pex)
def _resolve_requirements_for_versioned_target_closure(self, interpreter, vt):
reqs_pex_path = os.path.realpath(
os.path.join(self.workdir, str(interpreter.identity), vt.cache_key.hash)
)
if not os.path.isdir(reqs_pex_path):
req_libs = [t for t in vt.target.closure() if has_python_requirements(t)]
with safe_concurrent_creation(reqs_pex_path) as safe_path:
pex_builder = PexBuilderWrapper.Factory.create(
builder=PEXBuilder(safe_path, interpreter=interpreter, copy=True),
log=self.context.log,
)
pex_builder.add_requirement_libs_from(req_libs)
pex_builder.freeze()
return PEX(reqs_pex_path, interpreter=interpreter)
def _create_binary(self, binary_tgt, results_dir):
"""Create a .pex file for the specified binary target."""
# Note that we rebuild a chroot from scratch, instead of using the REQUIREMENTS_PEX
# and PYTHON_SOURCES products, because those products are already-built pexes, and there's
# no easy way to merge them into a single pex file (for example, they each have a __main__.py,
# metadata, and so on, which the merging code would have to handle specially).
interpreter = self.context.products.get_data(PythonInterpreter)
with temporary_dir() as tmpdir:
# Create the pex_info for the binary.
build_properties = PexInfo.make_build_properties()
if self.get_options().include_run_information:
run_info_dict = self.context.run_tracker.run_info.get_as_dict()
build_properties.update(run_info_dict)
pex_info = binary_tgt.pexinfo.copy()
pex_info.build_properties = build_properties
pex_builder = PexBuilderWrapper.Factory.create(
builder=PEXBuilder(path=tmpdir,
interpreter=interpreter,
pex_info=pex_info,
copy=True),
log=self.context.log,
generate_ipex=self._generate_ipex,
)
if binary_tgt.shebang:
self.context.log.info(
"Found Python binary target {} with customized shebang, using it: {}"
.format(binary_tgt.name, binary_tgt.shebang))
pex_builder.set_shebang(binary_tgt.shebang)
else:
self.context.log.debug(
f"No customized shebang found for {binary_tgt.name}")
# Find which targets provide sources and which specify requirements.
source_tgts = []
req_tgts = []
constraint_tgts = []
for tgt in binary_tgt.closure(exclude_scopes=Scopes.COMPILE):
if has_python_sources(tgt) or has_resources(tgt):
source_tgts.append(tgt)
elif has_python_requirements(tgt):
req_tgts.append(tgt)
if is_python_target(tgt):
constraint_tgts.append(tgt)
# Add interpreter compatibility constraints to pex info. Note that we only add the constraints for the final
# binary target itself, not its dependencies. The upstream interpreter selection tasks will already validate that
# there are no compatibility conflicts among the dependencies and target. If the binary target does not have
# `compatibility` in its BUILD entry, the global --python-setup-interpreter-constraints will be used.
pex_builder.add_interpreter_constraints_from([binary_tgt])
# Dump everything into the builder's chroot.
for tgt in source_tgts:
pex_builder.add_sources_from(tgt)
# We need to ensure that we are resolving for only the current platform if we are
# including local python dist targets that have native extensions.
self._python_native_code_settings.check_build_for_current_platform_only(
self.context.targets())
pex_builder.add_requirement_libs_from(
req_tgts, platforms=binary_tgt.platforms)
# Build the .pex file.
pex_filename = self._get_output_pex_filename(binary_tgt.name)
pex_path = os.path.join(results_dir, pex_filename)
pex_builder.build(pex_path)
return pex_path
def generate_targets_map(self, targets, classpath_products=None):
"""Generates a dictionary containing all pertinent information about the target graph.
The return dictionary is suitable for serialization by json.dumps.
:param targets: The list of targets to generate the map for.
:param classpath_products: Optional classpath_products. If not provided when the --libraries
option is `True`, this task will perform its own jar resolution.
"""
targets_map = {}
resource_target_map = {}
python_interpreter_targets_mapping = defaultdict(list)
if self.get_options().libraries:
# NB(gmalmquist): This supports mocking the classpath_products in tests.
if classpath_products is None:
classpath_products = self.resolve_jars(targets)
else:
classpath_products = None
target_roots_set = set(self.context.target_roots)
def process_target(current_target):
"""
:type current_target:pants.build_graph.target.Target
"""
def get_target_type(tgt):
def is_test(t):
return isinstance(t, JUnitTests) or isinstance(
t, PythonTests)
if is_test(tgt):
return SourceRootTypes.TEST
else:
if (isinstance(tgt, Resources)
and tgt in resource_target_map
and is_test(resource_target_map[tgt])):
return SourceRootTypes.TEST_RESOURCE
elif isinstance(tgt, Resources):
return SourceRootTypes.RESOURCE
else:
return SourceRootTypes.SOURCE
info = {
"targets": [],
"libraries": [],
"roots": [],
"id":
current_target.id,
"target_type":
get_target_type(current_target),
# NB: is_code_gen should be removed when export format advances to 1.1.0 or higher
"is_code_gen":
current_target.is_synthetic,
"is_synthetic":
current_target.is_synthetic,
"pants_target_type":
self._get_pants_target_alias(type(current_target)),
}
if not current_target.is_synthetic:
info["globs"] = current_target.globs_relative_to_buildroot()
if self.get_options().sources:
info["sources"] = list(
current_target.sources_relative_to_buildroot())
info["transitive"] = current_target.transitive
info["scope"] = str(current_target.scope)
info["is_target_root"] = current_target in target_roots_set
if isinstance(current_target, PythonRequirementLibrary):
reqs = current_target.payload.get_field_value(
"requirements", set())
""":type : set[pants.python.python_requirement.PythonRequirement]"""
info["requirements"] = [req.key for req in reqs]
if isinstance(current_target, PythonTarget):
interpreter_for_target = self._interpreter_cache.select_interpreter_for_targets(
[current_target])
if interpreter_for_target is None:
raise TaskError(
"Unable to find suitable interpreter for {}".format(
current_target.address))
python_interpreter_targets_mapping[
interpreter_for_target].append(current_target)
info["python_interpreter"] = str(
interpreter_for_target.identity)
def iter_transitive_jars(jar_lib):
"""
:type jar_lib: :class:`pants.backend.jvm.targets.jar_library.JarLibrary`
:rtype: :class:`collections.Iterator` of
:class:`pants.java.jar.M2Coordinate`
"""
if classpath_products:
jar_products = classpath_products.get_artifact_classpath_entries_for_targets(
(jar_lib, ))
for _, jar_entry in jar_products:
coordinate = jar_entry.coordinate
# We drop classifier and type_ since those fields are represented in the global
# libraries dict and here we just want the key into that dict (see `_jar_id`).
yield M2Coordinate(org=coordinate.org,
name=coordinate.name,
rev=coordinate.rev)
target_libraries = OrderedSet()
if isinstance(current_target, JarLibrary):
target_libraries = OrderedSet(
iter_transitive_jars(current_target))
for dep in current_target.dependencies:
info["targets"].append(dep.address.spec)
if isinstance(dep, JarLibrary):
for jar in dep.jar_dependencies:
target_libraries.add(
M2Coordinate(jar.org, jar.name, jar.rev))
# Add all the jars pulled in by this jar_library
target_libraries.update(iter_transitive_jars(dep))
if isinstance(dep, Resources):
resource_target_map[dep] = current_target
if isinstance(current_target, ScalaLibrary):
for dep in current_target.java_sources:
info["targets"].append(dep.address.spec)
process_target(dep)
if isinstance(current_target, JvmTarget):
info["excludes"] = [
self._exclude_id(exclude)
for exclude in current_target.excludes
]
info["platform"] = current_target.platform.name
if hasattr(current_target, "runtime_platform"):
info[
"runtime_platform"] = current_target.runtime_platform.name
info["roots"] = [{
"source_root": source_root_package_prefix[0],
"package_prefix": source_root_package_prefix[1],
} for source_root_package_prefix in self._source_roots_for_target(
current_target)]
if classpath_products:
info["libraries"] = [
self._jar_id(lib) for lib in target_libraries
]
targets_map[current_target.address.spec] = info
for target in targets:
process_target(target)
scala_platform = ScalaPlatform.global_instance()
scala_platform_map = {
"scala_version":
scala_platform.version,
"compiler_classpath": [
cp_entry.path
for cp_entry in scala_platform.compiler_classpath_entries(
self.context.products)
],
}
jvm_platforms_map = {
"default_platform":
JvmPlatform.global_instance().default_platform.name,
"platforms": {
str(platform_name): {
"target_level": str(platform.target_level),
"source_level": str(platform.source_level),
"args": platform.args,
}
for platform_name, platform in
JvmPlatform.global_instance().platforms_by_name.items()
},
}
graph_info = {
"version": DEFAULT_EXPORT_VERSION,
"targets": targets_map,
"jvm_platforms": jvm_platforms_map,
"scala_platform": scala_platform_map,
# `jvm_distributions` are static distribution settings from config,
# `preferred_jvm_distributions` are distributions that pants actually uses for the
# given platform setting.
"preferred_jvm_distributions": {},
}
for platform_name, platform in JvmPlatform.global_instance(
).platforms_by_name.items():
preferred_distributions = {}
for strict, strict_key in [(True, "strict"),
(False, "non_strict")]:
try:
dist = JvmPlatform.preferred_jvm_distribution(
[platform], strict=strict)
preferred_distributions[strict_key] = dist.home
except DistributionLocator.Error:
pass
if preferred_distributions:
graph_info["preferred_jvm_distributions"][
platform_name] = preferred_distributions
if classpath_products:
graph_info["libraries"] = self._resolve_jars_info(
targets, classpath_products)
if python_interpreter_targets_mapping:
# NB: We've selected a python interpreter compatible with each python target individually into
# the `python_interpreter_targets_mapping`. These python targets may not be compatible, ie: we
# could have a python target requiring 'CPython>=2.7<3' (ie: CPython-2.7.x) and another
# requiring 'CPython>=3.6'. To pick a default interpreter then from among these two choices
# is arbitrary and not to be relied on to work as a default interpreter if ever needed by the
# export consumer.
#
# TODO(John Sirois): consider either eliminating the 'default_interpreter' field and pressing
# export consumers to make their own choice of a default (if needed) or else use
# `select.select_interpreter_for_targets` and fail fast if there is no interpreter compatible
# across all the python targets in-play.
#
# For now, make our arbitrary historical choice of a default interpreter explicit and use the
# lowest version.
default_interpreter = min(
python_interpreter_targets_mapping.keys())
interpreters_info = {}
for interpreter, targets in python_interpreter_targets_mapping.items(
):
req_libs = [
target for target in Target.closure_for_targets(targets)
if has_python_requirements(target)
]
chroot = self.resolve_requirements(interpreter, req_libs)
interpreters_info[str(interpreter.identity)] = {
"binary": interpreter.binary,
"chroot": chroot.path(),
}
graph_info["python_setup"] = {
"default_interpreter": str(default_interpreter.identity),
"interpreters": interpreters_info,
}
if self.get_options().available_target_types:
graph_info["available_target_types"] = self._target_types()
return graph_info