def test_multiple(self): subclasses_of_b_or_c = SubclassesOf(self.B, self.C) self.assertEqual((self.B, self.C), subclasses_of_b_or_c.types) self.assertTrue(subclasses_of_b_or_c.satisfied_by(self.B())) self.assertTrue(subclasses_of_b_or_c.satisfied_by(self.C())) self.assertFalse(subclasses_of_b_or_c.satisfied_by(self.BPrime())) self.assertFalse(subclasses_of_b_or_c.satisfied_by(self.A()))
def test_single(self): subclasses_of_b = SubclassesOf(self.B) self.assertEqual((self.B,), subclasses_of_b.types) self.assertFalse(subclasses_of_b.satisfied_by(self.A())) self.assertTrue(subclasses_of_b.satisfied_by(self.B())) self.assertFalse(subclasses_of_b.satisfied_by(self.BPrime())) self.assertTrue(subclasses_of_b.satisfied_by(self.C()))
def test_validate(self):
subclasses_of_a_or_b = SubclassesOf(self.A, self.B)
self.assertEqual(self.A(), subclasses_of_a_or_b.validate_satisfied_by(self.A()))
self.assertEqual(self.B(), subclasses_of_a_or_b.validate_satisfied_by(self.B()))
self.assertEqual(self.C(), subclasses_of_a_or_b.validate_satisfied_by(self.C()))
with self.assertRaisesWithMessage(
TypeConstraintError,
"value 1 (with type 'int') must satisfy this type constraint: SubclassesOf(A or B)."):
subclasses_of_a_or_b.validate_satisfied_by(1)
def test_str_and_repr(self):
collection_of_exactly_b = TypedCollection(Exactly(self.B))
self.assertEqual("TypedCollection(Exactly(B))", str(collection_of_exactly_b))
self.assertEqual("TypedCollection(Exactly(B))", repr(collection_of_exactly_b))
collection_of_multiple_subclasses = TypedCollection(
SubclassesOf(self.A, self.B))
self.assertEqual("TypedCollection(SubclassesOf(A or B))",
str(collection_of_multiple_subclasses))
self.assertEqual("TypedCollection(SubclassesOf(A, B))",
repr(collection_of_multiple_subclasses))
class NativeCompileRequest(datatype([
('compiler', SubclassesOf(Executable)),
# TODO: add type checking for Collection.of(<type>)!
'include_dirs',
'sources',
('fatal_warnings', bool),
'output_dir',
])): pass
# FIXME(#5950): perform all process execution in the v2 engine!
class ObjectFiles(datatype(['root_dir', 'filenames'])):
class Target(Struct, HasProducts):
def __init__(self, name=None, configurations=None, **kwargs):
super(Target, self).__init__(name=name, **kwargs)
self.configurations = configurations
@property
def products(self):
return self.configurations
@addressable_list(SubclassesOf(Struct))
def configurations(self):
pass
def test_single(self): subclasses_of_b = SubclassesOf(self.B) self.assertEqual((self.B,), subclasses_of_b.types) self.assertFalse(subclasses_of_b.satisfied_by(self.A())) self.assertTrue(subclasses_of_b.satisfied_by(self.B())) self.assertFalse(subclasses_of_b.satisfied_by(self.BPrime())) self.assertTrue(subclasses_of_b.satisfied_by(self.C()))
class StructWithDeps(Struct):
"""A subclass of Struct with dependencies."""
def __init__(self, dependencies=None, **kwargs):
"""
:param list dependencies: The direct dependencies of this struct.
"""
# TODO: enforce the type of variants using the Addressable framework.
super(StructWithDeps, self).__init__(**kwargs)
self.dependencies = dependencies
@addressable_list(SubclassesOf(Struct))
def dependencies(self):
"""The direct dependencies of this target.
class ExecuteProcessRequest(
datatype([
('argv', tuple),
('env', tuple),
('input_files', DirectoryDigest),
('output_files', tuple),
('output_directories', tuple),
# NB: timeout_seconds covers the whole remote operation including queuing and setup.
('timeout_seconds', Exactly(float, int)),
('description', SubclassesOf(*six.string_types)),
])):
"""Request for execution with args and snapshots to extract."""
@classmethod
def create_from_snapshot(cls,
argv,
env,
snapshot,
output_files=(),
output_directories=(),
timeout_seconds=_default_timeout_seconds,
description='process'):
cls._verify_env_is_dict(env)
return ExecuteProcessRequest(
argv=argv,
env=tuple(env.items()),
input_files=snapshot.directory_digest,
output_files=output_files,
output_directories=output_directories,
timeout_seconds=timeout_seconds,
description=description,
)
@classmethod
def create_with_empty_snapshot(cls,
argv,
env,
output_files=(),
output_directories=(),
timeout_seconds=_default_timeout_seconds,
description='process'):
return cls.create_from_snapshot(argv, env, EMPTY_SNAPSHOT,
output_files, output_directories,
timeout_seconds, description)
@classmethod
def _verify_env_is_dict(cls, env):
if not isinstance(env, dict):
raise TypeCheckError(
cls.__name__,
"arg 'env' was invalid: value {} (with type {}) must be a dict"
.format(env, type(env)))
def test_validate(self):
subclasses_of_a_or_b = SubclassesOf(self.A, self.B)
self.assertEqual(self.A(), subclasses_of_a_or_b.validate_satisfied_by(self.A()))
self.assertEqual(self.B(), subclasses_of_a_or_b.validate_satisfied_by(self.B()))
self.assertEqual(self.C(), subclasses_of_a_or_b.validate_satisfied_by(self.C()))
with self.assertRaisesWithMessage(
TypeConstraintError,
"value 1 (with type 'int') must satisfy this type constraint: SubclassesOf(A or B)."):
subclasses_of_a_or_b.validate_satisfied_by(1)
class ParseArgsRequest(
datatype([
('flag_value_map', SubclassesOf(dict)),
'namespace',
'get_all_scoped_flag_names',
('levenshtein_max_distance', int),
])):
@staticmethod
def _create_flag_value_map(flags):
"""Returns a map of flag -> list of values, based on the given flag strings.
None signals no value given (e.g., -x, --foo).
The value is a list because the user may specify the same flag multiple times, and that's
sometimes OK (e.g., when appending to list-valued options).
"""
flag_value_map = defaultdict(list)
for flag in flags:
key, has_equals_sign, flag_val = flag.partition('=')
if not has_equals_sign:
if not flag.startswith('--'): # '-xfoo' style.
key = flag[0:2]
flag_val = flag[2:]
if not flag_val:
# Either a short option with no value or a long option with no equals sign.
# Important so we can distinguish between no value ('--foo') and setting to an empty
# string ('--foo='), for options with an implicit_value.
flag_val = None
flag_value_map[key].append(flag_val)
return flag_value_map
def __new__(cls, flags_in_scope, namespace, get_all_scoped_flag_names,
levenshtein_max_distance):
"""
:param Iterable flags_in_scope: Iterable of arg strings to parse into flag values.
:param namespace: The object to register the flag values on
:param function get_all_scoped_flag_names: A 0-argument function which returns an iterable of
all registered option names in all their scopes. This
is used to create an error message with suggestions
when raising a `ParseError`.
:param int levenshtein_max_distance: The maximum Levenshtein edit distance between option names
to determine similarly named options when an option name
hasn't been registered.
"""
flag_value_map = cls._create_flag_value_map(flags_in_scope)
return super(Parser.ParseArgsRequest,
cls).__new__(cls, flag_value_map, namespace,
get_all_scoped_flag_names,
levenshtein_max_distance)
class UnpackJars(UnpackRemoteSourcesBase):
"""Unpack artifacts specified by unpacked_jars() targets.
Adds an entry to SourceRoot for the contents.
:API: public
"""
source_target_constraint = SubclassesOf(UnpackedJars)
@classmethod
def prepare(cls, options, round_manager):
super().prepare(options, round_manager)
round_manager.require_data(JarImportProducts)
@classmethod
def implementation_version(cls):
return super().implementation_version() + [("UnpackJars", 0)]
def get_fingerprint_strategy(self):
return UnpackJarsFingerprintStrategy()
def unpack_target(self, unpacked_jars, unpack_dir):
deprecated_conditional(
lambda: True,
removal_version="1.31.0.dev0",
entity_description="The `unpack-jars` goal",
hint_message=
"Contact the Pants team on Slack or [email protected] "
"if you need this functionality.",
)
direct_coords = {
jar.coordinate
for jar in unpacked_jars.all_imported_jar_deps
}
unpack_filter = self.get_unpack_filter(unpacked_jars)
jar_import_products = self.context.products.get_data(JarImportProducts)
for coordinate, jar_path in jar_import_products.imports(unpacked_jars):
if not unpacked_jars.payload.intransitive or coordinate in direct_coords:
self.context.log.info(
"Unpacking jar {coordinate} from {jar_path} to {unpack_dir}."
.format(coordinate=coordinate,
jar_path=jar_path,
unpack_dir=unpack_dir))
ZIP.extract(jar_path, unpack_dir, filter_func=unpack_filter)
class TaskRule(datatype([
('output_type', _type_field),
('input_selectors', TypedCollection(SubclassesOf(type))),
('input_gets', tuple),
'func',
('dependency_rules', tuple),
('dependency_optionables', tuple),
('cacheable', bool),
]), Rule):
"""A Rule that runs a task function when all of its input selectors are satisfied.
NB: This API is experimental, and not meant for direct consumption. To create a `TaskRule` you
should always prefer the `@rule` constructor, and in cases where that is too constraining
(likely due to #4535) please bump or open a ticket to explain the usecase.
"""
def __new__(cls,
output_type,
input_selectors,
func,
input_gets,
dependency_optionables=None,
dependency_rules=None,
cacheable=True):
# Create.
return super().__new__(
cls,
output_type,
input_selectors,
input_gets,
func,
dependency_rules or tuple(),
dependency_optionables or tuple(),
cacheable,
)
def __str__(self):
return ('({}, {!r}, {}, gets={}, opts={})'
.format(self.output_type.__name__,
self.input_selectors,
self.func.__name__,
self.input_gets,
self.dependency_optionables))
class Target(Struct, HasProducts):
"""A placeholder for the most-numerous Struct subclass.
This particular implementation holds a collection of other Structs in a `configurations` field.
"""
def __init__(self, name=None, configurations=None, **kwargs):
"""
:param string name: The name of this target which forms its address in its namespace.
:param list configurations: The configurations that apply to this target in various contexts.
"""
super(Target, self).__init__(name=name, **kwargs)
self.configurations = configurations
@property
def products(self):
return self.configurations
@addressable_list(SubclassesOf(Struct))
def configurations(self):
"""The configurations that apply to this target in various contexts.
def __init__(self, native, build_root, work_dir, ignore_patterns,
rule_index):
self._native = native
# TODO: The only (?) case where we use inheritance rather than exact type unions.
has_products_constraint = SubclassesOf(HasProducts)
self._root_subject_types = sorted(rule_index.roots)
# Create the ExternContext, and the native Scheduler.
self._tasks = native.new_tasks()
self._register_rules(rule_index)
self._scheduler = native.new_scheduler(
self._tasks,
self._root_subject_types,
build_root,
work_dir,
ignore_patterns,
Snapshot,
FileContent,
FilesContent,
Path,
Dir,
File,
Link,
ExecuteProcessResult,
has_products_constraint,
constraint_for(Address),
constraint_for(Variants),
constraint_for(PathGlobs),
constraint_for(Snapshot),
constraint_for(FilesContent),
constraint_for(Dir),
constraint_for(File),
constraint_for(Link),
constraint_for(ExecuteProcessRequest),
constraint_for(ExecuteProcessResult),
constraint_for(GeneratorType),
)
class UnpackJars(UnpackRemoteSourcesBase):
"""Unpack artifacts specified by unpacked_jars() targets.
Adds an entry to SourceRoot for the contents.
:API: public
"""
source_target_constraint = SubclassesOf(UnpackedJars)
@classmethod
def prepare(cls, options, round_manager):
super().prepare(options, round_manager)
round_manager.require_data(JarImportProducts)
@classmethod
def implementation_version(cls):
return super().implementation_version() + [("UnpackJars", 0)]
def get_fingerprint_strategy(self):
return UnpackJarsFingerprintStrategy()
def unpack_target(self, unpacked_jars, unpack_dir):
direct_coords = {
jar.coordinate
for jar in unpacked_jars.all_imported_jar_deps
}
unpack_filter = self.get_unpack_filter(unpacked_jars)
jar_import_products = self.context.products.get_data(JarImportProducts)
for coordinate, jar_path in jar_import_products.imports(unpacked_jars):
if not unpacked_jars.payload.intransitive or coordinate in direct_coords:
self.context.log.info(
"Unpacking jar {coordinate} from {jar_path} to {unpack_dir}."
.format(coordinate=coordinate,
jar_path=jar_path,
unpack_dir=unpack_dir))
ZIP.extract(jar_path, unpack_dir, filter_func=unpack_filter)
class CCompile(NativeCompile):
options_scope = "c-compile"
# Compile only C library targets.
source_target_constraint = SubclassesOf(CLibrary)
workunit_label = "c-compile"
@classmethod
def implementation_version(cls):
return super().implementation_version() + [("CCompile", 0)]
@classmethod
def subsystem_dependencies(cls):
return super().subsystem_dependencies() + (
CCompileSettings.scoped(cls), )
def get_compile_settings(self):
return CCompileSettings.scoped_instance(self)
def get_compiler(self, native_library_target):
return self.get_c_toolchain_variant(native_library_target).c_compiler
class CppCompile(NativeCompile):
options_scope = 'cpp-compile'
# Compile only C++ library targets.
source_target_constraint = SubclassesOf(CppLibrary)
workunit_label = 'cpp-compile'
@classmethod
def implementation_version(cls):
return super(CppCompile,
cls).implementation_version() + [('CppCompile', 0)]
@classmethod
def subsystem_dependencies(cls):
return super(CppCompile, cls).subsystem_dependencies() + (
CppCompileSettings.scoped(cls), )
def get_compile_settings(self):
return CppCompileSettings.scoped_instance(self)
def get_compiler(self):
return self.get_cpp_toolchain_variant().cpp_compiler
class NativeCompile(NativeTask, AbstractClass):
# `NativeCompile` will use the `source_target_constraint` to determine what targets have "sources"
# to compile, and the `dependent_target_constraint` to determine which dependent targets to
# operate on for `strict_deps` calculation.
# NB: `source_target_constraint` must be overridden.
source_target_constraint = None
dependent_target_constraint = SubclassesOf(NativeLibrary)
# `NativeCompile` will use `workunit_label` as the name of the workunit when executing the
# compiler process. `workunit_label` must be set to a string.
workunit_label = None
@classmethod
def product_types(cls):
return [ObjectFiles, NativeTargetDependencies]
@property
def cache_target_dirs(self):
return True
@abstractmethod
def get_compile_settings(self):
"""An instance of `NativeCompileSettings` which is used in `NativeCompile`.
:return: :class:`pants.backend.native.subsystems.native_compile_settings.NativeCompileSettings`
"""
@memoized_property
def _compile_settings(self):
return self.get_compile_settings()
@classmethod
def implementation_version(cls):
return super(NativeCompile,
cls).implementation_version() + [('NativeCompile', 0)]
class NativeCompileError(TaskError):
"""Raised for errors in this class's logic.
Subclasses are advised to create their own exception class.
"""
def native_deps(self, target):
return self.strict_deps_for_target(
target, predicate=self.dependent_target_constraint.satisfied_by)
def strict_deps_for_target(self, target, predicate=None):
"""Get the dependencies of `target` filtered by `predicate`, accounting for 'strict_deps'.
If 'strict_deps' is on, instead of using the transitive closure of dependencies, targets will
only be able to see their immediate dependencies declared in the BUILD file. The 'strict_deps'
setting is obtained from the result of `get_compile_settings()`.
NB: This includes the current target in the result.
"""
if self._compile_settings.get_subsystem_target_mirrored_field_value(
'strict_deps', target):
strict_deps = target.strict_dependencies(DependencyContext())
if predicate:
filtered_deps = filter(predicate, strict_deps)
else:
filtered_deps = strict_deps
deps = [target] + filtered_deps
else:
deps = self.context.build_graph.transitive_subgraph_of_addresses(
[target.address], predicate=predicate)
return deps
@staticmethod
def _add_product_at_target_base(product_mapping, target, value):
product_mapping.add(target, target.target_base).append(value)
def execute(self):
object_files_product = self.context.products.get(ObjectFiles)
native_deps_product = self.context.products.get(
NativeTargetDependencies)
source_targets = self.context.targets(
self.source_target_constraint.satisfied_by)
with self.invalidated(
source_targets,
invalidate_dependents=True) as invalidation_check:
for vt in invalidation_check.invalid_vts:
deps = self.native_deps(vt.target)
self._add_product_at_target_base(native_deps_product,
vt.target, deps)
compile_request = self._make_compile_request(vt, deps)
self.context.log.debug(
"compile_request: {}".format(compile_request))
self._compile(compile_request)
for vt in invalidation_check.all_vts:
object_files = self.collect_cached_objects(vt)
self._add_product_at_target_base(object_files_product,
vt.target, object_files)
# This may be calculated many times for a target, so we memoize it.
@memoized_method
def _include_dirs_for_target(self, target):
return target.sources_relative_to_target_base().rel_root
class NativeSourcesByType(datatype(['rel_root', 'headers', 'sources'])):
pass
def get_sources_headers_for_target(self, target):
"""Return a list of file arguments to provide to the compiler.
NB: result list will contain both header and source files!
:raises: :class:`NativeCompile.NativeCompileError` if there is an error processing the sources.
"""
# Get source paths relative to the target base so the exception message with the target and
# paths makes sense.
target_relative_sources = target.sources_relative_to_target_base()
rel_root = target_relative_sources.rel_root
# Unique file names are required because we just dump object files into a single directory, and
# the compiler will silently just produce a single object file if provided non-unique filenames.
# FIXME: add some shading to file names so we can remove this check.
# NB: It shouldn't matter if header files have the same name, but this will raise an error in
# that case as well. We won't need to do any shading of header file names.
seen_filenames = defaultdict(list)
for src in target_relative_sources:
seen_filenames[os.path.basename(src)].append(src)
duplicate_filename_err_msgs = []
for fname, source_paths in seen_filenames.items():
if len(source_paths) > 1:
duplicate_filename_err_msgs.append(
"filename: {}, paths: {}".format(fname, source_paths))
if duplicate_filename_err_msgs:
raise self.NativeCompileError(
"Error in target '{}': source files must have a unique filename within a '{}' target. "
"Conflicting filenames:\n{}".format(
target.address.spec, target.alias(),
'\n'.join(duplicate_filename_err_msgs)))
return [os.path.join(rel_root, src) for src in target_relative_sources]
# FIXME(#5951): expand `Executable` to cover argv generation (where an `Executable` is subclassed
# to modify or extend the argument list, as declaratively as possible) to remove
# `extra_compile_args(self)`!
@abstractmethod
def get_compiler(self):
"""An instance of `Executable` which can be invoked to compile files.
:return: :class:`pants.backend.native.config.environment.Executable`
"""
@memoized_property
def _compiler(self):
return self.get_compiler()
def _make_compile_request(self, versioned_target, dependencies):
target = versioned_target.target
include_dirs = [
self._include_dirs_for_target(dep_tgt) for dep_tgt in dependencies
]
sources_and_headers = self.get_sources_headers_for_target(target)
return NativeCompileRequest(compiler=self._compiler,
include_dirs=include_dirs,
sources=sources_and_headers,
fatal_warnings=self._compile_settings.
get_subsystem_target_mirrored_field_value(
'fatal_warnings', target),
output_dir=versioned_target.results_dir)
@abstractmethod
def extra_compile_args(self):
"""Return a list of task-specific arguments to use to compile sources."""
def _make_compile_argv(self, compile_request):
"""Return a list of arguments to use to compile sources. Subclasses can override and append."""
compiler = compile_request.compiler
err_flags = ['-Werror'] if compile_request.fatal_warnings else []
platform_specific_flags = compiler.platform.resolve_platform_specific({
'linux':
lambda: [],
'darwin':
lambda: ['-mmacosx-version-min=10.11'],
})
# We are going to execute in the target output, so get absolute paths for everything.
# TODO: If we need to produce static libs, don't add -fPIC! (could use Variants -- see #5788).
argv = ([compiler.exe_filename] + platform_specific_flags +
self.extra_compile_args() + err_flags + ['-c', '-fPIC'] + [
'-I{}'.format(os.path.abspath(inc_dir))
for inc_dir in compile_request.include_dirs
] + [os.path.abspath(src) for src in compile_request.sources])
return argv
def _compile(self, compile_request):
"""Perform the process of compilation, writing object files to the request's 'output_dir'.
NB: This method must arrange the output files so that `collect_cached_objects()` can collect all
of the results (or vice versa)!
"""
sources = compile_request.sources
if len(sources) == 0:
# TODO: do we need this log message? Should we still have it for intentionally header-only
# libraries (that might be a confusing message to see)?
self.context.log.debug(
"no sources in request {}, skipping".format(compile_request))
return
compiler = compile_request.compiler
output_dir = compile_request.output_dir
argv = self._make_compile_argv(compile_request)
with self.context.new_workunit(name=self.workunit_label,
labels=[WorkUnitLabel.COMPILER
]) as workunit:
try:
process = subprocess.Popen(
argv,
cwd=output_dir,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'),
env={'PATH': get_joined_path(compiler.path_entries)})
except OSError as e:
workunit.set_outcome(WorkUnit.FAILURE)
raise self.NativeCompileError(
"Error invoking '{exe}' with command {cmd} for request {req}: {err}"
.format(exe=compiler.exe_filename,
cmd=argv,
req=compile_request,
err=e))
rc = process.wait()
if rc != 0:
workunit.set_outcome(WorkUnit.FAILURE)
raise self.NativeCompileError(
"Error in '{section_name}' with command {cmd} for request {req}. Exit code was: {rc}."
.format(section_name=self.workunit_name,
cmd=argv,
req=compile_request,
rc=rc))
def collect_cached_objects(self, versioned_target):
"""Scan `versioned_target`'s results directory and return the output files from that directory.
:return: :class:`ObjectFiles`
"""
return ObjectFiles(versioned_target.results_dir,
os.listdir(versioned_target.results_dir))
def test_none(self):
with self.assertRaises(ValueError):
SubclassesOf()
class WithSubclassTypeConstraint(datatype([('some_value', SubclassesOf(SomeBaseClass))])): pass
class NonNegativeInt(datatype([('an_int', int)])):
class UnpackWheels(UnpackRemoteSourcesBase):
"""Extract native code from `NativePythonWheel` targets for use by downstream C/C++ sources."""
source_target_constraint = SubclassesOf(UnpackedWheels)
def get_fingerprint_strategy(self):
return UnpackWheelsFingerprintStrategy()
@classmethod
def subsystem_dependencies(cls):
return super().subsystem_dependencies() + (
PexBuilderWrapper.Factory,
PythonInterpreterCache,
PythonSetup,
)
def _get_matching_wheel(self, pex_path, interpreter, requirements,
module_name):
"""Use PexBuilderWrapper to resolve a single wheel from the requirement specs using pex.
N.B.: The resolved wheel is already "unpacked" by PEX. More accurately, it's installed in a
chroot.
"""
with self.context.new_workunit("extract-native-wheels"):
with safe_concurrent_creation(pex_path) as chroot:
pex_builder = PexBuilderWrapper.Factory.create(
builder=PEXBuilder(path=chroot, interpreter=interpreter),
log=self.context.log)
return pex_builder.extract_single_dist_for_current_platform(
requirements, dist_key=module_name)
@memoized_method
def _compatible_interpreter(self, unpacked_whls):
constraints = PythonSetup.global_instance(
).compatibility_or_constraints(unpacked_whls.compatibility)
allowable_interpreters = PythonInterpreterCache.global_instance(
).setup(filters=constraints)
return min(allowable_interpreters)
class WheelUnpackingError(TaskError):
pass
def unpack_target(self, unpacked_whls, unpack_dir):
interpreter = self._compatible_interpreter(unpacked_whls)
with temporary_dir() as resolve_dir:
try:
matched_dist = self._get_matching_wheel(
resolve_dir,
interpreter,
unpacked_whls.all_imported_requirements,
unpacked_whls.module_name,
)
wheel_chroot = matched_dist.location
if unpacked_whls.within_data_subdir:
# N.B.: Wheels with data dirs have the data installed under the top module.
dist_data_dir = os.path.join(wheel_chroot,
unpacked_whls.module_name)
else:
dist_data_dir = wheel_chroot
unpack_filter = self.get_unpack_filter(unpacked_whls)
# Copy over the module's data files into `unpack_dir`.
mergetree(dist_data_dir, unpack_dir, file_filter=unpack_filter)
except Exception as e:
raise self.WheelUnpackingError(
"Error extracting wheel for target {}: {}".format(
unpacked_whls, str(e)), e)
class NativeCompile(NativeTask, AbstractClass):
# `NativeCompile` will use the `source_target_constraint` to determine what targets have "sources"
# to compile, and the `dependent_target_constraint` to determine which dependent targets to
# operate on for `strict_deps` calculation.
# NB: `source_target_constraint` must be overridden.
source_target_constraint = None
dependent_target_constraint = SubclassesOf(ExternalNativeLibrary, NativeLibrary)
# `NativeCompile` will use `workunit_label` as the name of the workunit when executing the
# compiler process. `workunit_label` must be set to a string.
@classproperty
def workunit_label(cls):
raise NotImplementedError('subclasses of NativeCompile must override workunit_label!')
@classmethod
def product_types(cls):
return [ObjectFiles]
@classmethod
def prepare(cls, options, round_manager):
super(NativeCompile, cls).prepare(options, round_manager)
round_manager.optional_data(NativeExternalLibraryFiles)
@property
def cache_target_dirs(self):
return True
@classmethod
def implementation_version(cls):
return super(NativeCompile, cls).implementation_version() + [('NativeCompile', 1)]
class NativeCompileError(TaskError):
"""Raised for errors in this class's logic.
Subclasses are advised to create their own exception class.
"""
def execute(self):
object_files_product = self.context.products.get(ObjectFiles)
external_libs_product = self.context.products.get_data(NativeExternalLibraryFiles)
source_targets = self.context.targets(self.source_target_constraint.satisfied_by)
with self.invalidated(source_targets, invalidate_dependents=True) as invalidation_check:
for vt in invalidation_check.all_vts:
deps = self.native_deps(vt.target)
if not vt.valid:
compile_request = self._make_compile_request(vt, deps, external_libs_product)
self.context.log.debug("compile_request: {}".format(compile_request))
self._compile(compile_request)
object_files = self.collect_cached_objects(vt)
self._add_product_at_target_base(object_files_product, vt.target, object_files)
# This may be calculated many times for a target, so we memoize it.
@memoized_method
def _include_dirs_for_target(self, target):
return os.path.join(get_buildroot(), target.address.spec_path)
class NativeSourcesByType(datatype(['rel_root', 'headers', 'sources'])): pass
def get_sources_headers_for_target(self, target):
"""Return a list of file arguments to provide to the compiler.
NB: result list will contain both header and source files!
:raises: :class:`NativeCompile.NativeCompileError` if there is an error processing the sources.
"""
# Get source paths relative to the target base so the exception message with the target and
# paths makes sense.
target_relative_sources = target.sources_relative_to_target_base()
rel_root = target_relative_sources.rel_root
# Unique file names are required because we just dump object files into a single directory, and
# the compiler will silently just produce a single object file if provided non-unique filenames.
# TODO: add some shading to file names so we can remove this check.
# NB: It shouldn't matter if header files have the same name, but this will raise an error in
# that case as well. We won't need to do any shading of header file names.
seen_filenames = defaultdict(list)
for src in target_relative_sources:
seen_filenames[os.path.basename(src)].append(src)
duplicate_filename_err_msgs = []
for fname, source_paths in seen_filenames.items():
if len(source_paths) > 1:
duplicate_filename_err_msgs.append("filename: {}, paths: {}".format(fname, source_paths))
if duplicate_filename_err_msgs:
raise self.NativeCompileError(
"Error in target '{}': source files must have a unique filename within a '{}' target. "
"Conflicting filenames:\n{}"
.format(target.address.spec, target.alias(), '\n'.join(duplicate_filename_err_msgs)))
return [os.path.join(get_buildroot(), rel_root, src) for src in target_relative_sources]
@abstractmethod
def get_compile_settings(self):
"""Return a subclass of NativeBuildStepSettingsBase.
NB: Subclasses will be queried for the compile settings once and the result cached.
"""
@memoized_property
def _compile_settings(self):
return self.get_compile_settings()
@abstractmethod
def get_compiler(self):
"""An instance of `Executable` which can be invoked to compile files.
NB: Subclasses will be queried for the compiler instance once and the result cached.
:return: :class:`pants.backend.native.config.environment.Executable`
"""
@memoized_property
def _compiler(self):
return self.get_compiler()
def _get_third_party_include_dirs(self, external_libs_product, dependencies):
if not external_libs_product:
return []
return [nelf.include_dir
for nelf in external_libs_product.get_for_targets(dependencies)
if nelf.include_dir]
def _make_compile_request(self, versioned_target, dependencies, external_libs_product):
target = versioned_target.target
include_dirs = [self._include_dirs_for_target(dep_tgt) for dep_tgt in dependencies]
include_dirs.extend(self._get_third_party_include_dirs(external_libs_product, dependencies))
sources_and_headers = self.get_sources_headers_for_target(target)
return NativeCompileRequest(
compiler=self._compiler,
include_dirs=include_dirs,
sources=sources_and_headers,
fatal_warnings=self._compile_settings.get_fatal_warnings_value_for_target(target),
output_dir=versioned_target.results_dir)
def _make_compile_argv(self, compile_request):
"""Return a list of arguments to use to compile sources. Subclasses can override and append."""
compiler = compile_request.compiler
err_flags = ['-Werror'] if compile_request.fatal_warnings else []
# We are going to execute in the target output, so get absolute paths for everything.
buildroot = get_buildroot()
argv = (
[compiler.exe_filename] +
compiler.extra_args +
err_flags +
# TODO: If we need to produce static libs, don't add -fPIC! (could use Variants -- see #5788).
['-c', '-fPIC'] +
[
'-I{}'.format(os.path.join(buildroot, inc_dir))
for inc_dir in compile_request.include_dirs
] +
[os.path.join(buildroot, src) for src in compile_request.sources])
self.context.log.debug("compile argv: {}".format(argv))
return argv
def _compile(self, compile_request):
"""Perform the process of compilation, writing object files to the request's 'output_dir'.
NB: This method must arrange the output files so that `collect_cached_objects()` can collect all
of the results (or vice versa)!
"""
sources = compile_request.sources
if len(sources) == 0:
# TODO: do we need this log message? Should we still have it for intentionally header-only
# libraries (that might be a confusing message to see)?
self.context.log.debug("no sources in request {}, skipping".format(compile_request))
return
compiler = compile_request.compiler
output_dir = compile_request.output_dir
argv = self._make_compile_argv(compile_request)
env = compiler.as_invocation_environment_dict
with self.context.new_workunit(
name=self.workunit_label, labels=[WorkUnitLabel.COMPILER]) as workunit:
try:
process = subprocess.Popen(
argv,
cwd=output_dir,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'),
env=env)
except OSError as e:
workunit.set_outcome(WorkUnit.FAILURE)
raise self.NativeCompileError(
"Error invoking '{exe}' with command {cmd} and environment {env} for request {req}: {err}"
.format(exe=compiler.exe_filename, cmd=argv, env=env, req=compile_request, err=e))
rc = process.wait()
if rc != 0:
workunit.set_outcome(WorkUnit.FAILURE)
raise self.NativeCompileError(
"Error in '{section_name}' with command {cmd} and environment {env} for request {req}. "
"Exit code was: {rc}."
.format(section_name=self.workunit_label, cmd=argv, env=env, req=compile_request, rc=rc))
def collect_cached_objects(self, versioned_target):
"""Scan `versioned_target`'s results directory and return the output files from that directory.
:return: :class:`ObjectFiles`
"""
return ObjectFiles(versioned_target.results_dir, os.listdir(versioned_target.results_dir))
class PythonNativeCode(Subsystem):
"""A subsystem which exposes components of the native backend to the python backend."""
options_scope = 'python-native-code'
default_native_source_extensions = ['.c', '.cpp', '.cc']
class PythonNativeCodeError(Exception):
pass
@classmethod
def register_options(cls, register):
super(PythonNativeCode, cls).register_options(register)
register(
'--native-source-extensions',
type=list,
default=cls.default_native_source_extensions,
fingerprint=True,
advanced=True,
help=
'The extensions recognized for native source files in `python_dist()` sources.'
)
@classmethod
def subsystem_dependencies(cls):
return super(PythonNativeCode, cls).subsystem_dependencies() + (
NativeToolchain.scoped(cls),
PythonSetup.scoped(cls),
)
@memoized_property
def _native_source_extensions(self):
return self.get_options().native_source_extensions
@memoized_property
def native_toolchain(self):
return NativeToolchain.scoped_instance(self)
@memoized_property
def _python_setup(self):
return PythonSetup.scoped_instance(self)
def pydist_has_native_sources(self, target):
return target.has_sources(
extension=tuple(self._native_source_extensions))
def native_target_has_native_sources(self, target):
return target.has_sources()
@memoized_property
def _native_target_matchers(self):
return {
SubclassesOf(PythonDistribution): self.pydist_has_native_sources,
SubclassesOf(NativeLibrary): self.native_target_has_native_sources,
}
def _any_targets_have_native_sources(self, targets):
# TODO(#5949): convert this to checking if the closure of python requirements has any
# platform-specific packages (maybe find the platforms there too?).
for tgt in targets:
for type_constraint, target_predicate in self._native_target_matchers.items(
):
if type_constraint.satisfied_by(tgt) and target_predicate(tgt):
return True
return False
def get_targets_by_declared_platform(self, targets):
"""
Aggregates a dict that maps a platform string to a list of targets that specify the platform.
If no targets have platforms arguments, return a dict containing platforms inherited from
the PythonSetup object.
:param tgts: a list of :class:`Target` objects.
:returns: a dict mapping a platform string to a list of targets that specify the platform.
"""
targets_by_platforms = defaultdict(list)
for tgt in targets:
for platform in tgt.platforms:
targets_by_platforms[platform].append(tgt)
if not targets_by_platforms:
for platform in self._python_setup.platforms:
targets_by_platforms[platform] = [
'(No target) Platform inherited from either the '
'--platforms option or a pants.ini file.'
]
return targets_by_platforms
_PYTHON_PLATFORM_TARGETS_CONSTRAINT = SubclassesOf(PythonBinary,
PythonDistribution)
def check_build_for_current_platform_only(self, targets):
"""
Performs a check of whether the current target closure has native sources and if so, ensures
that Pants is only targeting the current platform.
:param tgts: a list of :class:`Target` objects.
:return: a boolean value indicating whether the current target closure has native sources.
:raises: :class:`pants.base.exceptions.IncompatiblePlatformsError`
"""
if not self._any_targets_have_native_sources(targets):
return False
targets_with_platforms = [
target for target in targets
if self._PYTHON_PLATFORM_TARGETS_CONSTRAINT.satisfied_by(target)
]
platforms_with_sources = self.get_targets_by_declared_platform(
targets_with_platforms)
platform_names = list(platforms_with_sources.keys())
if len(platform_names) < 1:
raise self.PythonNativeCodeError(
"Error: there should be at least one platform in the target closure, because "
"we checked that there are native sources.")
if platform_names == ['current']:
return True
raise IncompatiblePlatformsError(
'The target set contains one or more targets that depend on '
'native code. Please ensure that the platform arguments in all relevant targets and build '
'options are compatible with the current platform. Found targets for platforms: {}'
.format(str(platforms_with_sources)))
class CFFIExternMethodRuntimeErrorInfo(datatype([
('exc_type', type),
('exc_value', SubclassesOf(Exception)),
'traceback',
])):
"""Encapsulates an exception raised when a CFFI extern is called so that it can be displayed.
def _native_target_matchers(self):
return {
SubclassesOf(PythonDistribution): self.pydist_has_native_sources,
SubclassesOf(NativeLibrary): NativeLibrary.produces_ctypes_native_library,
}
def __init__(
self,
native,
project_tree,
work_dir,
rules,
execution_options,
include_trace_on_error=True,
validate=True,
):
"""
:param native: An instance of engine.native.Native.
:param project_tree: An instance of ProjectTree for the current build root.
:param work_dir: The pants work dir.
:param rules: A set of Rules which is used to compute values in the graph.
:param execution_options: Execution options for (remote) processes.
:param include_trace_on_error: Include the trace through the graph upon encountering errors.
:type include_trace_on_error: bool
:param validate: True to assert that the ruleset is valid.
"""
if execution_options.remote_execution_server and not execution_options.remote_store_server:
raise ValueError(
"Cannot set remote execution server without setting remote store server"
)
self._native = native
self.include_trace_on_error = include_trace_on_error
# TODO: The only (?) case where we use inheritance rather than exact type unions.
has_products_constraint = SubclassesOf(HasProducts)
# Validate and register all provided and intrinsic tasks.
rule_index = RuleIndex.create(list(rules))
self._root_subject_types = sorted(rule_index.roots)
# Create the native Scheduler and Session.
# TODO: This `_tasks` reference could be a local variable, since it is not used
# after construction.
self._tasks = native.new_tasks()
self._register_rules(rule_index)
self._scheduler = native.new_scheduler(
self._tasks,
self._root_subject_types,
project_tree.build_root,
work_dir,
project_tree.ignore_patterns,
execution_options,
DirectoryDigest,
Snapshot,
FileContent,
FilesContent,
Path,
Dir,
File,
Link,
FallibleExecuteProcessResult,
has_products_constraint,
constraint_for(Address),
constraint_for(Variants),
constraint_for(PathGlobs),
constraint_for(DirectoryDigest),
constraint_for(Snapshot),
constraint_for(FilesContent),
constraint_for(Dir),
constraint_for(File),
constraint_for(Link),
constraint_for(ExecuteProcessRequest),
constraint_for(FallibleExecuteProcessResult),
constraint_for(GeneratorType),
)
# If configured, visualize the rule graph before asserting that it is valid.
if self.visualize_to_dir() is not None:
rule_graph_name = 'rule_graph.dot'
self.visualize_rule_graph_to_file(
os.path.join(self.visualize_to_dir(), rule_graph_name))
if validate:
self._assert_ruleset_valid()
# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
import ast
from pants.util.objects import SubclassesOf, TypeConstraint, datatype
_type_field = SubclassesOf(type)
class Get(datatype([
('product', _type_field),
('subject_declared_type', _type_field),
'subject',
])):
"""Experimental synchronous generator API.
May be called equivalently as either:
# verbose form: Get(product_type, subject_declared_type, subject)
# shorthand form: Get(product_type, subject_type(subject))
"""
@staticmethod
def extract_constraints(call_node):
"""Parses a `Get(..)` call in one of its two legal forms to return its type constraints.
:param call_node: An `ast.Call` node representing a call to `Get(..)`.
:return: A tuple of product type id and subject type id.
"""
def render_args():
class WithSubclassTypeConstraint(
datatype([('some_value', SubclassesOf(SomeBaseClass))])):
pass
def test_multiple(self):
subclasses_of_b_or_c = SubclassesOf(self.B, self.C)
self.assertTrue(subclasses_of_b_or_c.satisfied_by(self.B()))
self.assertTrue(subclasses_of_b_or_c.satisfied_by(self.C()))
self.assertFalse(subclasses_of_b_or_c.satisfied_by(self.BPrime()))
self.assertFalse(subclasses_of_b_or_c.satisfied_by(self.A()))
def test_collection_multiple(self): collection_constraint = TypedCollection(SubclassesOf(self.B, self.BPrime)) self.assertTrue(collection_constraint.satisfied_by([self.B(), self.C(), self.BPrime()])) self.assertFalse(collection_constraint.satisfied_by([self.B(), self.A()]))
def test_none(self):
with self.assertRaisesWithMessage(ValueError,
'Must supply at least one type'):
SubclassesOf()
class LinkSharedLibraries(NativeTask):
options_scope = 'link-shared-libraries'
# TODO(#6486): change this to include ExternalNativeLibrary, then add a test that strict-deps
# works on external libs.
source_target_constraint = SubclassesOf(NativeLibrary)
@classmethod
def product_types(cls):
return [SharedLibrary]
@classmethod
def prepare(cls, options, round_manager):
super(LinkSharedLibraries, cls).prepare(options, round_manager)
round_manager.require(ObjectFiles)
round_manager.optional_product(NativeExternalLibraryFiles)
@property
def cache_target_dirs(self):
return True
@classmethod
def implementation_version(cls):
return super(LinkSharedLibraries, cls).implementation_version() + [('LinkSharedLibraries', 1)]
class LinkSharedLibrariesError(TaskError): pass
@memoized_property
def linker(self):
# NB: we are using the C++ toolchain here for linking every type of input, including compiled C
# source files.
return self.get_cpp_toolchain_variant().cpp_linker
@memoized_property
def platform(self):
# TODO: convert this to a v2 engine dependency injection.
return Platform.create()
def execute(self):
targets_providing_artifacts = self.context.targets(NativeLibrary.produces_ctypes_native_library)
compiled_objects_product = self.context.products.get(ObjectFiles)
shared_libs_product = self.context.products.get(SharedLibrary)
external_libs_product = self.context.products.get_data(NativeExternalLibraryFiles)
all_shared_libs_by_name = {}
with self.invalidated(targets_providing_artifacts,
invalidate_dependents=True) as invalidation_check:
for vt in invalidation_check.all_vts:
if vt.valid:
shared_library = self._retrieve_shared_lib_from_cache(vt)
else:
# TODO: We need to partition links based on proper dependency edges and not
# perform a link to every native_external_library for all targets in the closure.
# https://github.com/pantsbuild/pants/issues/6178
link_request = self._make_link_request(
vt, compiled_objects_product, external_libs_product)
self.context.log.debug("link_request: {}".format(link_request))
shared_library = self._execute_link_request(link_request)
same_name_shared_lib = all_shared_libs_by_name.get(shared_library.name, None)
if same_name_shared_lib:
# TODO: test this branch!
raise self.LinkSharedLibrariesError(
"The name '{name}' was used for two shared libraries: {prev} and {cur}."
.format(name=shared_library.name,
prev=same_name_shared_lib,
cur=shared_library))
else:
all_shared_libs_by_name[shared_library.name] = shared_library
self._add_product_at_target_base(shared_libs_product, vt.target, shared_library)
def _retrieve_shared_lib_from_cache(self, vt):
native_artifact = vt.target.ctypes_native_library
path_to_cached_lib = os.path.join(
vt.results_dir, native_artifact.as_shared_lib(self.platform))
if not os.path.isfile(path_to_cached_lib):
raise self.LinkSharedLibrariesError("The shared library at {} does not exist!"
.format(path_to_cached_lib))
return SharedLibrary(name=native_artifact.lib_name, path=path_to_cached_lib)
def _make_link_request(self, vt, compiled_objects_product, external_libs_product):
self.context.log.debug("link target: {}".format(vt.target))
deps = self.native_deps(vt.target)
all_compiled_object_files = []
for dep_tgt in deps:
if compiled_objects_product.get(dep_tgt):
self.context.log.debug("dep_tgt: {}".format(dep_tgt))
object_files = self._retrieve_single_product_at_target_base(compiled_objects_product, dep_tgt)
self.context.log.debug("object_files: {}".format(object_files))
object_file_paths = object_files.file_paths()
self.context.log.debug("object_file_paths: {}".format(object_file_paths))
all_compiled_object_files.extend(object_file_paths)
external_lib_dirs = []
external_lib_names = []
if external_libs_product is not None:
for nelf in external_libs_product.get_for_targets(deps):
if nelf.lib_dir:
external_lib_dirs.append(nelf.lib_dir)
external_lib_names.extend(nelf.lib_names)
link_request = LinkSharedLibraryRequest(
linker=self.linker,
object_files=tuple(all_compiled_object_files),
native_artifact=vt.target.ctypes_native_library,
output_dir=vt.results_dir,
external_lib_dirs=tuple(external_lib_dirs),
external_lib_names=tuple(external_lib_names))
self.context.log.debug(repr(link_request))
return link_request
_SHARED_CMDLINE_ARGS = {
'darwin': lambda: ['-Wl,-dylib'],
'linux': lambda: ['-shared'],
}
def _execute_link_request(self, link_request):
object_files = link_request.object_files
if len(object_files) == 0:
raise self.LinkSharedLibrariesError("No object files were provided in request {}!"
.format(link_request))
linker = link_request.linker
native_artifact = link_request.native_artifact
output_dir = link_request.output_dir
resulting_shared_lib_path = os.path.join(output_dir,
native_artifact.as_shared_lib(self.platform))
self.context.log.debug("resulting_shared_lib_path: {}".format(resulting_shared_lib_path))
# We are executing in the results_dir, so get absolute paths for everything.
cmd = ([linker.exe_filename] +
self.platform.resolve_platform_specific(self._SHARED_CMDLINE_ARGS) +
linker.extra_args +
['-o', os.path.abspath(resulting_shared_lib_path)] +
['-L{}'.format(lib_dir) for lib_dir in link_request.external_lib_dirs] +
['-l{}'.format(lib_name) for lib_name in link_request.external_lib_names] +
[os.path.abspath(obj) for obj in object_files])
self.context.log.info("selected linker exe name: '{}'".format(linker.exe_filename))
self.context.log.debug("linker argv: {}".format(cmd))
env = linker.as_invocation_environment_dict
self.context.log.debug("linker invocation environment: {}".format(env))
with self.context.new_workunit(name='link-shared-libraries',
labels=[WorkUnitLabel.LINKER]) as workunit:
try:
process = subprocess.Popen(
cmd,
cwd=output_dir,
stdout=workunit.output('stdout'),
stderr=workunit.output('stderr'),
env=env)
except OSError as e:
workunit.set_outcome(WorkUnit.FAILURE)
raise self.LinkSharedLibrariesError(
"Error invoking the native linker with command {cmd} and environment {env} "
"for request {req}: {err}."
.format(cmd=cmd, env=env, req=link_request, err=e),
e)
rc = process.wait()
if rc != 0:
workunit.set_outcome(WorkUnit.FAILURE)
raise self.LinkSharedLibrariesError(
"Error linking native objects with command {cmd} and environment {env} "
"for request {req}. Exit code was: {rc}."
.format(cmd=cmd, env=env, req=link_request, rc=rc))
return SharedLibrary(name=native_artifact.lib_name, path=resulting_shared_lib_path)
