def test_multiple(self): subclasses_of_b_or_c = SubclassesOf(self.B, self.C) self.assertEqual((self.B, self.C), subclasses_of_b_or_c.types) self.assertTrue(subclasses_of_b_or_c.satisfied_by(self.B())) self.assertTrue(subclasses_of_b_or_c.satisfied_by(self.C())) self.assertFalse(subclasses_of_b_or_c.satisfied_by(self.BPrime())) self.assertFalse(subclasses_of_b_or_c.satisfied_by(self.A()))
def test_single(self): subclasses_of_b = SubclassesOf(self.B) self.assertEqual((self.B,), subclasses_of_b.types) self.assertFalse(subclasses_of_b.satisfied_by(self.A())) self.assertTrue(subclasses_of_b.satisfied_by(self.B())) self.assertFalse(subclasses_of_b.satisfied_by(self.BPrime())) self.assertTrue(subclasses_of_b.satisfied_by(self.C()))
def test_validate(self): subclasses_of_a_or_b = SubclassesOf(self.A, self.B) self.assertEqual(self.A(), subclasses_of_a_or_b.validate_satisfied_by(self.A())) self.assertEqual(self.B(), subclasses_of_a_or_b.validate_satisfied_by(self.B())) self.assertEqual(self.C(), subclasses_of_a_or_b.validate_satisfied_by(self.C())) with self.assertRaisesWithMessage( TypeConstraintError, "value 1 (with type 'int') must satisfy this type constraint: SubclassesOf(A or B)."): subclasses_of_a_or_b.validate_satisfied_by(1)
def test_str_and_repr(self): collection_of_exactly_b = TypedCollection(Exactly(self.B)) self.assertEqual("TypedCollection(Exactly(B))", str(collection_of_exactly_b)) self.assertEqual("TypedCollection(Exactly(B))", repr(collection_of_exactly_b)) collection_of_multiple_subclasses = TypedCollection( SubclassesOf(self.A, self.B)) self.assertEqual("TypedCollection(SubclassesOf(A or B))", str(collection_of_multiple_subclasses)) self.assertEqual("TypedCollection(SubclassesOf(A, B))", repr(collection_of_multiple_subclasses))
class NativeCompileRequest(datatype([ ('compiler', SubclassesOf(Executable)), # TODO: add type checking for Collection.of(<type>)! 'include_dirs', 'sources', ('fatal_warnings', bool), 'output_dir', ])): pass # FIXME(#5950): perform all process execution in the v2 engine! class ObjectFiles(datatype(['root_dir', 'filenames'])):
class Target(Struct, HasProducts): def __init__(self, name=None, configurations=None, **kwargs): super(Target, self).__init__(name=name, **kwargs) self.configurations = configurations @property def products(self): return self.configurations @addressable_list(SubclassesOf(Struct)) def configurations(self): pass
def test_single(self): subclasses_of_b = SubclassesOf(self.B) self.assertEqual((self.B,), subclasses_of_b.types) self.assertFalse(subclasses_of_b.satisfied_by(self.A())) self.assertTrue(subclasses_of_b.satisfied_by(self.B())) self.assertFalse(subclasses_of_b.satisfied_by(self.BPrime())) self.assertTrue(subclasses_of_b.satisfied_by(self.C()))
class StructWithDeps(Struct): """A subclass of Struct with dependencies.""" def __init__(self, dependencies=None, **kwargs): """ :param list dependencies: The direct dependencies of this struct. """ # TODO: enforce the type of variants using the Addressable framework. super(StructWithDeps, self).__init__(**kwargs) self.dependencies = dependencies @addressable_list(SubclassesOf(Struct)) def dependencies(self): """The direct dependencies of this target.
class ExecuteProcessRequest( datatype([ ('argv', tuple), ('env', tuple), ('input_files', DirectoryDigest), ('output_files', tuple), ('output_directories', tuple), # NB: timeout_seconds covers the whole remote operation including queuing and setup. ('timeout_seconds', Exactly(float, int)), ('description', SubclassesOf(*six.string_types)), ])): """Request for execution with args and snapshots to extract.""" @classmethod def create_from_snapshot(cls, argv, env, snapshot, output_files=(), output_directories=(), timeout_seconds=_default_timeout_seconds, description='process'): cls._verify_env_is_dict(env) return ExecuteProcessRequest( argv=argv, env=tuple(env.items()), input_files=snapshot.directory_digest, output_files=output_files, output_directories=output_directories, timeout_seconds=timeout_seconds, description=description, ) @classmethod def create_with_empty_snapshot(cls, argv, env, output_files=(), output_directories=(), timeout_seconds=_default_timeout_seconds, description='process'): return cls.create_from_snapshot(argv, env, EMPTY_SNAPSHOT, output_files, output_directories, timeout_seconds, description) @classmethod def _verify_env_is_dict(cls, env): if not isinstance(env, dict): raise TypeCheckError( cls.__name__, "arg 'env' was invalid: value {} (with type {}) must be a dict" .format(env, type(env)))
def test_validate(self): subclasses_of_a_or_b = SubclassesOf(self.A, self.B) self.assertEqual(self.A(), subclasses_of_a_or_b.validate_satisfied_by(self.A())) self.assertEqual(self.B(), subclasses_of_a_or_b.validate_satisfied_by(self.B())) self.assertEqual(self.C(), subclasses_of_a_or_b.validate_satisfied_by(self.C())) with self.assertRaisesWithMessage( TypeConstraintError, "value 1 (with type 'int') must satisfy this type constraint: SubclassesOf(A or B)."): subclasses_of_a_or_b.validate_satisfied_by(1)
class ParseArgsRequest( datatype([ ('flag_value_map', SubclassesOf(dict)), 'namespace', 'get_all_scoped_flag_names', ('levenshtein_max_distance', int), ])): @staticmethod def _create_flag_value_map(flags): """Returns a map of flag -> list of values, based on the given flag strings. None signals no value given (e.g., -x, --foo). The value is a list because the user may specify the same flag multiple times, and that's sometimes OK (e.g., when appending to list-valued options). """ flag_value_map = defaultdict(list) for flag in flags: key, has_equals_sign, flag_val = flag.partition('=') if not has_equals_sign: if not flag.startswith('--'): # '-xfoo' style. key = flag[0:2] flag_val = flag[2:] if not flag_val: # Either a short option with no value or a long option with no equals sign. # Important so we can distinguish between no value ('--foo') and setting to an empty # string ('--foo='), for options with an implicit_value. flag_val = None flag_value_map[key].append(flag_val) return flag_value_map def __new__(cls, flags_in_scope, namespace, get_all_scoped_flag_names, levenshtein_max_distance): """ :param Iterable flags_in_scope: Iterable of arg strings to parse into flag values. :param namespace: The object to register the flag values on :param function get_all_scoped_flag_names: A 0-argument function which returns an iterable of all registered option names in all their scopes. This is used to create an error message with suggestions when raising a `ParseError`. :param int levenshtein_max_distance: The maximum Levenshtein edit distance between option names to determine similarly named options when an option name hasn't been registered. """ flag_value_map = cls._create_flag_value_map(flags_in_scope) return super(Parser.ParseArgsRequest, cls).__new__(cls, flag_value_map, namespace, get_all_scoped_flag_names, levenshtein_max_distance)
class UnpackJars(UnpackRemoteSourcesBase): """Unpack artifacts specified by unpacked_jars() targets. Adds an entry to SourceRoot for the contents. :API: public """ source_target_constraint = SubclassesOf(UnpackedJars) @classmethod def prepare(cls, options, round_manager): super().prepare(options, round_manager) round_manager.require_data(JarImportProducts) @classmethod def implementation_version(cls): return super().implementation_version() + [("UnpackJars", 0)] def get_fingerprint_strategy(self): return UnpackJarsFingerprintStrategy() def unpack_target(self, unpacked_jars, unpack_dir): deprecated_conditional( lambda: True, removal_version="1.31.0.dev0", entity_description="The `unpack-jars` goal", hint_message= "Contact the Pants team on Slack or [email protected] " "if you need this functionality.", ) direct_coords = { jar.coordinate for jar in unpacked_jars.all_imported_jar_deps } unpack_filter = self.get_unpack_filter(unpacked_jars) jar_import_products = self.context.products.get_data(JarImportProducts) for coordinate, jar_path in jar_import_products.imports(unpacked_jars): if not unpacked_jars.payload.intransitive or coordinate in direct_coords: self.context.log.info( "Unpacking jar {coordinate} from {jar_path} to {unpack_dir}." .format(coordinate=coordinate, jar_path=jar_path, unpack_dir=unpack_dir)) ZIP.extract(jar_path, unpack_dir, filter_func=unpack_filter)
class TaskRule(datatype([ ('output_type', _type_field), ('input_selectors', TypedCollection(SubclassesOf(type))), ('input_gets', tuple), 'func', ('dependency_rules', tuple), ('dependency_optionables', tuple), ('cacheable', bool), ]), Rule): """A Rule that runs a task function when all of its input selectors are satisfied. NB: This API is experimental, and not meant for direct consumption. To create a `TaskRule` you should always prefer the `@rule` constructor, and in cases where that is too constraining (likely due to #4535) please bump or open a ticket to explain the usecase. """ def __new__(cls, output_type, input_selectors, func, input_gets, dependency_optionables=None, dependency_rules=None, cacheable=True): # Create. return super().__new__( cls, output_type, input_selectors, input_gets, func, dependency_rules or tuple(), dependency_optionables or tuple(), cacheable, ) def __str__(self): return ('({}, {!r}, {}, gets={}, opts={})' .format(self.output_type.__name__, self.input_selectors, self.func.__name__, self.input_gets, self.dependency_optionables))
class Target(Struct, HasProducts): """A placeholder for the most-numerous Struct subclass. This particular implementation holds a collection of other Structs in a `configurations` field. """ def __init__(self, name=None, configurations=None, **kwargs): """ :param string name: The name of this target which forms its address in its namespace. :param list configurations: The configurations that apply to this target in various contexts. """ super(Target, self).__init__(name=name, **kwargs) self.configurations = configurations @property def products(self): return self.configurations @addressable_list(SubclassesOf(Struct)) def configurations(self): """The configurations that apply to this target in various contexts.
def __init__(self, native, build_root, work_dir, ignore_patterns, rule_index): self._native = native # TODO: The only (?) case where we use inheritance rather than exact type unions. has_products_constraint = SubclassesOf(HasProducts) self._root_subject_types = sorted(rule_index.roots) # Create the ExternContext, and the native Scheduler. self._tasks = native.new_tasks() self._register_rules(rule_index) self._scheduler = native.new_scheduler( self._tasks, self._root_subject_types, build_root, work_dir, ignore_patterns, Snapshot, FileContent, FilesContent, Path, Dir, File, Link, ExecuteProcessResult, has_products_constraint, constraint_for(Address), constraint_for(Variants), constraint_for(PathGlobs), constraint_for(Snapshot), constraint_for(FilesContent), constraint_for(Dir), constraint_for(File), constraint_for(Link), constraint_for(ExecuteProcessRequest), constraint_for(ExecuteProcessResult), constraint_for(GeneratorType), )
class UnpackJars(UnpackRemoteSourcesBase): """Unpack artifacts specified by unpacked_jars() targets. Adds an entry to SourceRoot for the contents. :API: public """ source_target_constraint = SubclassesOf(UnpackedJars) @classmethod def prepare(cls, options, round_manager): super().prepare(options, round_manager) round_manager.require_data(JarImportProducts) @classmethod def implementation_version(cls): return super().implementation_version() + [("UnpackJars", 0)] def get_fingerprint_strategy(self): return UnpackJarsFingerprintStrategy() def unpack_target(self, unpacked_jars, unpack_dir): direct_coords = { jar.coordinate for jar in unpacked_jars.all_imported_jar_deps } unpack_filter = self.get_unpack_filter(unpacked_jars) jar_import_products = self.context.products.get_data(JarImportProducts) for coordinate, jar_path in jar_import_products.imports(unpacked_jars): if not unpacked_jars.payload.intransitive or coordinate in direct_coords: self.context.log.info( "Unpacking jar {coordinate} from {jar_path} to {unpack_dir}." .format(coordinate=coordinate, jar_path=jar_path, unpack_dir=unpack_dir)) ZIP.extract(jar_path, unpack_dir, filter_func=unpack_filter)
class CCompile(NativeCompile): options_scope = "c-compile" # Compile only C library targets. source_target_constraint = SubclassesOf(CLibrary) workunit_label = "c-compile" @classmethod def implementation_version(cls): return super().implementation_version() + [("CCompile", 0)] @classmethod def subsystem_dependencies(cls): return super().subsystem_dependencies() + ( CCompileSettings.scoped(cls), ) def get_compile_settings(self): return CCompileSettings.scoped_instance(self) def get_compiler(self, native_library_target): return self.get_c_toolchain_variant(native_library_target).c_compiler
class CppCompile(NativeCompile): options_scope = 'cpp-compile' # Compile only C++ library targets. source_target_constraint = SubclassesOf(CppLibrary) workunit_label = 'cpp-compile' @classmethod def implementation_version(cls): return super(CppCompile, cls).implementation_version() + [('CppCompile', 0)] @classmethod def subsystem_dependencies(cls): return super(CppCompile, cls).subsystem_dependencies() + ( CppCompileSettings.scoped(cls), ) def get_compile_settings(self): return CppCompileSettings.scoped_instance(self) def get_compiler(self): return self.get_cpp_toolchain_variant().cpp_compiler
class NativeCompile(NativeTask, AbstractClass): # `NativeCompile` will use the `source_target_constraint` to determine what targets have "sources" # to compile, and the `dependent_target_constraint` to determine which dependent targets to # operate on for `strict_deps` calculation. # NB: `source_target_constraint` must be overridden. source_target_constraint = None dependent_target_constraint = SubclassesOf(NativeLibrary) # `NativeCompile` will use `workunit_label` as the name of the workunit when executing the # compiler process. `workunit_label` must be set to a string. workunit_label = None @classmethod def product_types(cls): return [ObjectFiles, NativeTargetDependencies] @property def cache_target_dirs(self): return True @abstractmethod def get_compile_settings(self): """An instance of `NativeCompileSettings` which is used in `NativeCompile`. :return: :class:`pants.backend.native.subsystems.native_compile_settings.NativeCompileSettings` """ @memoized_property def _compile_settings(self): return self.get_compile_settings() @classmethod def implementation_version(cls): return super(NativeCompile, cls).implementation_version() + [('NativeCompile', 0)] class NativeCompileError(TaskError): """Raised for errors in this class's logic. Subclasses are advised to create their own exception class. """ def native_deps(self, target): return self.strict_deps_for_target( target, predicate=self.dependent_target_constraint.satisfied_by) def strict_deps_for_target(self, target, predicate=None): """Get the dependencies of `target` filtered by `predicate`, accounting for 'strict_deps'. If 'strict_deps' is on, instead of using the transitive closure of dependencies, targets will only be able to see their immediate dependencies declared in the BUILD file. The 'strict_deps' setting is obtained from the result of `get_compile_settings()`. NB: This includes the current target in the result. """ if self._compile_settings.get_subsystem_target_mirrored_field_value( 'strict_deps', target): strict_deps = target.strict_dependencies(DependencyContext()) if predicate: filtered_deps = filter(predicate, strict_deps) else: filtered_deps = strict_deps deps = [target] + filtered_deps else: deps = self.context.build_graph.transitive_subgraph_of_addresses( [target.address], predicate=predicate) return deps @staticmethod def _add_product_at_target_base(product_mapping, target, value): product_mapping.add(target, target.target_base).append(value) def execute(self): object_files_product = self.context.products.get(ObjectFiles) native_deps_product = self.context.products.get( NativeTargetDependencies) source_targets = self.context.targets( self.source_target_constraint.satisfied_by) with self.invalidated( source_targets, invalidate_dependents=True) as invalidation_check: for vt in invalidation_check.invalid_vts: deps = self.native_deps(vt.target) self._add_product_at_target_base(native_deps_product, vt.target, deps) compile_request = self._make_compile_request(vt, deps) self.context.log.debug( "compile_request: {}".format(compile_request)) self._compile(compile_request) for vt in invalidation_check.all_vts: object_files = self.collect_cached_objects(vt) self._add_product_at_target_base(object_files_product, vt.target, object_files) # This may be calculated many times for a target, so we memoize it. @memoized_method def _include_dirs_for_target(self, target): return target.sources_relative_to_target_base().rel_root class NativeSourcesByType(datatype(['rel_root', 'headers', 'sources'])): pass def get_sources_headers_for_target(self, target): """Return a list of file arguments to provide to the compiler. NB: result list will contain both header and source files! :raises: :class:`NativeCompile.NativeCompileError` if there is an error processing the sources. """ # Get source paths relative to the target base so the exception message with the target and # paths makes sense. target_relative_sources = target.sources_relative_to_target_base() rel_root = target_relative_sources.rel_root # Unique file names are required because we just dump object files into a single directory, and # the compiler will silently just produce a single object file if provided non-unique filenames. # FIXME: add some shading to file names so we can remove this check. # NB: It shouldn't matter if header files have the same name, but this will raise an error in # that case as well. We won't need to do any shading of header file names. seen_filenames = defaultdict(list) for src in target_relative_sources: seen_filenames[os.path.basename(src)].append(src) duplicate_filename_err_msgs = [] for fname, source_paths in seen_filenames.items(): if len(source_paths) > 1: duplicate_filename_err_msgs.append( "filename: {}, paths: {}".format(fname, source_paths)) if duplicate_filename_err_msgs: raise self.NativeCompileError( "Error in target '{}': source files must have a unique filename within a '{}' target. " "Conflicting filenames:\n{}".format( target.address.spec, target.alias(), '\n'.join(duplicate_filename_err_msgs))) return [os.path.join(rel_root, src) for src in target_relative_sources] # FIXME(#5951): expand `Executable` to cover argv generation (where an `Executable` is subclassed # to modify or extend the argument list, as declaratively as possible) to remove # `extra_compile_args(self)`! @abstractmethod def get_compiler(self): """An instance of `Executable` which can be invoked to compile files. :return: :class:`pants.backend.native.config.environment.Executable` """ @memoized_property def _compiler(self): return self.get_compiler() def _make_compile_request(self, versioned_target, dependencies): target = versioned_target.target include_dirs = [ self._include_dirs_for_target(dep_tgt) for dep_tgt in dependencies ] sources_and_headers = self.get_sources_headers_for_target(target) return NativeCompileRequest(compiler=self._compiler, include_dirs=include_dirs, sources=sources_and_headers, fatal_warnings=self._compile_settings. get_subsystem_target_mirrored_field_value( 'fatal_warnings', target), output_dir=versioned_target.results_dir) @abstractmethod def extra_compile_args(self): """Return a list of task-specific arguments to use to compile sources.""" def _make_compile_argv(self, compile_request): """Return a list of arguments to use to compile sources. Subclasses can override and append.""" compiler = compile_request.compiler err_flags = ['-Werror'] if compile_request.fatal_warnings else [] platform_specific_flags = compiler.platform.resolve_platform_specific({ 'linux': lambda: [], 'darwin': lambda: ['-mmacosx-version-min=10.11'], }) # We are going to execute in the target output, so get absolute paths for everything. # TODO: If we need to produce static libs, don't add -fPIC! (could use Variants -- see #5788). argv = ([compiler.exe_filename] + platform_specific_flags + self.extra_compile_args() + err_flags + ['-c', '-fPIC'] + [ '-I{}'.format(os.path.abspath(inc_dir)) for inc_dir in compile_request.include_dirs ] + [os.path.abspath(src) for src in compile_request.sources]) return argv def _compile(self, compile_request): """Perform the process of compilation, writing object files to the request's 'output_dir'. NB: This method must arrange the output files so that `collect_cached_objects()` can collect all of the results (or vice versa)! """ sources = compile_request.sources if len(sources) == 0: # TODO: do we need this log message? Should we still have it for intentionally header-only # libraries (that might be a confusing message to see)? self.context.log.debug( "no sources in request {}, skipping".format(compile_request)) return compiler = compile_request.compiler output_dir = compile_request.output_dir argv = self._make_compile_argv(compile_request) with self.context.new_workunit(name=self.workunit_label, labels=[WorkUnitLabel.COMPILER ]) as workunit: try: process = subprocess.Popen( argv, cwd=output_dir, stdout=workunit.output('stdout'), stderr=workunit.output('stderr'), env={'PATH': get_joined_path(compiler.path_entries)}) except OSError as e: workunit.set_outcome(WorkUnit.FAILURE) raise self.NativeCompileError( "Error invoking '{exe}' with command {cmd} for request {req}: {err}" .format(exe=compiler.exe_filename, cmd=argv, req=compile_request, err=e)) rc = process.wait() if rc != 0: workunit.set_outcome(WorkUnit.FAILURE) raise self.NativeCompileError( "Error in '{section_name}' with command {cmd} for request {req}. Exit code was: {rc}." .format(section_name=self.workunit_name, cmd=argv, req=compile_request, rc=rc)) def collect_cached_objects(self, versioned_target): """Scan `versioned_target`'s results directory and return the output files from that directory. :return: :class:`ObjectFiles` """ return ObjectFiles(versioned_target.results_dir, os.listdir(versioned_target.results_dir))
def test_none(self): with self.assertRaises(ValueError): SubclassesOf()
class WithSubclassTypeConstraint(datatype([('some_value', SubclassesOf(SomeBaseClass))])): pass class NonNegativeInt(datatype([('an_int', int)])):
class UnpackWheels(UnpackRemoteSourcesBase): """Extract native code from `NativePythonWheel` targets for use by downstream C/C++ sources.""" source_target_constraint = SubclassesOf(UnpackedWheels) def get_fingerprint_strategy(self): return UnpackWheelsFingerprintStrategy() @classmethod def subsystem_dependencies(cls): return super().subsystem_dependencies() + ( PexBuilderWrapper.Factory, PythonInterpreterCache, PythonSetup, ) def _get_matching_wheel(self, pex_path, interpreter, requirements, module_name): """Use PexBuilderWrapper to resolve a single wheel from the requirement specs using pex. N.B.: The resolved wheel is already "unpacked" by PEX. More accurately, it's installed in a chroot. """ with self.context.new_workunit("extract-native-wheels"): with safe_concurrent_creation(pex_path) as chroot: pex_builder = PexBuilderWrapper.Factory.create( builder=PEXBuilder(path=chroot, interpreter=interpreter), log=self.context.log) return pex_builder.extract_single_dist_for_current_platform( requirements, dist_key=module_name) @memoized_method def _compatible_interpreter(self, unpacked_whls): constraints = PythonSetup.global_instance( ).compatibility_or_constraints(unpacked_whls.compatibility) allowable_interpreters = PythonInterpreterCache.global_instance( ).setup(filters=constraints) return min(allowable_interpreters) class WheelUnpackingError(TaskError): pass def unpack_target(self, unpacked_whls, unpack_dir): interpreter = self._compatible_interpreter(unpacked_whls) with temporary_dir() as resolve_dir: try: matched_dist = self._get_matching_wheel( resolve_dir, interpreter, unpacked_whls.all_imported_requirements, unpacked_whls.module_name, ) wheel_chroot = matched_dist.location if unpacked_whls.within_data_subdir: # N.B.: Wheels with data dirs have the data installed under the top module. dist_data_dir = os.path.join(wheel_chroot, unpacked_whls.module_name) else: dist_data_dir = wheel_chroot unpack_filter = self.get_unpack_filter(unpacked_whls) # Copy over the module's data files into `unpack_dir`. mergetree(dist_data_dir, unpack_dir, file_filter=unpack_filter) except Exception as e: raise self.WheelUnpackingError( "Error extracting wheel for target {}: {}".format( unpacked_whls, str(e)), e)
class NativeCompile(NativeTask, AbstractClass): # `NativeCompile` will use the `source_target_constraint` to determine what targets have "sources" # to compile, and the `dependent_target_constraint` to determine which dependent targets to # operate on for `strict_deps` calculation. # NB: `source_target_constraint` must be overridden. source_target_constraint = None dependent_target_constraint = SubclassesOf(ExternalNativeLibrary, NativeLibrary) # `NativeCompile` will use `workunit_label` as the name of the workunit when executing the # compiler process. `workunit_label` must be set to a string. @classproperty def workunit_label(cls): raise NotImplementedError('subclasses of NativeCompile must override workunit_label!') @classmethod def product_types(cls): return [ObjectFiles] @classmethod def prepare(cls, options, round_manager): super(NativeCompile, cls).prepare(options, round_manager) round_manager.optional_data(NativeExternalLibraryFiles) @property def cache_target_dirs(self): return True @classmethod def implementation_version(cls): return super(NativeCompile, cls).implementation_version() + [('NativeCompile', 1)] class NativeCompileError(TaskError): """Raised for errors in this class's logic. Subclasses are advised to create their own exception class. """ def execute(self): object_files_product = self.context.products.get(ObjectFiles) external_libs_product = self.context.products.get_data(NativeExternalLibraryFiles) source_targets = self.context.targets(self.source_target_constraint.satisfied_by) with self.invalidated(source_targets, invalidate_dependents=True) as invalidation_check: for vt in invalidation_check.all_vts: deps = self.native_deps(vt.target) if not vt.valid: compile_request = self._make_compile_request(vt, deps, external_libs_product) self.context.log.debug("compile_request: {}".format(compile_request)) self._compile(compile_request) object_files = self.collect_cached_objects(vt) self._add_product_at_target_base(object_files_product, vt.target, object_files) # This may be calculated many times for a target, so we memoize it. @memoized_method def _include_dirs_for_target(self, target): return os.path.join(get_buildroot(), target.address.spec_path) class NativeSourcesByType(datatype(['rel_root', 'headers', 'sources'])): pass def get_sources_headers_for_target(self, target): """Return a list of file arguments to provide to the compiler. NB: result list will contain both header and source files! :raises: :class:`NativeCompile.NativeCompileError` if there is an error processing the sources. """ # Get source paths relative to the target base so the exception message with the target and # paths makes sense. target_relative_sources = target.sources_relative_to_target_base() rel_root = target_relative_sources.rel_root # Unique file names are required because we just dump object files into a single directory, and # the compiler will silently just produce a single object file if provided non-unique filenames. # TODO: add some shading to file names so we can remove this check. # NB: It shouldn't matter if header files have the same name, but this will raise an error in # that case as well. We won't need to do any shading of header file names. seen_filenames = defaultdict(list) for src in target_relative_sources: seen_filenames[os.path.basename(src)].append(src) duplicate_filename_err_msgs = [] for fname, source_paths in seen_filenames.items(): if len(source_paths) > 1: duplicate_filename_err_msgs.append("filename: {}, paths: {}".format(fname, source_paths)) if duplicate_filename_err_msgs: raise self.NativeCompileError( "Error in target '{}': source files must have a unique filename within a '{}' target. " "Conflicting filenames:\n{}" .format(target.address.spec, target.alias(), '\n'.join(duplicate_filename_err_msgs))) return [os.path.join(get_buildroot(), rel_root, src) for src in target_relative_sources] @abstractmethod def get_compile_settings(self): """Return a subclass of NativeBuildStepSettingsBase. NB: Subclasses will be queried for the compile settings once and the result cached. """ @memoized_property def _compile_settings(self): return self.get_compile_settings() @abstractmethod def get_compiler(self): """An instance of `Executable` which can be invoked to compile files. NB: Subclasses will be queried for the compiler instance once and the result cached. :return: :class:`pants.backend.native.config.environment.Executable` """ @memoized_property def _compiler(self): return self.get_compiler() def _get_third_party_include_dirs(self, external_libs_product, dependencies): if not external_libs_product: return [] return [nelf.include_dir for nelf in external_libs_product.get_for_targets(dependencies) if nelf.include_dir] def _make_compile_request(self, versioned_target, dependencies, external_libs_product): target = versioned_target.target include_dirs = [self._include_dirs_for_target(dep_tgt) for dep_tgt in dependencies] include_dirs.extend(self._get_third_party_include_dirs(external_libs_product, dependencies)) sources_and_headers = self.get_sources_headers_for_target(target) return NativeCompileRequest( compiler=self._compiler, include_dirs=include_dirs, sources=sources_and_headers, fatal_warnings=self._compile_settings.get_fatal_warnings_value_for_target(target), output_dir=versioned_target.results_dir) def _make_compile_argv(self, compile_request): """Return a list of arguments to use to compile sources. Subclasses can override and append.""" compiler = compile_request.compiler err_flags = ['-Werror'] if compile_request.fatal_warnings else [] # We are going to execute in the target output, so get absolute paths for everything. buildroot = get_buildroot() argv = ( [compiler.exe_filename] + compiler.extra_args + err_flags + # TODO: If we need to produce static libs, don't add -fPIC! (could use Variants -- see #5788). ['-c', '-fPIC'] + [ '-I{}'.format(os.path.join(buildroot, inc_dir)) for inc_dir in compile_request.include_dirs ] + [os.path.join(buildroot, src) for src in compile_request.sources]) self.context.log.debug("compile argv: {}".format(argv)) return argv def _compile(self, compile_request): """Perform the process of compilation, writing object files to the request's 'output_dir'. NB: This method must arrange the output files so that `collect_cached_objects()` can collect all of the results (or vice versa)! """ sources = compile_request.sources if len(sources) == 0: # TODO: do we need this log message? Should we still have it for intentionally header-only # libraries (that might be a confusing message to see)? self.context.log.debug("no sources in request {}, skipping".format(compile_request)) return compiler = compile_request.compiler output_dir = compile_request.output_dir argv = self._make_compile_argv(compile_request) env = compiler.as_invocation_environment_dict with self.context.new_workunit( name=self.workunit_label, labels=[WorkUnitLabel.COMPILER]) as workunit: try: process = subprocess.Popen( argv, cwd=output_dir, stdout=workunit.output('stdout'), stderr=workunit.output('stderr'), env=env) except OSError as e: workunit.set_outcome(WorkUnit.FAILURE) raise self.NativeCompileError( "Error invoking '{exe}' with command {cmd} and environment {env} for request {req}: {err}" .format(exe=compiler.exe_filename, cmd=argv, env=env, req=compile_request, err=e)) rc = process.wait() if rc != 0: workunit.set_outcome(WorkUnit.FAILURE) raise self.NativeCompileError( "Error in '{section_name}' with command {cmd} and environment {env} for request {req}. " "Exit code was: {rc}." .format(section_name=self.workunit_label, cmd=argv, env=env, req=compile_request, rc=rc)) def collect_cached_objects(self, versioned_target): """Scan `versioned_target`'s results directory and return the output files from that directory. :return: :class:`ObjectFiles` """ return ObjectFiles(versioned_target.results_dir, os.listdir(versioned_target.results_dir))
class PythonNativeCode(Subsystem): """A subsystem which exposes components of the native backend to the python backend.""" options_scope = 'python-native-code' default_native_source_extensions = ['.c', '.cpp', '.cc'] class PythonNativeCodeError(Exception): pass @classmethod def register_options(cls, register): super(PythonNativeCode, cls).register_options(register) register( '--native-source-extensions', type=list, default=cls.default_native_source_extensions, fingerprint=True, advanced=True, help= 'The extensions recognized for native source files in `python_dist()` sources.' ) @classmethod def subsystem_dependencies(cls): return super(PythonNativeCode, cls).subsystem_dependencies() + ( NativeToolchain.scoped(cls), PythonSetup.scoped(cls), ) @memoized_property def _native_source_extensions(self): return self.get_options().native_source_extensions @memoized_property def native_toolchain(self): return NativeToolchain.scoped_instance(self) @memoized_property def _python_setup(self): return PythonSetup.scoped_instance(self) def pydist_has_native_sources(self, target): return target.has_sources( extension=tuple(self._native_source_extensions)) def native_target_has_native_sources(self, target): return target.has_sources() @memoized_property def _native_target_matchers(self): return { SubclassesOf(PythonDistribution): self.pydist_has_native_sources, SubclassesOf(NativeLibrary): self.native_target_has_native_sources, } def _any_targets_have_native_sources(self, targets): # TODO(#5949): convert this to checking if the closure of python requirements has any # platform-specific packages (maybe find the platforms there too?). for tgt in targets: for type_constraint, target_predicate in self._native_target_matchers.items( ): if type_constraint.satisfied_by(tgt) and target_predicate(tgt): return True return False def get_targets_by_declared_platform(self, targets): """ Aggregates a dict that maps a platform string to a list of targets that specify the platform. If no targets have platforms arguments, return a dict containing platforms inherited from the PythonSetup object. :param tgts: a list of :class:`Target` objects. :returns: a dict mapping a platform string to a list of targets that specify the platform. """ targets_by_platforms = defaultdict(list) for tgt in targets: for platform in tgt.platforms: targets_by_platforms[platform].append(tgt) if not targets_by_platforms: for platform in self._python_setup.platforms: targets_by_platforms[platform] = [ '(No target) Platform inherited from either the ' '--platforms option or a pants.ini file.' ] return targets_by_platforms _PYTHON_PLATFORM_TARGETS_CONSTRAINT = SubclassesOf(PythonBinary, PythonDistribution) def check_build_for_current_platform_only(self, targets): """ Performs a check of whether the current target closure has native sources and if so, ensures that Pants is only targeting the current platform. :param tgts: a list of :class:`Target` objects. :return: a boolean value indicating whether the current target closure has native sources. :raises: :class:`pants.base.exceptions.IncompatiblePlatformsError` """ if not self._any_targets_have_native_sources(targets): return False targets_with_platforms = [ target for target in targets if self._PYTHON_PLATFORM_TARGETS_CONSTRAINT.satisfied_by(target) ] platforms_with_sources = self.get_targets_by_declared_platform( targets_with_platforms) platform_names = list(platforms_with_sources.keys()) if len(platform_names) < 1: raise self.PythonNativeCodeError( "Error: there should be at least one platform in the target closure, because " "we checked that there are native sources.") if platform_names == ['current']: return True raise IncompatiblePlatformsError( 'The target set contains one or more targets that depend on ' 'native code. Please ensure that the platform arguments in all relevant targets and build ' 'options are compatible with the current platform. Found targets for platforms: {}' .format(str(platforms_with_sources)))
class CFFIExternMethodRuntimeErrorInfo(datatype([ ('exc_type', type), ('exc_value', SubclassesOf(Exception)), 'traceback', ])): """Encapsulates an exception raised when a CFFI extern is called so that it can be displayed.
def _native_target_matchers(self): return { SubclassesOf(PythonDistribution): self.pydist_has_native_sources, SubclassesOf(NativeLibrary): NativeLibrary.produces_ctypes_native_library, }
def __init__( self, native, project_tree, work_dir, rules, execution_options, include_trace_on_error=True, validate=True, ): """ :param native: An instance of engine.native.Native. :param project_tree: An instance of ProjectTree for the current build root. :param work_dir: The pants work dir. :param rules: A set of Rules which is used to compute values in the graph. :param execution_options: Execution options for (remote) processes. :param include_trace_on_error: Include the trace through the graph upon encountering errors. :type include_trace_on_error: bool :param validate: True to assert that the ruleset is valid. """ if execution_options.remote_execution_server and not execution_options.remote_store_server: raise ValueError( "Cannot set remote execution server without setting remote store server" ) self._native = native self.include_trace_on_error = include_trace_on_error # TODO: The only (?) case where we use inheritance rather than exact type unions. has_products_constraint = SubclassesOf(HasProducts) # Validate and register all provided and intrinsic tasks. rule_index = RuleIndex.create(list(rules)) self._root_subject_types = sorted(rule_index.roots) # Create the native Scheduler and Session. # TODO: This `_tasks` reference could be a local variable, since it is not used # after construction. self._tasks = native.new_tasks() self._register_rules(rule_index) self._scheduler = native.new_scheduler( self._tasks, self._root_subject_types, project_tree.build_root, work_dir, project_tree.ignore_patterns, execution_options, DirectoryDigest, Snapshot, FileContent, FilesContent, Path, Dir, File, Link, FallibleExecuteProcessResult, has_products_constraint, constraint_for(Address), constraint_for(Variants), constraint_for(PathGlobs), constraint_for(DirectoryDigest), constraint_for(Snapshot), constraint_for(FilesContent), constraint_for(Dir), constraint_for(File), constraint_for(Link), constraint_for(ExecuteProcessRequest), constraint_for(FallibleExecuteProcessResult), constraint_for(GeneratorType), ) # If configured, visualize the rule graph before asserting that it is valid. if self.visualize_to_dir() is not None: rule_graph_name = 'rule_graph.dot' self.visualize_rule_graph_to_file( os.path.join(self.visualize_to_dir(), rule_graph_name)) if validate: self._assert_ruleset_valid()
# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). import ast from pants.util.objects import SubclassesOf, TypeConstraint, datatype _type_field = SubclassesOf(type) class Get(datatype([ ('product', _type_field), ('subject_declared_type', _type_field), 'subject', ])): """Experimental synchronous generator API. May be called equivalently as either: # verbose form: Get(product_type, subject_declared_type, subject) # shorthand form: Get(product_type, subject_type(subject)) """ @staticmethod def extract_constraints(call_node): """Parses a `Get(..)` call in one of its two legal forms to return its type constraints. :param call_node: An `ast.Call` node representing a call to `Get(..)`. :return: A tuple of product type id and subject type id. """ def render_args():
class WithSubclassTypeConstraint( datatype([('some_value', SubclassesOf(SomeBaseClass))])): pass
def test_multiple(self): subclasses_of_b_or_c = SubclassesOf(self.B, self.C) self.assertTrue(subclasses_of_b_or_c.satisfied_by(self.B())) self.assertTrue(subclasses_of_b_or_c.satisfied_by(self.C())) self.assertFalse(subclasses_of_b_or_c.satisfied_by(self.BPrime())) self.assertFalse(subclasses_of_b_or_c.satisfied_by(self.A()))
def test_collection_multiple(self): collection_constraint = TypedCollection(SubclassesOf(self.B, self.BPrime)) self.assertTrue(collection_constraint.satisfied_by([self.B(), self.C(), self.BPrime()])) self.assertFalse(collection_constraint.satisfied_by([self.B(), self.A()]))
def test_none(self): with self.assertRaisesWithMessage(ValueError, 'Must supply at least one type'): SubclassesOf()
class LinkSharedLibraries(NativeTask): options_scope = 'link-shared-libraries' # TODO(#6486): change this to include ExternalNativeLibrary, then add a test that strict-deps # works on external libs. source_target_constraint = SubclassesOf(NativeLibrary) @classmethod def product_types(cls): return [SharedLibrary] @classmethod def prepare(cls, options, round_manager): super(LinkSharedLibraries, cls).prepare(options, round_manager) round_manager.require(ObjectFiles) round_manager.optional_product(NativeExternalLibraryFiles) @property def cache_target_dirs(self): return True @classmethod def implementation_version(cls): return super(LinkSharedLibraries, cls).implementation_version() + [('LinkSharedLibraries', 1)] class LinkSharedLibrariesError(TaskError): pass @memoized_property def linker(self): # NB: we are using the C++ toolchain here for linking every type of input, including compiled C # source files. return self.get_cpp_toolchain_variant().cpp_linker @memoized_property def platform(self): # TODO: convert this to a v2 engine dependency injection. return Platform.create() def execute(self): targets_providing_artifacts = self.context.targets(NativeLibrary.produces_ctypes_native_library) compiled_objects_product = self.context.products.get(ObjectFiles) shared_libs_product = self.context.products.get(SharedLibrary) external_libs_product = self.context.products.get_data(NativeExternalLibraryFiles) all_shared_libs_by_name = {} with self.invalidated(targets_providing_artifacts, invalidate_dependents=True) as invalidation_check: for vt in invalidation_check.all_vts: if vt.valid: shared_library = self._retrieve_shared_lib_from_cache(vt) else: # TODO: We need to partition links based on proper dependency edges and not # perform a link to every native_external_library for all targets in the closure. # https://github.com/pantsbuild/pants/issues/6178 link_request = self._make_link_request( vt, compiled_objects_product, external_libs_product) self.context.log.debug("link_request: {}".format(link_request)) shared_library = self._execute_link_request(link_request) same_name_shared_lib = all_shared_libs_by_name.get(shared_library.name, None) if same_name_shared_lib: # TODO: test this branch! raise self.LinkSharedLibrariesError( "The name '{name}' was used for two shared libraries: {prev} and {cur}." .format(name=shared_library.name, prev=same_name_shared_lib, cur=shared_library)) else: all_shared_libs_by_name[shared_library.name] = shared_library self._add_product_at_target_base(shared_libs_product, vt.target, shared_library) def _retrieve_shared_lib_from_cache(self, vt): native_artifact = vt.target.ctypes_native_library path_to_cached_lib = os.path.join( vt.results_dir, native_artifact.as_shared_lib(self.platform)) if not os.path.isfile(path_to_cached_lib): raise self.LinkSharedLibrariesError("The shared library at {} does not exist!" .format(path_to_cached_lib)) return SharedLibrary(name=native_artifact.lib_name, path=path_to_cached_lib) def _make_link_request(self, vt, compiled_objects_product, external_libs_product): self.context.log.debug("link target: {}".format(vt.target)) deps = self.native_deps(vt.target) all_compiled_object_files = [] for dep_tgt in deps: if compiled_objects_product.get(dep_tgt): self.context.log.debug("dep_tgt: {}".format(dep_tgt)) object_files = self._retrieve_single_product_at_target_base(compiled_objects_product, dep_tgt) self.context.log.debug("object_files: {}".format(object_files)) object_file_paths = object_files.file_paths() self.context.log.debug("object_file_paths: {}".format(object_file_paths)) all_compiled_object_files.extend(object_file_paths) external_lib_dirs = [] external_lib_names = [] if external_libs_product is not None: for nelf in external_libs_product.get_for_targets(deps): if nelf.lib_dir: external_lib_dirs.append(nelf.lib_dir) external_lib_names.extend(nelf.lib_names) link_request = LinkSharedLibraryRequest( linker=self.linker, object_files=tuple(all_compiled_object_files), native_artifact=vt.target.ctypes_native_library, output_dir=vt.results_dir, external_lib_dirs=tuple(external_lib_dirs), external_lib_names=tuple(external_lib_names)) self.context.log.debug(repr(link_request)) return link_request _SHARED_CMDLINE_ARGS = { 'darwin': lambda: ['-Wl,-dylib'], 'linux': lambda: ['-shared'], } def _execute_link_request(self, link_request): object_files = link_request.object_files if len(object_files) == 0: raise self.LinkSharedLibrariesError("No object files were provided in request {}!" .format(link_request)) linker = link_request.linker native_artifact = link_request.native_artifact output_dir = link_request.output_dir resulting_shared_lib_path = os.path.join(output_dir, native_artifact.as_shared_lib(self.platform)) self.context.log.debug("resulting_shared_lib_path: {}".format(resulting_shared_lib_path)) # We are executing in the results_dir, so get absolute paths for everything. cmd = ([linker.exe_filename] + self.platform.resolve_platform_specific(self._SHARED_CMDLINE_ARGS) + linker.extra_args + ['-o', os.path.abspath(resulting_shared_lib_path)] + ['-L{}'.format(lib_dir) for lib_dir in link_request.external_lib_dirs] + ['-l{}'.format(lib_name) for lib_name in link_request.external_lib_names] + [os.path.abspath(obj) for obj in object_files]) self.context.log.info("selected linker exe name: '{}'".format(linker.exe_filename)) self.context.log.debug("linker argv: {}".format(cmd)) env = linker.as_invocation_environment_dict self.context.log.debug("linker invocation environment: {}".format(env)) with self.context.new_workunit(name='link-shared-libraries', labels=[WorkUnitLabel.LINKER]) as workunit: try: process = subprocess.Popen( cmd, cwd=output_dir, stdout=workunit.output('stdout'), stderr=workunit.output('stderr'), env=env) except OSError as e: workunit.set_outcome(WorkUnit.FAILURE) raise self.LinkSharedLibrariesError( "Error invoking the native linker with command {cmd} and environment {env} " "for request {req}: {err}." .format(cmd=cmd, env=env, req=link_request, err=e), e) rc = process.wait() if rc != 0: workunit.set_outcome(WorkUnit.FAILURE) raise self.LinkSharedLibrariesError( "Error linking native objects with command {cmd} and environment {env} " "for request {req}. Exit code was: {rc}." .format(cmd=cmd, env=env, req=link_request, rc=rc)) return SharedLibrary(name=native_artifact.lib_name, path=resulting_shared_lib_path)