def create_legacy_graph_tasks(symbol_table_cls): """Create tasks to recursively parse the legacy graph.""" symbol_table_constraint = symbol_table_cls.constraint() return [ # Recursively requests HydratedTargets, which will result in an eager, transitive graph walk. (HydratedTargets, [SelectDependencies(HydratedTarget, Addresses, field_types=(Address,), transitive=True)], HydratedTargets), (HydratedTarget, [Select(symbol_table_constraint), SelectDependencies(HydratedField, symbol_table_constraint, 'field_adaptors', field_types=(SourcesField, BundlesField,))], hydrate_target), (HydratedField, [Select(SourcesField), SelectProjection(FilesDigest, PathGlobs, ('path_globs',), SourcesField), SelectProjection(Files, PathGlobs, ('excluded_path_globs',), SourcesField)], hydrate_sources), (HydratedField, [Select(BundlesField), SelectDependencies(FilesDigest, BundlesField, 'path_globs_list', field_types=(PathGlobs,)), SelectDependencies(Files, BundlesField, 'excluded_path_globs_list', field_types=(PathGlobs,))], hydrate_bundles), ]
def create_graph_tasks(address_mapper, symbol_table_cls): """Creates tasks used to parse Structs from BUILD files. :param address_mapper_key: The subject key for an AddressMapper instance. :param symbol_table_cls: A SymbolTable class to provide symbols for Address lookups. """ symbol_table_constraint = symbol_table_cls.constraint() return [ # Support for resolving Structs from Addresses (symbol_table_constraint, [ Select(UnhydratedStruct), SelectDependencies(symbol_table_constraint, UnhydratedStruct, field_types=(Address, )) ], hydrate_struct), (UnhydratedStruct, [ SelectProjection(AddressFamily, Dir, ('spec_path', ), Address), Select(Address) ], resolve_unhydrated_struct), ] + [ # BUILD file parsing. (AddressFamily, [ SelectLiteral(address_mapper, AddressMapper), Select(Dir), SelectProjection(FilesContent, Files, ('files', ), BuildFiles) ], parse_address_family), (BuildFiles, [ SelectLiteral(address_mapper, AddressMapper), Select(DirectoryListing) ], filter_buildfile_paths), ] + [ # Simple spec handling. (Addresses, [ SelectProjection(AddressFamily, Dir, ('directory', ), SingleAddress), Select(SingleAddress) ], address_from_address_family), (Addresses, [ SelectProjection(AddressFamily, Dir, ('directory', ), SiblingAddresses) ], addresses_from_address_family), ] + [ # Recursive spec handling: locate directories that contain build files, and request # AddressFamilies for each of them. (Addresses, [ SelectDependencies(AddressFamily, BuildDirs, field_types=(Dir, )) ], addresses_from_address_families), (BuildDirs, [SelectLiteral(address_mapper, AddressMapper), Select(Files)], filter_build_dirs), (PathGlobs, [ SelectLiteral(address_mapper, AddressMapper), Select(DescendantAddresses) ], descendant_addresses_to_globs), (PathGlobs, [ SelectLiteral(address_mapper, AddressMapper), Select(AscendantAddresses) ], ascendant_addresses_to_globs), ]
def create_fs_tasks(): """Creates tasks that consume the intrinsic filesystem types.""" return [ # Glob execution: to avoid memoizing lots of incremental results, we recursively expand PathGlobs, and then # convert them to Paths independently. (Paths, [ SelectDependencies( PathsExpansion, PathGlobs, field_types=(PathWildcard, PathDirWildcard, PathRoot), transitive=True) ], finalize_path_expansion), (PathsExpansion, [Select(PathRoot)], apply_path_root), (PathsExpansion, [ SelectProjection(DirectoryListing, Dir, ('canonical_stat', ), PathWildcard), Select(PathWildcard) ], apply_path_wildcard), (PathsExpansion, [ SelectProjection(Dirs, Paths, ('paths', ), FilteredPaths), Select(PathDirWildcard) ], apply_path_dir_wildcard), (FilteredPaths, [ SelectProjection(DirectoryListing, Dir, ('canonical_stat', ), PathDirWildcard), Select(PathDirWildcard) ], filter_paths), ] + [ # Link resolution. (Dirs, [ Select(Paths), SelectDependencies( Dirs, Paths, field='link_stats', field_types=(Link, )) ], resolve_dir_links), (Files, [ Select(Paths), SelectDependencies( Files, Paths, field='link_stats', field_types=(Link, )) ], resolve_file_links), (Dirs, [SelectProjection(Dirs, PathGlobs, ('path_globs', ), ReadLink)], resolve_link), (Files, [SelectProjection(Files, PathGlobs, ('path_globs', ), ReadLink)], resolve_link), ] + [ # File content. (FilesContent, [ Select(Files), SelectDependencies( FileContent, Files, field='stats', field_types=(File, )) ], files_content), (FilesDigest, [ Select(Files), SelectDependencies( FileDigest, Files, field='stats', field_types=(File, )) ], files_digest), ]
def create_fs_tasks(): """Creates tasks that consume the native filesystem Node type.""" return [ # Glob execution. (Paths, [ SelectDependencies( Paths, PathGlobs, field_types=(PathWildcard, PathDirWildcard, PathRoot)) ], merge_paths), (Paths, [Select(PathRoot)], apply_path_root), (Paths, [ SelectProjection(DirectoryListing, Dir, ('canonical_stat', ), PathWildcard), Select(PathWildcard) ], apply_path_wildcard), (PathGlobs, [ SelectProjection(Dirs, Paths, ('paths', ), FilteredPaths), Select(PathDirWildcard) ], apply_path_dir_wildcard), (FilteredPaths, [ SelectProjection(DirectoryListing, Dir, ('canonical_stat', ), PathDirWildcard), Select(PathDirWildcard) ], filter_paths), ] + [ # Link resolution. (Dirs, [ Select(Paths), SelectDependencies( Dirs, Paths, field='link_stats', field_types=(Link, )) ], resolve_dir_links), (Files, [ Select(Paths), SelectDependencies( Files, Paths, field='link_stats', field_types=(Link, )) ], resolve_file_links), (Dirs, [SelectProjection(Dirs, PathGlobs, ('path_globs', ), ReadLink)], resolve_link), (Files, [SelectProjection(Files, PathGlobs, ('path_globs', ), ReadLink)], resolve_link), ] + [ # File content. (FilesContent, [ Select(Files), SelectDependencies( FileContent, Files, field='stats', field_types=(File, )) ], files_content), (FilesDigest, [ Select(Files), SelectDependencies( FileDigest, Files, field='stats', field_types=(File, )) ], files_digest), ]
def create_graph_tasks(address_mapper, symbol_table_cls): """Creates tasks used to parse Structs from BUILD files. :param address_mapper_key: The subject key for an AddressMapper instance. :param symbol_table_cls: A SymbolTable class to provide symbols for Address lookups. """ return [ # Support for resolving Structs from Addresses (Struct, [ Select(UnhydratedStruct), SelectDependencies(Struct, UnhydratedStruct) ], hydrate_struct), (UnhydratedStruct, [ SelectProjection(AddressFamily, Dir, ('spec_path', ), Address), Select(Address) ], resolve_unhydrated_struct), ] + [ # BUILD file parsing. (AddressFamily, [ SelectLiteral(address_mapper, AddressMapper), Select(Dir), SelectProjection(FilesContent, Files, ('files', ), BuildFiles) ], parse_address_family), (BuildFiles, [ SelectLiteral(address_mapper, AddressMapper), Select(DirectoryListing) ], filter_buildfile_paths), ] + [ # Addresses for user-defined products might possibly be resolvable from BLD files. These tasks # define that lookup for each literal product. (product, [Select(Struct)], identity) for product in symbol_table_cls.table().values() if product is not Struct ] + [ # Simple spec handling. (Addresses, [ SelectProjection(AddressFamily, Dir, ('directory', ), SingleAddress), Select(SingleAddress) ], address_from_address_family), (Addresses, [ SelectProjection(AddressFamily, Dir, ('directory', ), SiblingAddresses) ], addresses_from_address_family), ] + [ # Recursive spec handling: locate directories that contain build files, and request # AddressFamilies for each of them. (Addresses, [SelectDependencies(AddressFamily, BuildDirs) ], addresses_from_address_families), (BuildDirs, [Select(Files)], filter_build_dirs), (PathGlobs, [ SelectLiteral(address_mapper, AddressMapper), Select(DescendantAddresses) ], descendant_addresses_to_globs), ]
def create_legacy_graph_tasks(): """Create tasks to recursively parse the legacy graph.""" return [ # Recursively requests the dependencies and adapted fields of TargetAdaptors, which # will result in an eager, transitive graph walk. (LegacyTarget, [Select(TargetAdaptor), SelectDependencies(LegacyTarget, TargetAdaptor, 'dependencies'), SelectDependencies(HydratedField, TargetAdaptor, 'field_adaptors')], reify_legacy_graph), (HydratedField, [Select(SourcesField), SelectProjection(FilesDigest, PathGlobs, ('path_globs',), SourcesField), SelectProjection(Files, PathGlobs, ('excluded_path_globs',), SourcesField)], hydrate_sources), (HydratedField, [Select(BundlesField), SelectDependencies(FilesDigest, BundlesField, 'path_globs_list'), SelectDependencies(Files, BundlesField, 'excluded_path_globs_list')], hydrate_bundles), ]
def test_initial_select_projection_failure(self): rules = _suba_root_rules + [ TaskRule(Exactly(A), [SelectProjection(B, D, 'some', C)], noop), ] validator = self.create_validator({}, rules) with self.assertRaises(ValueError) as cm: validator.assert_ruleset_valid() self.assert_equal_with_printing( dedent(""" Rules with errors: 1 (A, (SelectProjection(B, D, 'some', C),), noop): no matches for Select(C) when resolving SelectProjection(B, D, 'some', C) with subject types: SubA """).strip(), str(cm.exception))
def test_secondary_select_projection_failure(self): rules = [ (Exactly(A), (SelectProjection(B, D, ('some',), C),), noop), (C, tuple(), noop) ] validator = self.create_validator({}, tuple(), _suba_root_subject_types, rules) with self.assertRaises(ValueError) as cm: validator.assert_ruleset_valid() self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (A, (SelectProjection(B, D, ('some',), C),), noop): no matches for Select(B) when resolving SelectProjection(B, D, ('some',), C) with subject types: D """).strip(), str(cm.exception))
def test_secondary_select_projection_failure(self): rules = [(Exactly(A), (SelectProjection(B, D, ('some', ), C), ), noop), (C, tuple(), noop)] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns=_suba_root_subject_fns) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing( dedent(""" Rules with errors: 1 (Exactly(A), (SelectProjection(B, D, (u'some',), C),), noop): no matches for Select(B) when resolving SelectProjection(B, D, (u'some',), C) with subject types: D """).strip(), str(cm.exception))
def test_select_projection_simple(self): rules = [ (Exactly(A), (SelectProjection(B, D, ('some',), SubA),), noop), (B, (Select(D),), noop), ] graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: (Exactly(A), (SelectProjection(B, D, (u'some',), SubA),), noop) of SubA (B, (Select(D),), noop) of D => (SubjectIsProduct(D),) (Exactly(A), (SelectProjection(B, D, (u'some',), SubA),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(D),), noop) of D,) }""").strip(), subgraph)
def test_select_projection_simple(self): rules = [ TaskRule(Exactly(A), [SelectProjection(B, D, 'some', SubA)], noop), TaskRule(B, [Select(D)], noop), ] subgraph = self.create_subgraph(A, rules, SubA()) self.assert_equal_with_printing( dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (SelectProjection(B, D, 'some', SubA),), noop) of SubA"} // internal entries "(A, (SelectProjection(B, D, 'some', SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(D),), noop) of D"} "(B, (Select(D),), noop) of D" -> {"SubjectIsProduct(D)"} }""").strip(), subgraph)
def create_fs_tasks(): """Creates tasks that consume the native filesystem Node type.""" return [ # Glob execution. (Stats, [ SelectProjection(Stats, Dir, ('directory', ), PathWildcard), Select(PathWildcard) ], apply_path_wildcard), (PathGlobs, [ SelectProjection(Dirs, Path, ('directory', ), PathLiteral), Select(PathLiteral) ], apply_path_literal), (PathGlobs, [ SelectProjection(Dirs, Dir, ('directory', ), PathDirWildcard), Select(PathDirWildcard) ], apply_path_dir_wildcard), ] + [ # Link resolution. (Dirs, [Select(Stats), SelectProjection(Dirs, Links, ('links', ), Stats)], resolve_dir_links), (Dirs, [SelectProjection(Dirs, Path, ('path', ), ReadLink)], identity), (Files, [Select(Stats), SelectProjection(Files, Links, ('links', ), Stats)], resolve_file_links), (Files, [SelectProjection(Files, Path, ('path', ), ReadLink)], identity), ] + [ # TODO: These are boilerplatey: aggregation should become native: # see https://github.com/pantsbuild/pants/issues/3169 (Stats, [SelectDependencies(Stats, PathGlobs)], merge_stats), (Stats, [SelectDependencies(Stats, DirectoryListing, field='paths') ], merge_stats), (Files, [SelectDependencies(Files, Links)], merge_files), (Dirs, [SelectDependencies(Dirs, Links)], merge_dirs), (FilesContent, [SelectDependencies(FileContent, Files)], FilesContent), (FilesDigest, [SelectDependencies(FileDigest, Files)], FilesDigest), ]
filespec['globs'], spec_path) if filespec.has_key('exclude'): relpath_adjusted_filespec['exclude'] = [ FilesetRelPathWrapper.to_filespec(e['globs'], spec_path) for e in filespec['exclude'] ] return EagerFilesetWithSpec(spec_path, relpath_adjusted_filespec, files=files, files_hash=snapshot.fingerprint) @rule(HydratedField, [ Select(SourcesField), SelectProjection(Snapshot, PathGlobs, 'path_globs', SourcesField) ]) def hydrate_sources(sources_field, snapshot): """Given a SourcesField and a Snapshot for its path_globs, create an EagerFilesetWithSpec.""" fileset_with_spec = _eager_fileset_with_spec( sources_field.address.spec_path, sources_field.filespecs, snapshot) return HydratedField(sources_field.arg, fileset_with_spec) @rule(HydratedField, [ Select(BundlesField), SelectDependencies( Snapshot, BundlesField, 'path_globs_list', field_types=(PathGlobs, )) ]) def hydrate_bundles(bundles_field, snapshot_list): """Given a BundlesField and a Snapshot for each of its filesets create a list of BundleAdaptors."""
def setup_json_scheduler(build_root, native): """Return a build graph and scheduler configured for BLD.json files under the given build root. :rtype :class:`pants.engine.scheduler.LocalScheduler` """ symbol_table_cls = ExampleTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, build_patterns=('BLD.json',), parser_cls=JsonParser) source_roots = SourceRoots(('src/java','src/scala')) scrooge_tool_address = Address.parse('src/scala/scrooge') goals = { 'compile': Classpath, # TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product. 'resolve': Classpath, 'list': Address, GenGoal.name(): GenGoal, 'ls': Files, 'cat': FilesContent, } tasks = [ # Codegen GenGoal.signature(), (JavaSources, [Select(ThriftSources), SelectVariant(ApacheThriftJavaConfiguration, 'thrift')], gen_apache_thrift), (PythonSources, [Select(ThriftSources), SelectVariant(ApacheThriftPythonConfiguration, 'thrift')], gen_apache_thrift), (ScalaSources, [Select(ThriftSources), SelectVariant(ScroogeScalaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath)], gen_scrooge_thrift), (JavaSources, [Select(ThriftSources), SelectVariant(ScroogeJavaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath)], gen_scrooge_thrift), ] + [ # scala dependency inference (ScalaSources, [Select(ScalaInferredDepsSources), SelectDependencies(Address, ImportedJVMPackages, field_types=(JVMPackageName,))], reify_scala_sources), (ImportedJVMPackages, [SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)], extract_scala_imports), (Address, [Select(JVMPackageName), SelectDependencies(AddressFamily, Dirs, field='stats', field_types=(Dir,))], select_package_address), (PathGlobs, [Select(JVMPackageName), SelectLiteral(source_roots, SourceRoots)], calculate_package_search_path), ] + [ # Remote dependency resolution (Classpath, [Select(Jar)], ivy_resolve), (Jar, [Select(ManagedJar), SelectVariant(ManagedResolve, 'resolve')], select_rev), ] + [ # Compilers (Classpath, [Select(ResourceSources)], isolate_resources), (Classpath, [Select(BuildPropertiesConfiguration)], write_name_file), # NB: Not sure these SelectDependencies should allow Jar, but they currently produce jars. (Classpath, [Select(JavaSources), SelectDependencies(Classpath, JavaSources, field_types=(Address, Jar))], javac), (Classpath, [Select(ScalaSources), SelectDependencies(Classpath, ScalaSources, field_types=(Address, Jar))], scalac), ] + ( create_graph_tasks(address_mapper, symbol_table_cls) ) + ( create_fs_tasks() ) project_tree = FileSystemProjectTree(build_root) return LocalScheduler(goals, tasks, project_tree, native, graph_lock=None)
class BuildDirs(datatype('BuildDirs', ['dependencies'])): """A list of Stat objects for directories containing build files.""" class BuildFiles(datatype('BuildFiles', ['files_content'])): """The FileContents of BUILD files in some directory""" class BuildFileGlobs(datatype('BuildFilesGlobs', ['path_globs'])): """A wrapper around PathGlobs that are known to match a build file pattern.""" @rule(BuildFiles, [SelectProjection(FilesContent, PathGlobs, 'path_globs', BuildFileGlobs)]) def build_files(files_content): return BuildFiles(files_content) @rule(BuildFileGlobs, [Select(AddressMapper), Select(Dir)]) def buildfile_path_globs_for_dir(address_mapper, directory): patterns = address_mapper.build_patterns return BuildFileGlobs(PathGlobs.create(directory.path, include=patterns, exclude=())) @rule(AddressFamily, [Select(AddressMapper), Select(Dir), Select(BuildFiles)]) def parse_address_family(address_mapper, path, build_files): """Given the contents of the build files in one directory, return an AddressFamily. The AddressFamily may be empty, but it will not be None.
def test_projection_repr(self): self.assert_repr( "SelectProjection(AClass, AClass, (u'field',), AClass)", SelectProjection(AClass, AClass, ('field', ), AClass))
raise ValueError('Multiple targets might be able to provide {}:\n {}'.format( jvm_package_name, '\n '.join(str(a) for a in addresses))) return addresses[0].to_address() @rule(PathGlobs, [Select(JVMPackageName), Select(SourceRoots)]) @printing_func def calculate_package_search_path(jvm_package_name, source_roots): """Return PathGlobs to match directories where the given JVMPackageName might exist.""" rel_package_dir = jvm_package_name.name.replace('.', os_sep) specs = [os_path_join(srcroot, rel_package_dir) for srcroot in source_roots.srcroots] return PathGlobs.create('', include=specs) @rule(ImportedJVMPackages, [SelectProjection(FilesContent, PathGlobs, 'path_globs', ScalaInferredDepsSources)]) @printing_func def extract_scala_imports(source_files_content): """A toy example of dependency inference. Would usually be a compiler plugin.""" packages = set() import_re = re.compile(r'^import ([^;]*);?$') for filecontent in source_files_content.dependencies: for line in filecontent.content.splitlines(): match = import_re.search(line) if match: packages.add(match.group(1).rsplit('.', 1)[0]) return ImportedJVMPackages([JVMPackageName(p) for p in packages]) @rule(ScalaSources, [Select(ScalaInferredDepsSources),