def create_legacy_graph_tasks(symbol_table_cls):
"""Create tasks to recursively parse the legacy graph."""
symbol_table_constraint = symbol_table_cls.constraint()
return [
# Recursively requests HydratedTargets, which will result in an eager, transitive graph walk.
(HydratedTargets,
[SelectDependencies(HydratedTarget,
Addresses,
field_types=(Address,), transitive=True)],
HydratedTargets),
(HydratedTarget,
[Select(symbol_table_constraint),
SelectDependencies(HydratedField,
symbol_table_constraint,
'field_adaptors',
field_types=(SourcesField, BundlesField,))],
hydrate_target),
(HydratedField,
[Select(SourcesField),
SelectProjection(FilesDigest, PathGlobs, ('path_globs',), SourcesField),
SelectProjection(Files, PathGlobs, ('excluded_path_globs',), SourcesField)],
hydrate_sources),
(HydratedField,
[Select(BundlesField),
SelectDependencies(FilesDigest, BundlesField, 'path_globs_list', field_types=(PathGlobs,)),
SelectDependencies(Files, BundlesField, 'excluded_path_globs_list', field_types=(PathGlobs,))],
hydrate_bundles),
]
def create_graph_tasks(address_mapper, symbol_table_cls):
"""Creates tasks used to parse Structs from BUILD files.
:param address_mapper_key: The subject key for an AddressMapper instance.
:param symbol_table_cls: A SymbolTable class to provide symbols for Address lookups.
"""
symbol_table_constraint = symbol_table_cls.constraint()
return [
# Support for resolving Structs from Addresses
(symbol_table_constraint, [
Select(UnhydratedStruct),
SelectDependencies(symbol_table_constraint,
UnhydratedStruct,
field_types=(Address, ))
], hydrate_struct),
(UnhydratedStruct, [
SelectProjection(AddressFamily, Dir, ('spec_path', ), Address),
Select(Address)
], resolve_unhydrated_struct),
] + [
# BUILD file parsing.
(AddressFamily, [
SelectLiteral(address_mapper, AddressMapper),
Select(Dir),
SelectProjection(FilesContent, Files, ('files', ), BuildFiles)
], parse_address_family),
(BuildFiles, [
SelectLiteral(address_mapper, AddressMapper),
Select(DirectoryListing)
], filter_buildfile_paths),
] + [
# Simple spec handling.
(Addresses, [
SelectProjection(AddressFamily, Dir,
('directory', ), SingleAddress),
Select(SingleAddress)
], address_from_address_family),
(Addresses, [
SelectProjection(AddressFamily, Dir,
('directory', ), SiblingAddresses)
], addresses_from_address_family),
] + [
# Recursive spec handling: locate directories that contain build files, and request
# AddressFamilies for each of them.
(Addresses, [
SelectDependencies(AddressFamily, BuildDirs, field_types=(Dir, ))
], addresses_from_address_families),
(BuildDirs,
[SelectLiteral(address_mapper, AddressMapper),
Select(Files)], filter_build_dirs),
(PathGlobs, [
SelectLiteral(address_mapper, AddressMapper),
Select(DescendantAddresses)
], descendant_addresses_to_globs),
(PathGlobs, [
SelectLiteral(address_mapper, AddressMapper),
Select(AscendantAddresses)
], ascendant_addresses_to_globs),
]
def create_fs_tasks():
"""Creates tasks that consume the intrinsic filesystem types."""
return [
# Glob execution: to avoid memoizing lots of incremental results, we recursively expand PathGlobs, and then
# convert them to Paths independently.
(Paths, [
SelectDependencies(
PathsExpansion,
PathGlobs,
field_types=(PathWildcard, PathDirWildcard, PathRoot),
transitive=True)
], finalize_path_expansion),
(PathsExpansion, [Select(PathRoot)], apply_path_root),
(PathsExpansion, [
SelectProjection(DirectoryListing, Dir,
('canonical_stat', ), PathWildcard),
Select(PathWildcard)
], apply_path_wildcard),
(PathsExpansion, [
SelectProjection(Dirs, Paths, ('paths', ), FilteredPaths),
Select(PathDirWildcard)
], apply_path_dir_wildcard),
(FilteredPaths, [
SelectProjection(DirectoryListing, Dir,
('canonical_stat', ), PathDirWildcard),
Select(PathDirWildcard)
], filter_paths),
] + [
# Link resolution.
(Dirs, [
Select(Paths),
SelectDependencies(
Dirs, Paths, field='link_stats', field_types=(Link, ))
], resolve_dir_links),
(Files, [
Select(Paths),
SelectDependencies(
Files, Paths, field='link_stats', field_types=(Link, ))
], resolve_file_links),
(Dirs, [SelectProjection(Dirs, PathGlobs,
('path_globs', ), ReadLink)], resolve_link),
(Files,
[SelectProjection(Files, PathGlobs,
('path_globs', ), ReadLink)], resolve_link),
] + [
# File content.
(FilesContent, [
Select(Files),
SelectDependencies(
FileContent, Files, field='stats', field_types=(File, ))
], files_content),
(FilesDigest, [
Select(Files),
SelectDependencies(
FileDigest, Files, field='stats', field_types=(File, ))
], files_digest),
]
def create_fs_tasks():
"""Creates tasks that consume the native filesystem Node type."""
return [
# Glob execution.
(Paths, [
SelectDependencies(
Paths,
PathGlobs,
field_types=(PathWildcard, PathDirWildcard, PathRoot))
], merge_paths),
(Paths, [Select(PathRoot)], apply_path_root),
(Paths, [
SelectProjection(DirectoryListing, Dir,
('canonical_stat', ), PathWildcard),
Select(PathWildcard)
], apply_path_wildcard),
(PathGlobs, [
SelectProjection(Dirs, Paths, ('paths', ), FilteredPaths),
Select(PathDirWildcard)
], apply_path_dir_wildcard),
(FilteredPaths, [
SelectProjection(DirectoryListing, Dir,
('canonical_stat', ), PathDirWildcard),
Select(PathDirWildcard)
], filter_paths),
] + [
# Link resolution.
(Dirs, [
Select(Paths),
SelectDependencies(
Dirs, Paths, field='link_stats', field_types=(Link, ))
], resolve_dir_links),
(Files, [
Select(Paths),
SelectDependencies(
Files, Paths, field='link_stats', field_types=(Link, ))
], resolve_file_links),
(Dirs, [SelectProjection(Dirs, PathGlobs,
('path_globs', ), ReadLink)], resolve_link),
(Files,
[SelectProjection(Files, PathGlobs,
('path_globs', ), ReadLink)], resolve_link),
] + [
# File content.
(FilesContent, [
Select(Files),
SelectDependencies(
FileContent, Files, field='stats', field_types=(File, ))
], files_content),
(FilesDigest, [
Select(Files),
SelectDependencies(
FileDigest, Files, field='stats', field_types=(File, ))
], files_digest),
]
def create_graph_tasks(address_mapper, symbol_table_cls):
"""Creates tasks used to parse Structs from BUILD files.
:param address_mapper_key: The subject key for an AddressMapper instance.
:param symbol_table_cls: A SymbolTable class to provide symbols for Address lookups.
"""
return [
# Support for resolving Structs from Addresses
(Struct, [
Select(UnhydratedStruct),
SelectDependencies(Struct, UnhydratedStruct)
], hydrate_struct),
(UnhydratedStruct, [
SelectProjection(AddressFamily, Dir, ('spec_path', ), Address),
Select(Address)
], resolve_unhydrated_struct),
] + [
# BUILD file parsing.
(AddressFamily, [
SelectLiteral(address_mapper, AddressMapper),
Select(Dir),
SelectProjection(FilesContent, Files, ('files', ), BuildFiles)
], parse_address_family),
(BuildFiles, [
SelectLiteral(address_mapper, AddressMapper),
Select(DirectoryListing)
], filter_buildfile_paths),
] + [
# Addresses for user-defined products might possibly be resolvable from BLD files. These tasks
# define that lookup for each literal product.
(product, [Select(Struct)], identity)
for product in symbol_table_cls.table().values()
if product is not Struct
] + [
# Simple spec handling.
(Addresses, [
SelectProjection(AddressFamily, Dir,
('directory', ), SingleAddress),
Select(SingleAddress)
], address_from_address_family),
(Addresses, [
SelectProjection(AddressFamily, Dir,
('directory', ), SiblingAddresses)
], addresses_from_address_family),
] + [
# Recursive spec handling: locate directories that contain build files, and request
# AddressFamilies for each of them.
(Addresses, [SelectDependencies(AddressFamily, BuildDirs)
], addresses_from_address_families),
(BuildDirs, [Select(Files)], filter_build_dirs),
(PathGlobs, [
SelectLiteral(address_mapper, AddressMapper),
Select(DescendantAddresses)
], descendant_addresses_to_globs),
]
def create_legacy_graph_tasks():
"""Create tasks to recursively parse the legacy graph."""
return [
# Recursively requests the dependencies and adapted fields of TargetAdaptors, which
# will result in an eager, transitive graph walk.
(LegacyTarget,
[Select(TargetAdaptor),
SelectDependencies(LegacyTarget, TargetAdaptor, 'dependencies'),
SelectDependencies(HydratedField, TargetAdaptor, 'field_adaptors')],
reify_legacy_graph),
(HydratedField,
[Select(SourcesField),
SelectProjection(FilesDigest, PathGlobs, ('path_globs',), SourcesField),
SelectProjection(Files, PathGlobs, ('excluded_path_globs',), SourcesField)],
hydrate_sources),
(HydratedField,
[Select(BundlesField),
SelectDependencies(FilesDigest, BundlesField, 'path_globs_list'),
SelectDependencies(Files, BundlesField, 'excluded_path_globs_list')],
hydrate_bundles),
]
def test_initial_select_projection_failure(self):
rules = _suba_root_rules + [
TaskRule(Exactly(A), [SelectProjection(B, D, 'some', C)], noop),
]
validator = self.create_validator({}, rules)
with self.assertRaises(ValueError) as cm:
validator.assert_ruleset_valid()
self.assert_equal_with_printing(
dedent("""
Rules with errors: 1
(A, (SelectProjection(B, D, 'some', C),), noop):
no matches for Select(C) when resolving SelectProjection(B, D, 'some', C) with subject types: SubA
""").strip(), str(cm.exception))
def test_secondary_select_projection_failure(self):
rules = [
(Exactly(A), (SelectProjection(B, D, ('some',), C),), noop),
(C, tuple(), noop)
]
validator = self.create_validator({}, tuple(), _suba_root_subject_types, rules)
with self.assertRaises(ValueError) as cm:
validator.assert_ruleset_valid()
self.assert_equal_with_printing(dedent("""
Rules with errors: 1
(A, (SelectProjection(B, D, ('some',), C),), noop):
no matches for Select(B) when resolving SelectProjection(B, D, ('some',), C) with subject types: D
""").strip(),
str(cm.exception))
def test_secondary_select_projection_failure(self):
rules = [(Exactly(A), (SelectProjection(B, D, ('some', ), C), ), noop),
(C, tuple(), noop)]
validator = RulesetValidator(RuleIndex.create(rules, tuple()),
goal_to_product={},
root_subject_fns=_suba_root_subject_fns)
with self.assertRaises(ValueError) as cm:
validator.validate()
self.assert_equal_with_printing(
dedent("""
Rules with errors: 1
(Exactly(A), (SelectProjection(B, D, (u'some',), C),), noop):
no matches for Select(B) when resolving SelectProjection(B, D, (u'some',), C) with subject types: D
""").strip(), str(cm.exception))
def test_select_projection_simple(self):
rules = [
(Exactly(A), (SelectProjection(B, D, ('some',), SubA),), noop),
(B, (Select(D),), noop),
]
graphmaker = GraphMaker(NodeBuilder.create(rules, tuple()),
root_subject_fns=_suba_root_subject_fns)
subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A)
self.assert_equal_with_printing(dedent("""
{
root_subject_types: (SubA,)
root_rules: (Exactly(A), (SelectProjection(B, D, (u'some',), SubA),), noop) of SubA
(B, (Select(D),), noop) of D => (SubjectIsProduct(D),)
(Exactly(A), (SelectProjection(B, D, (u'some',), SubA),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(D),), noop) of D,)
}""").strip(),
subgraph)
def test_select_projection_simple(self):
rules = [
TaskRule(Exactly(A), [SelectProjection(B, D, 'some', SubA)], noop),
TaskRule(B, [Select(D)], noop),
]
subgraph = self.create_subgraph(A, rules, SubA())
self.assert_equal_with_printing(
dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, (SelectProjection(B, D, 'some', SubA),), noop) of SubA"}
// internal entries
"(A, (SelectProjection(B, D, 'some', SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(D),), noop) of D"}
"(B, (Select(D),), noop) of D" -> {"SubjectIsProduct(D)"}
}""").strip(), subgraph)
def create_fs_tasks():
"""Creates tasks that consume the native filesystem Node type."""
return [
# Glob execution.
(Stats, [
SelectProjection(Stats, Dir, ('directory', ), PathWildcard),
Select(PathWildcard)
], apply_path_wildcard),
(PathGlobs, [
SelectProjection(Dirs, Path, ('directory', ), PathLiteral),
Select(PathLiteral)
], apply_path_literal),
(PathGlobs, [
SelectProjection(Dirs, Dir, ('directory', ), PathDirWildcard),
Select(PathDirWildcard)
], apply_path_dir_wildcard),
] + [
# Link resolution.
(Dirs,
[Select(Stats),
SelectProjection(Dirs, Links,
('links', ), Stats)], resolve_dir_links),
(Dirs, [SelectProjection(Dirs, Path, ('path', ), ReadLink)], identity),
(Files,
[Select(Stats),
SelectProjection(Files, Links,
('links', ), Stats)], resolve_file_links),
(Files, [SelectProjection(Files, Path,
('path', ), ReadLink)], identity),
] + [
# TODO: These are boilerplatey: aggregation should become native:
# see https://github.com/pantsbuild/pants/issues/3169
(Stats, [SelectDependencies(Stats, PathGlobs)], merge_stats),
(Stats, [SelectDependencies(Stats, DirectoryListing, field='paths')
], merge_stats),
(Files, [SelectDependencies(Files, Links)], merge_files),
(Dirs, [SelectDependencies(Dirs, Links)], merge_dirs),
(FilesContent, [SelectDependencies(FileContent, Files)], FilesContent),
(FilesDigest, [SelectDependencies(FileDigest, Files)], FilesDigest),
]
filespec['globs'], spec_path)
if filespec.has_key('exclude'):
relpath_adjusted_filespec['exclude'] = [
FilesetRelPathWrapper.to_filespec(e['globs'], spec_path)
for e in filespec['exclude']
]
return EagerFilesetWithSpec(spec_path,
relpath_adjusted_filespec,
files=files,
files_hash=snapshot.fingerprint)
@rule(HydratedField, [
Select(SourcesField),
SelectProjection(Snapshot, PathGlobs, 'path_globs', SourcesField)
])
def hydrate_sources(sources_field, snapshot):
"""Given a SourcesField and a Snapshot for its path_globs, create an EagerFilesetWithSpec."""
fileset_with_spec = _eager_fileset_with_spec(
sources_field.address.spec_path, sources_field.filespecs, snapshot)
return HydratedField(sources_field.arg, fileset_with_spec)
@rule(HydratedField, [
Select(BundlesField),
SelectDependencies(
Snapshot, BundlesField, 'path_globs_list', field_types=(PathGlobs, ))
])
def hydrate_bundles(bundles_field, snapshot_list):
"""Given a BundlesField and a Snapshot for each of its filesets create a list of BundleAdaptors."""
def setup_json_scheduler(build_root, native):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype :class:`pants.engine.scheduler.LocalScheduler`
"""
symbol_table_cls = ExampleTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_patterns=('BLD.json',),
parser_cls=JsonParser)
source_roots = SourceRoots(('src/java','src/scala'))
scrooge_tool_address = Address.parse('src/scala/scrooge')
goals = {
'compile': Classpath,
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': Classpath,
'list': Address,
GenGoal.name(): GenGoal,
'ls': Files,
'cat': FilesContent,
}
tasks = [
# Codegen
GenGoal.signature(),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftJavaConfiguration, 'thrift')],
gen_apache_thrift),
(PythonSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftPythonConfiguration, 'thrift')],
gen_apache_thrift),
(ScalaSources,
[Select(ThriftSources),
SelectVariant(ScroogeScalaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ScroogeJavaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
] + [
# scala dependency inference
(ScalaSources,
[Select(ScalaInferredDepsSources),
SelectDependencies(Address, ImportedJVMPackages, field_types=(JVMPackageName,))],
reify_scala_sources),
(ImportedJVMPackages,
[SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)],
extract_scala_imports),
(Address,
[Select(JVMPackageName),
SelectDependencies(AddressFamily, Dirs, field='stats', field_types=(Dir,))],
select_package_address),
(PathGlobs,
[Select(JVMPackageName),
SelectLiteral(source_roots, SourceRoots)],
calculate_package_search_path),
] + [
# Remote dependency resolution
(Classpath,
[Select(Jar)],
ivy_resolve),
(Jar,
[Select(ManagedJar),
SelectVariant(ManagedResolve, 'resolve')],
select_rev),
] + [
# Compilers
(Classpath,
[Select(ResourceSources)],
isolate_resources),
(Classpath,
[Select(BuildPropertiesConfiguration)],
write_name_file),
# NB: Not sure these SelectDependencies should allow Jar, but they currently produce jars.
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources, field_types=(Address, Jar))],
javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources, field_types=(Address, Jar))],
scalac),
] + (
create_graph_tasks(address_mapper, symbol_table_cls)
) + (
create_fs_tasks()
)
project_tree = FileSystemProjectTree(build_root)
return LocalScheduler(goals,
tasks,
project_tree,
native,
graph_lock=None)
class BuildDirs(datatype('BuildDirs', ['dependencies'])):
"""A list of Stat objects for directories containing build files."""
class BuildFiles(datatype('BuildFiles', ['files_content'])):
"""The FileContents of BUILD files in some directory"""
class BuildFileGlobs(datatype('BuildFilesGlobs', ['path_globs'])):
"""A wrapper around PathGlobs that are known to match a build file pattern."""
@rule(BuildFiles,
[SelectProjection(FilesContent, PathGlobs, 'path_globs', BuildFileGlobs)])
def build_files(files_content):
return BuildFiles(files_content)
@rule(BuildFileGlobs, [Select(AddressMapper), Select(Dir)])
def buildfile_path_globs_for_dir(address_mapper, directory):
patterns = address_mapper.build_patterns
return BuildFileGlobs(PathGlobs.create(directory.path, include=patterns, exclude=()))
@rule(AddressFamily, [Select(AddressMapper), Select(Dir), Select(BuildFiles)])
def parse_address_family(address_mapper, path, build_files):
"""Given the contents of the build files in one directory, return an AddressFamily.
The AddressFamily may be empty, but it will not be None.
def test_projection_repr(self):
self.assert_repr(
"SelectProjection(AClass, AClass, (u'field',), AClass)",
SelectProjection(AClass, AClass, ('field', ), AClass))
raise ValueError('Multiple targets might be able to provide {}:\n {}'.format(
jvm_package_name, '\n '.join(str(a) for a in addresses)))
return addresses[0].to_address()
@rule(PathGlobs, [Select(JVMPackageName), Select(SourceRoots)])
@printing_func
def calculate_package_search_path(jvm_package_name, source_roots):
"""Return PathGlobs to match directories where the given JVMPackageName might exist."""
rel_package_dir = jvm_package_name.name.replace('.', os_sep)
specs = [os_path_join(srcroot, rel_package_dir) for srcroot in source_roots.srcroots]
return PathGlobs.create('', include=specs)
@rule(ImportedJVMPackages,
[SelectProjection(FilesContent, PathGlobs, 'path_globs', ScalaInferredDepsSources)])
@printing_func
def extract_scala_imports(source_files_content):
"""A toy example of dependency inference. Would usually be a compiler plugin."""
packages = set()
import_re = re.compile(r'^import ([^;]*);?$')
for filecontent in source_files_content.dependencies:
for line in filecontent.content.splitlines():
match = import_re.search(line)
if match:
packages.add(match.group(1).rsplit('.', 1)[0])
return ImportedJVMPackages([JVMPackageName(p) for p in packages])
@rule(ScalaSources,
[Select(ScalaInferredDepsSources),
