def create_fs_tasks(buildroot):
"""Creates tasks that consume the filesystem.
These tasks are all considered "native", and should have their outputs re-validated
for every build. TODO: They should likely get their own ProductGraph.Node type
for efficiency/invalidation.
:param project_tree: A ProjectTree instance to use for filesystem operations.
"""
fspt = FileSystemProjectTree(buildroot)
return [
# Unfiltered requests for subdirectories.
(RecursiveSubDirectories, [
Select(Path),
SelectDependencies(
RecursiveSubDirectories, DirectoryListing, field='directories')
], recursive_subdirectories),
] + [
# "Native" operations.
(Paths, [SelectDependencies(Paths, PathGlobs)], merge_paths),
(Paths, [SelectLiteral(fspt, ProjectTree),
Select(PathGlob)], file_exists),
(FilesContent, [SelectLiteral(fspt, ProjectTree),
Select(Paths)], files_content),
(DirectoryListing, [SelectLiteral(fspt, ProjectTree),
Select(Path)], list_directory),
]
def create_graph_tasks(address_mapper, symbol_table_cls):
"""Creates tasks used to parse Structs from BUILD files.
:param address_mapper: An AddressMapper instance.
:param symbol_table_cls: A symbol table class.
"""
return [
# Support for resolving Structs from Addresses
(Struct, [
Select(UnhydratedStruct),
SelectDependencies(Struct, UnhydratedStruct)
], hydrate_struct),
(UnhydratedStruct, [
SelectProjection(AddressFamily, Path, ('spec_path', ), Address),
Select(Address)
], resolve_unhydrated_struct),
] + [
# BUILD file parsing.
(AddressFamily, [
SelectLiteral(address_mapper, AddressMapper),
Select(Path),
SelectProjection(FilesContent, Paths, ('paths', ), BuildFilePaths)
], parse_address_family),
(BuildFilePaths, [
SelectLiteral(address_mapper, AddressMapper),
Select(DirectoryListing)
], filter_buildfile_paths),
] + [
# Addresses for 'literal' products might possibly be resolvable from BLD files. These tasks
# define that lookup for each literal product.
(product, [Select(Struct)], identity)
for product in symbol_table_cls.table().values()
] + [
# Spec handling.
(Addresses, [
SelectProjection(AddressFamily, Path,
('directory', ), SiblingAddresses)
], addresses_from_address_family),
(Addresses, [
SelectDependencies(AddressFamily, RecursiveSubDirectories)
], addresses_from_address_families),
# TODO: This is a workaround for the fact that we can't currently "project" in a
# SelectDependencies clause: we launch the recursion by requesting RecursiveSubDirectories
# for a Directory projected from DescendantAddresses.
(RecursiveSubDirectories, [
SelectProjection(RecursiveSubDirectories, Path,
('directory', ), DescendantAddresses)
], identity),
]
def signature(cls):
"""Returns a task triple for this Goal, used to install the Goal.
A Goal is it's own synthetic output product, and its constructor acts as its task function. It
selects each of its products as optional, but fails synchronously if none of them are available.
"""
return (cls, [Select(p, optional=True) for p in cls.products()], cls)
def create_legacy_graph_tasks():
"""Create tasks to recursively parse the legacy graph."""
return [
# Recursively requests LegacyGraphNodes for TargetAdaptors, which will result in a
# transitive graph walk.
(LegacyBuildGraphNode, [
Select(TargetAdaptor),
SelectDependencies(LegacyBuildGraphNode, TargetAdaptor)
], reify_legacy_graph),
]
def create_fs_tasks():
"""Creates tasks that consume the native filesystem Node type."""
return [
# Glob execution.
(Stats, [
SelectProjection(Stats, Dir, ('directory', ), PathWildcard),
Select(PathWildcard)
], apply_path_wildcard),
(PathGlobs, [
SelectProjection(Dirs, Path, ('directory', ), PathLiteral),
Select(PathLiteral)
], apply_path_literal),
(PathGlobs, [
SelectProjection(Dirs, Dir, ('directory', ), PathDirWildcard),
Select(PathDirWildcard)
], apply_path_dir_wildcard),
] + [
# Link resolution.
(Dirs,
[Select(Stats),
SelectProjection(Dirs, Links,
('links', ), Stats)], resolve_dir_links),
(Dirs, [SelectProjection(Dirs, Path, ('path', ), ReadLink)], identity),
(Files,
[Select(Stats),
SelectProjection(Files, Links,
('links', ), Stats)], resolve_file_links),
(Files, [SelectProjection(Files, Path,
('path', ), ReadLink)], identity),
] + [
# TODO: These are boilerplatey: aggregation should become native:
# see https://github.com/pantsbuild/pants/issues/3169
(Stats, [SelectDependencies(Stats, PathGlobs)], merge_stats),
(Stats, [SelectDependencies(Stats, DirectoryListing, field='paths')
], merge_stats),
(Files, [SelectDependencies(Files, Links)], merge_files),
(Dirs, [SelectDependencies(Dirs, Links)], merge_dirs),
(FilesContent, [SelectDependencies(FileContent, Files)
], files_content),
]
def create_fs_tasks():
"""Creates tasks that consume the native filesystem Node type."""
return [
(RecursiveSubDirectories, [
Select(Path),
SelectDependencies(RecursiveSubDirectories,
DirectoryListing,
field='directories')
], recursive_subdirectories),
(Paths, [SelectDependencies(Paths, PathGlobs)], merge_paths),
(Paths, [
SelectProjection(DirectoryListing, Path, ('directory', ),
PathWildcard),
Select(PathWildcard)
], filter_file_listing),
(PathGlobs, [
SelectProjection(DirectoryListing, Path, ('directory', ),
PathDirWildcard),
Select(PathDirWildcard)
], filter_dir_listing),
(FilesContent, [SelectDependencies(FileContent,
Paths)], files_content),
]
def create_graph_tasks(address_mapper, symbol_table_cls):
"""Creates tasks used to parse Structs from BUILD files.
:param address_mapper_key: The subject key for an AddressMapper instance.
:param symbol_table_cls: A SymbolTable class to provide symbols for Address lookups.
"""
return [
# Support for resolving Structs from Addresses
(Struct, [
Select(UnhydratedStruct),
SelectDependencies(Struct, UnhydratedStruct)
], hydrate_struct),
(UnhydratedStruct, [
SelectProjection(AddressFamily, Dir, ('spec_path', ), Address),
Select(Address)
], resolve_unhydrated_struct),
] + [
# BUILD file parsing.
(AddressFamily, [
SelectLiteral(address_mapper, AddressMapper),
Select(Dir),
SelectProjection(FilesContent, Files, ('files', ), BuildFiles)
], parse_address_family),
(BuildFiles,
[SelectLiteral(address_mapper, AddressMapper),
Select(Files)], filter_buildfile_paths),
] + [
# Addresses for user-defined products might possibly be resolvable from BLD files. These tasks
# define that lookup for each literal product.
(product, [Select(Struct)], identity)
for product in symbol_table_cls.table().values()
] + [
# Spec handling.
(Addresses, [
SelectProjection(AddressFamily, Dir,
('directory', ), SingleAddress),
Select(SingleAddress)
], address_from_address_family),
(Addresses, [
SelectProjection(AddressFamily, Dir,
('directory', ), SiblingAddresses)
], addresses_from_address_family),
(Addresses, [SelectDependencies(AddressFamily, Dirs)
], addresses_from_address_families),
(PathGlobs, [Select(DescendantAddresses)
], descendant_addresses_to_globs),
]
def setup_json_scheduler(build_root, debug=True):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype A tuple of :class:`pants.engine.exp.scheduler.LocalScheduler`,
:class:`pants.engine.exp.storage.Storage`.
"""
storage = Storage.create(debug=debug)
symbol_table_cls = ExampleTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=r'^BLD.json$',
parser_cls=JsonParser)
source_roots = SourceRoots(('src/java','src/scala'))
scrooge_tool_address = Address.parse('src/scala/scrooge')
goals = {
'compile': Classpath,
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': Classpath,
'list': Address,
GenGoal.name(): GenGoal,
'unpickleable': UnpickleableResult,
'ls': File,
'cat': FileContent,
}
tasks = [
# Codegen
GenGoal.signature(),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftJavaConfiguration, 'thrift')],
gen_apache_thrift),
(PythonSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftPythonConfiguration, 'thrift')],
gen_apache_thrift),
(ScalaSources,
[Select(ThriftSources),
SelectVariant(ScroogeScalaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ScroogeJavaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
] + [
# scala dependency inference
(ScalaSources,
[Select(ScalaInferredDepsSources),
SelectDependencies(Address, ImportedJVMPackages)],
reify_scala_sources),
(ImportedJVMPackages,
[SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)],
extract_scala_imports),
(Address,
[Select(JVMPackageName),
SelectDependencies(AddressFamily, Dirs)],
select_package_address),
(PathGlobs,
[Select(JVMPackageName),
SelectLiteral(source_roots, SourceRoots)],
calculate_package_search_path),
] + [
# Remote dependency resolution
(Classpath,
[Select(Jar)],
ivy_resolve),
(Jar,
[Select(ManagedJar),
SelectVariant(ManagedResolve, 'resolve')],
select_rev),
] + [
# Compilers
(Classpath,
[Select(ResourceSources)],
isolate_resources),
(Classpath,
[Select(BuildPropertiesConfiguration)],
write_name_file),
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources)],
javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources)],
scalac),
] + [
# TODO
(UnpickleableOutput,
[],
unpickleable_output),
(UnpickleableResult,
[Select(UnpickleableOutput)],
unpickleable_input),
] + (
create_graph_tasks(address_mapper, symbol_table_cls)
) + (
create_fs_tasks()
)
project_tree = FileSystemProjectTree(build_root)
return LocalScheduler(goals, tasks, storage, project_tree, None, GraphValidator(symbol_table_cls)), storage