def test_full_graph_for_planner_example(self):
symbol_table_cls = TargetTable
address_mapper = AddressMapper(symbol_table_cls, JsonParser, '*.BUILD.json')
tasks = create_graph_tasks(address_mapper, symbol_table_cls) + create_fs_tasks()
intrinsics = create_fs_intrinsics('Let us pretend that this is a ProjectTree!')
rule_index = NodeBuilder.create(tasks, intrinsics)
graphmaker = GraphMaker(rule_index,
root_subject_fns={k: lambda p: Select(p) for k in (Address, # TODO, use the actual fns.
PathGlobs,
SingleAddress,
SiblingAddresses,
DescendantAddresses,
AscendantAddresses
)})
fullgraph = graphmaker.full_graph()
print('---diagnostic------')
print(fullgraph.error_message())
print('/---diagnostic------')
print(fullgraph)
# Assert that all of the rules specified the various task fns are present
declared_rules = rule_index.all_rules()
rules_remaining_in_graph_strs = set(str(r.rule) for r in fullgraph.rule_dependencies.keys())
declared_rule_strings = set(str(r) for r in declared_rules)
self.assertEquals(declared_rule_strings,
rules_remaining_in_graph_strs
)
def test_full_graph_for_planner_example(self):
symbol_table_cls = TargetTable
address_mapper = AddressMapper(symbol_table_cls, JsonParser, '*.BUILD.json')
tasks = create_graph_tasks(address_mapper, symbol_table_cls) + create_fs_tasks()
intrinsics = create_fs_intrinsics('Let us pretend that this is a ProjectTree!')
rule_index = RuleIndex.create(tasks, intrinsics)
graphmaker = GraphMaker(rule_index,
root_subject_fns={k: lambda p: Select(p) for k in (Address, # TODO, use the actual fns.
PathGlobs,
SingleAddress,
SiblingAddresses,
DescendantAddresses,
AscendantAddresses
)})
fullgraph = graphmaker.full_graph()
print('---diagnostic------')
print(fullgraph.error_message())
print('/---diagnostic------')
print(fullgraph)
# Assert that all of the rules specified the various task fns are present
declared_rules = rule_index.all_rules()
rules_remaining_in_graph_strs = set(str(r.rule) for r in fullgraph.rule_dependencies.keys())
declared_rule_strings = set(str(r) for r in declared_rules)
self.assertEquals(declared_rule_strings,
rules_remaining_in_graph_strs
)
# statically assert that the number of dependency keys is fixed
self.assertEquals(41, len(fullgraph.rule_dependencies))
def create(self, build_pattern=None, parser_cls=None):
symbol_table_cls = TestTable
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=build_pattern,
parser_cls=parser_cls)
tasks = create_graph_tasks(address_mapper, symbol_table_cls)
project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples'))
scheduler = self.mk_scheduler(tasks=tasks, project_tree=project_tree)
return scheduler
def setup_legacy_graph(pants_ignore_patterns,
build_root=None,
native=None,
symbol_table_cls=None,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None):
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree,
usually taken from the '--pants-ignore' global option.
:param str build_root: A path to be used as the build root. If None, then default is used.
:param Native native: An instance of the native-engine subsystem.
:param SymbolTable symbol_table_cls: A SymbolTable class to use for build file parsing, or
None to use the default.
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:param list subproject_roots: Paths that correspond with embedded build roots
under the current build root.
:returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls).
"""
build_root = build_root or get_buildroot()
scm = get_scm()
symbol_table_cls = symbol_table_cls or LegacySymbolTable
project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns)
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots)
# Load the native backend.
native = native or Native.Factory.global_instance().create()
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
tasks = (
create_legacy_graph_tasks(symbol_table_cls) +
create_fs_tasks(project_tree) +
create_graph_tasks(address_mapper, symbol_table_cls)
)
# TODO: Do not use the cache yet, as it incurs a high overhead.
scheduler = LocalScheduler(dict(), tasks, project_tree, native)
engine = LocalSerialEngine(scheduler, use_cache=False)
change_calculator = EngineChangeCalculator(engine, scm) if scm else None
return LegacyGraphHelper(scheduler, engine, symbol_table_cls, change_calculator)
def create(self, build_pattern=None, parser_cls=None):
symbol_table_cls = TestTable
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=build_pattern,
parser_cls=parser_cls)
tasks = create_graph_tasks(address_mapper, symbol_table_cls)
project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples'))
scheduler = self.mk_scheduler(tasks=tasks, project_tree=project_tree)
return scheduler
def setUp(self):
# Set up a scheduler that supports address mapping.
symbol_table_cls = TargetTable
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=JsonParser,
build_pattern='*.BUILD.json')
tasks = create_graph_tasks(address_mapper, symbol_table_cls)
project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'))
self.build_root = project_tree.build_root
self.scheduler = self.mk_scheduler(tasks=tasks, project_tree=project_tree)
self.a_b = Address.parse('a/b')
self.a_b_target = Target(name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')],
type_alias='target')
def setUp(self):
# Set up a scheduler that supports address mapping.
symbol_table_cls = TargetTable
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=JsonParser,
build_patterns=('*.BUILD.json',))
tasks = create_graph_tasks(address_mapper, symbol_table_cls)
project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'))
self.build_root = project_tree.build_root
self.scheduler = self.mk_scheduler(tasks=tasks, project_tree=project_tree)
self.a_b = Address.parse('a/b')
self.a_b_target = Target(name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')],
type_alias='target')
def test_full_graph_for_planner_example(self):
symbol_table_cls = TargetTable
address_mapper = AddressMapper(symbol_table_cls, JsonParser, '*.BUILD.json')
project_tree = 'Let us pretend that this is a ProjectTree!'
tasks = create_graph_tasks(address_mapper, symbol_table_cls) + create_fs_tasks(project_tree)
rule_index = RuleIndex.create(tasks, tuple())
root_subject_types = {Address,
PathGlobs,
SingleAddress,
SiblingAddresses,
DescendantAddresses,
AscendantAddresses}
fullgraph_str = self.create_full_graph(root_subject_types, rule_index)
print('---diagnostic------')
print(fullgraph_str)
print('/---diagnostic------')
in_root_rules = False
in_all_rules = False
all_rules = []
root_rule_lines = []
for line in fullgraph_str.splitlines():
if line.startswith(' // root subject types:'):
pass
elif line.startswith(' // root entries'):
in_root_rules = True
elif line.startswith(' // internal entries'):
in_all_rules = True
elif in_all_rules:
all_rules.append(line)
elif in_root_rules:
root_rule_lines.append(line)
else:
pass
self.assertEquals(31, len(all_rules))
self.assertEquals(56, len(root_rule_lines)) # 2 lines per entry
def setup_json_scheduler(build_root, native):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype :class:`pants.engine.scheduler.LocalScheduler`
"""
symbol_table_cls = ExampleTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_patterns=('BLD.json',),
parser_cls=JsonParser)
source_roots = SourceRoots(('src/java','src/scala'))
scrooge_tool_address = Address.parse('src/scala/scrooge')
goals = {
'compile': Classpath,
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': Classpath,
'list': Address,
GenGoal.name(): GenGoal,
'ls': Files,
'cat': FilesContent,
}
tasks = [
# Codegen
GenGoal.signature(),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftJavaConfiguration, 'thrift')],
gen_apache_thrift),
(PythonSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftPythonConfiguration, 'thrift')],
gen_apache_thrift),
(ScalaSources,
[Select(ThriftSources),
SelectVariant(ScroogeScalaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ScroogeJavaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
] + [
# scala dependency inference
(ScalaSources,
[Select(ScalaInferredDepsSources),
SelectDependencies(Address, ImportedJVMPackages, field_types=(JVMPackageName,))],
reify_scala_sources),
(ImportedJVMPackages,
[SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)],
extract_scala_imports),
(Address,
[Select(JVMPackageName),
SelectDependencies(AddressFamily, Dirs, field='stats', field_types=(Dir,))],
select_package_address),
(PathGlobs,
[Select(JVMPackageName),
SelectLiteral(source_roots, SourceRoots)],
calculate_package_search_path),
] + [
# Remote dependency resolution
(Classpath,
[Select(Jar)],
ivy_resolve),
(Jar,
[Select(ManagedJar),
SelectVariant(ManagedResolve, 'resolve')],
select_rev),
] + [
# Compilers
(Classpath,
[Select(ResourceSources)],
isolate_resources),
(Classpath,
[Select(BuildPropertiesConfiguration)],
write_name_file),
# NB: Not sure these SelectDependencies should allow Jar, but they currently produce jars.
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources, field_types=(Address, Jar))],
javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources, field_types=(Address, Jar))],
scalac),
] + (
create_graph_tasks(address_mapper, symbol_table_cls)
) + (
create_fs_tasks()
)
project_tree = FileSystemProjectTree(build_root)
return LocalScheduler(goals,
tasks,
project_tree,
native,
graph_lock=None)
def setup_json_scheduler(build_root, native):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype :class:`pants.engine.scheduler.LocalScheduler`
"""
symbol_table_cls = ExampleTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern='BLD.json',
parser_cls=JsonParser)
source_roots = SourceRoots(('src/java','src/scala'))
scrooge_tool_address = Address.parse('src/scala/scrooge')
goals = {
'compile': Classpath,
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': Classpath,
'list': Address,
GenGoal.name(): GenGoal,
'ls': Files,
'cat': FilesContent,
}
tasks = [
# Codegen
GenGoal.signature(),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftJavaConfiguration, 'thrift')],
gen_apache_thrift),
(PythonSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftPythonConfiguration, 'thrift')],
gen_apache_thrift),
(ScalaSources,
[Select(ThriftSources),
SelectVariant(ScroogeScalaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ScroogeJavaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
] + [
# scala dependency inference
(ScalaSources,
[Select(ScalaInferredDepsSources),
SelectDependencies(Address, ImportedJVMPackages, field_types=(JVMPackageName,))],
reify_scala_sources),
(ImportedJVMPackages,
[SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)],
extract_scala_imports),
(Address,
[Select(JVMPackageName),
SelectDependencies(AddressFamily, Dirs, field='stats', field_types=(Dir,))],
select_package_address),
(PathGlobs,
[Select(JVMPackageName),
SelectLiteral(source_roots, SourceRoots)],
calculate_package_search_path),
] + [
# Remote dependency resolution
(Classpath,
[Select(Jar)],
ivy_resolve),
(Jar,
[Select(ManagedJar),
SelectVariant(ManagedResolve, 'resolve')],
select_rev),
] + [
# Compilers
(Classpath,
[Select(ResourceSources)],
isolate_resources),
(Classpath,
[Select(BuildPropertiesConfiguration)],
write_name_file),
# NB: Not sure these SelectDependencies should allow Jar, but they currently produce jars.
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources, field_types=(Address, Jar))],
javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources, field_types=(Address, Jar))],
scalac),
] + (
create_graph_tasks(address_mapper, symbol_table_cls)
) + (
create_fs_tasks()
)
project_tree = FileSystemProjectTree(build_root)
return LocalScheduler(goals,
tasks,
project_tree,
native,
graph_lock=None)
