def test_full_graph_for_planner_example(self): symbol_table_cls = TargetTable address_mapper = AddressMapper(symbol_table_cls, JsonParser, '*.BUILD.json') tasks = create_graph_tasks(address_mapper, symbol_table_cls) + create_fs_tasks() rule_index = NodeBuilder.create(tasks) graphmaker = GraphMaker(rule_index, root_subject_fns={k: lambda p: Select(p) for k in (Address, # TODO, use the actual fns. PathGlobs, SingleAddress, SiblingAddresses, DescendantAddresses, AscendantAddresses )}) fullgraph = graphmaker.full_graph() print('---diagnostic------') print(fullgraph.error_message()) print('/---diagnostic------') print(fullgraph) # Assert that all of the rules specified the various task fns are present declared_rules = rule_index.all_rules() rules_remaining_in_graph_strs = set(str(r.rule) for r in fullgraph.rule_dependencies.keys()) declared_rule_strings = set(str(r) for r in declared_rules) self.assertEquals(declared_rule_strings, rules_remaining_in_graph_strs ) # statically assert that the number of dependency keys is fixed self.assertEquals(43, len(fullgraph.rule_dependencies))
def setup_legacy_graph(path_ignore_patterns): """Construct and return the components necessary for LegacyBuildGraph construction. :param list path_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the `--pants-ignore` global option. :returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls). """ build_root = get_buildroot() project_tree = FileSystemProjectTree(build_root, path_ignore_patterns) symbol_table_cls = LegacySymbolTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks() + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls) ) scheduler = LocalScheduler(dict(), tasks, project_tree) engine = LocalSerialEngine(scheduler, Storage.create(debug=False)) return LegacyGraphHelper(scheduler, engine, symbol_table_cls, LegacyBuildGraph)
def create(self, build_pattern=None, parser_cls=None): symbol_table_cls = TestTable address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, build_pattern=build_pattern, parser_cls=parser_cls) tasks = create_graph_tasks(address_mapper, symbol_table_cls) project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples')) scheduler = self.mk_scheduler(tasks=tasks, project_tree=project_tree) return scheduler
def setup_legacy_graph(pants_ignore_patterns, build_root=None, symbol_table_cls=None, build_ignore_patterns=None, exclude_target_regexps=None): """Construct and return the components necessary for LegacyBuildGraph construction. :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the '--pants-ignore' global option. :param str build_root: A path to be used as the build root. If None, then default is used. :param SymbolTable symbol_table_cls: A SymbolTable class to use for build file parsing, or None to use the default. :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls). """ build_root = build_root or get_buildroot() scm = get_scm() symbol_table_cls = symbol_table_cls or LegacySymbolTable project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns) # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper( symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. tasks = (create_legacy_graph_tasks(symbol_table_cls) + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls)) scheduler = LocalScheduler(dict(), tasks, project_tree) # TODO: Do not use the cache yet, as it incurs a high overhead. engine = LocalSerialEngine(scheduler, Storage.create(), use_cache=False) change_calculator = EngineChangeCalculator(engine, scm) if scm else None return LegacyGraphHelper(scheduler, engine, symbol_table_cls, change_calculator)
def setUp(self): # Set up a scheduler that supports address mapping. symbol_table_cls = TargetTable address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=JsonParser, build_pattern='*.BUILD.json') tasks = create_graph_tasks(address_mapper, symbol_table_cls) project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test')) self.build_root = project_tree.build_root self.scheduler = self.mk_scheduler(tasks=tasks, project_tree=project_tree) self.a_b = Address.parse('a/b') self.a_b_target = Target(name='b', dependencies=['//d:e'], configurations=['//a', Struct(embedded='yes')], type_alias='target')
def setUp(self): # Set up a scheduler that supports address mapping. symbol_table_cls = TargetTable address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=JsonParser, build_pattern=r'.+\.BUILD.json$') tasks = create_graph_tasks(address_mapper, symbol_table_cls) project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test')) self.build_root = project_tree.build_root self.scheduler = self.mk_scheduler(tasks=tasks, project_tree=project_tree) self.a_b = Address.parse('a/b') self.a_b_target = Target(name='b', dependencies=['//d:e'], configurations=['//a', Struct(embedded='yes')], type_alias='target')
def setup_legacy_graph(pants_ignore_patterns, build_root=None, symbol_table_cls=None, build_ignore_patterns=None, exclude_target_regexps=None): """Construct and return the components necessary for LegacyBuildGraph construction. :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the '--pants-ignore' global option. :param str build_root: A path to be used as the build root. If None, then default is used. :param SymbolTable symbol_table_cls: A SymbolTable class to use for build file parsing, or None to use the default. :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls). """ build_root = build_root or get_buildroot() scm = get_scm() symbol_table_cls = symbol_table_cls or LegacySymbolTable project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns) # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks(symbol_table_cls) + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls) ) scheduler = LocalScheduler(dict(), tasks, project_tree) # TODO: Do not use the cache yet, as it incurs a high overhead. engine = LocalSerialEngine(scheduler, Storage.create(), use_cache=False) change_calculator = EngineChangeCalculator(engine, scm) if scm else None return LegacyGraphHelper(scheduler, engine, symbol_table_cls, change_calculator)
def setup_json_scheduler(build_root, inline_nodes=True): """Return a build graph and scheduler configured for BLD.json files under the given build root. :rtype :class:`pants.engine.scheduler.LocalScheduler` """ symbol_table_cls = ExampleTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, build_pattern='BLD.json', parser_cls=JsonParser) source_roots = SourceRoots(('src/java','src/scala')) scrooge_tool_address = Address.parse('src/scala/scrooge') goals = { 'compile': Classpath, # TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product. 'resolve': Classpath, 'list': Address, GenGoal.name(): GenGoal, 'unpickleable': UnpickleableResult, 'ls': Files, 'cat': FilesContent, } tasks = [ # Codegen GenGoal.signature(), (JavaSources, [Select(ThriftSources), SelectVariant(ApacheThriftJavaConfiguration, 'thrift')], gen_apache_thrift), (PythonSources, [Select(ThriftSources), SelectVariant(ApacheThriftPythonConfiguration, 'thrift')], gen_apache_thrift), (ScalaSources, [Select(ThriftSources), SelectVariant(ScroogeScalaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath)], gen_scrooge_thrift), (JavaSources, [Select(ThriftSources), SelectVariant(ScroogeJavaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath)], gen_scrooge_thrift), ] + [ # scala dependency inference (ScalaSources, [Select(ScalaInferredDepsSources), SelectDependencies(Address, ImportedJVMPackages)], reify_scala_sources), (ImportedJVMPackages, [SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)], extract_scala_imports), (Address, [Select(JVMPackageName), SelectDependencies(AddressFamily, Dirs, field='stats')], select_package_address), (PathGlobs, [Select(JVMPackageName), SelectLiteral(source_roots, SourceRoots)], calculate_package_search_path), ] + [ # Remote dependency resolution (Classpath, [Select(Jar)], ivy_resolve), (Jar, [Select(ManagedJar), SelectVariant(ManagedResolve, 'resolve')], select_rev), ] + [ # Compilers (Classpath, [Select(ResourceSources)], isolate_resources), (Classpath, [Select(BuildPropertiesConfiguration)], write_name_file), (Classpath, [Select(JavaSources), SelectDependencies(Classpath, JavaSources)], javac), (Classpath, [Select(ScalaSources), SelectDependencies(Classpath, ScalaSources)], scalac), ] + [ # TODO (UnpickleableOutput, [], unpickleable_output), (UnpickleableResult, [Select(UnpickleableOutput)], unpickleable_input), ] + ( create_graph_tasks(address_mapper, symbol_table_cls) ) + ( create_fs_tasks() ) project_tree = FileSystemProjectTree(build_root) return LocalScheduler(goals, tasks, project_tree, graph_lock=None, inline_nodes=inline_nodes, graph_validator=GraphValidator(symbol_table_cls))
def setup_json_scheduler(build_root, debug=True): """Return a build graph and scheduler configured for BLD.json files under the given build root. :rtype A tuple of :class:`pants.engine.scheduler.LocalScheduler`, :class:`pants.engine.storage.Storage`. """ storage = Storage.create(debug=debug) symbol_table_cls = ExampleTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, build_pattern=r'^BLD.json$', parser_cls=JsonParser) source_roots = SourceRoots(('src/java', 'src/scala')) scrooge_tool_address = Address.parse('src/scala/scrooge') goals = { 'compile': Classpath, # TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product. 'resolve': Classpath, 'list': Address, GenGoal.name(): GenGoal, 'unpickleable': UnpickleableResult, 'ls': File, 'cat': FileContent, } tasks = [ # Codegen GenGoal.signature(), (JavaSources, [ Select(ThriftSources), SelectVariant(ApacheThriftJavaConfiguration, 'thrift') ], gen_apache_thrift), (PythonSources, [ Select(ThriftSources), SelectVariant(ApacheThriftPythonConfiguration, 'thrift') ], gen_apache_thrift), (ScalaSources, [ Select(ThriftSources), SelectVariant(ScroogeScalaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath) ], gen_scrooge_thrift), (JavaSources, [ Select(ThriftSources), SelectVariant(ScroogeJavaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath) ], gen_scrooge_thrift), ] + [ # scala dependency inference (ScalaSources, [ Select(ScalaInferredDepsSources), SelectDependencies(Address, ImportedJVMPackages) ], reify_scala_sources), (ImportedJVMPackages, [ SelectProjection(FilesContent, PathGlobs, ('path_globs', ), ScalaInferredDepsSources) ], extract_scala_imports), (Address, [Select(JVMPackageName), SelectDependencies(AddressFamily, Dirs)], select_package_address), (PathGlobs, [ Select(JVMPackageName), SelectLiteral(source_roots, SourceRoots) ], calculate_package_search_path), ] + [ # Remote dependency resolution (Classpath, [Select(Jar)], ivy_resolve), (Jar, [Select(ManagedJar), SelectVariant(ManagedResolve, 'resolve')], select_rev), ] + [ # Compilers (Classpath, [Select(ResourceSources)], isolate_resources), (Classpath, [Select(BuildPropertiesConfiguration)], write_name_file), (Classpath, [Select(JavaSources), SelectDependencies(Classpath, JavaSources)], javac), (Classpath, [Select(ScalaSources), SelectDependencies(Classpath, ScalaSources)], scalac), ] + [ # TODO (UnpickleableOutput, [], unpickleable_output), (UnpickleableResult, [Select(UnpickleableOutput)], unpickleable_input), ] + (create_graph_tasks(address_mapper, symbol_table_cls)) + (create_fs_tasks()) project_tree = FileSystemProjectTree(build_root) return LocalScheduler(goals, tasks, storage, project_tree, None, GraphValidator(symbol_table_cls)), storage