def create(self, build_pattern=None, parser_cls=None, inline=False): mapper = AddressMapper(build_root=os.path.dirname(__file__), symbol_table_cls=TestTable, build_pattern=build_pattern, parser_cls=parser_cls) return LocalScheduler({self._goal: [self._product]}, create_graph_tasks(mapper))
def setup(options=None): if not options: options, _ = OptionsInitializer(OptionsBootstrapper()).setup() build_root = get_buildroot() cmd_line_spec_parser = CmdLineSpecParser(build_root) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs] storage = Storage.create(debug=False) # Ignore any dotfile below build_root except . itself project_tree = FileSystemProjectTree(build_root, ['.*', 'build-support/*.venv/']) symbol_table_cls = LegacyTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph # will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks() + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls) ) return ( LocalScheduler(dict(), tasks, storage, project_tree), storage, options, spec_roots, symbol_table_cls )
def list(): """Lists all addresses under the current build root.""" build_root = get_buildroot() symbol_table_cls = LegacyTable address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser) # Create a Scheduler containing only the graph tasks, with a single installed goal that # requests an Address. goal = 'list' tasks = create_fs_tasks(build_root) + create_graph_tasks(address_mapper, symbol_table_cls) scheduler = LocalScheduler({goal: Address}, symbol_table_cls, tasks) # Execute a request for the root. build_request = BuildRequest(goals=[goal], subjects=[DescendantAddresses('')]) result = LocalSerialEngine(scheduler).execute(build_request) if result.error: raise result.error # Render the output. for state in result.root_products.values(): if type(state) is Throw: raise state.exc elif type(state) is not Return: State.raise_unrecognized(dep_state) for address in state.value: print(address)
def setup(options=None): if not options: options, _ = OptionsInitializer(OptionsBootstrapper()).setup() build_root = get_buildroot() cmd_line_spec_parser = CmdLineSpecParser(build_root) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs] storage = Storage.create(debug=False) project_tree = FileSystemProjectTree(build_root) symbol_table_cls = LegacyTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph # will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks() + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls) ) return ( LocalScheduler(dict(), tasks, storage, project_tree), storage, options, spec_roots, symbol_table_cls )
def setUp(self): self.work_dir = safe_mkdtemp() self.addCleanup(safe_rmtree, self.work_dir) self.build_root = os.path.join(self.work_dir, 'build_root') shutil.copytree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'), self.build_root) subjects = Subjects() self._goal = 'list' symbol_table_cls = TargetTable project_tree_key = subjects.put( FileSystemProjectTree(self.build_root)) address_mapper_key = subjects.put( AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=JsonParser, build_pattern=r'.+\.BUILD.json$')) tasks = ( create_fs_tasks(project_tree_key) + create_graph_tasks(address_mapper_key, symbol_table_cls) ) self.scheduler = LocalScheduler({self._goal: UnhydratedStruct}, tasks, subjects, symbol_table_cls) self.a_b = Address.parse('a/b') self.a_b_target = Target(name='b', dependencies=['//d:e'], configurations=['//a', Struct(embedded='yes')], type_alias='target')
def create(self, build_pattern=None, parser_cls=None, inline=False): symbol_table_cls = TestTable mapper = AddressMapper(symbol_table_cls=symbol_table_cls, build_pattern=build_pattern, parser_cls=parser_cls) tasks = (create_fs_tasks(self._build_root) + create_graph_tasks(mapper, symbol_table_cls)) return LocalScheduler({self._goal: self._product}, symbol_table_cls, tasks)
def create(self, build_pattern=None, parser_cls=None, inline=False): symbol_table_cls = TestTable address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, build_pattern=build_pattern, parser_cls=parser_cls) tasks = create_graph_tasks(address_mapper, symbol_table_cls) project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples')) scheduler, _ = self.mk_scheduler(tasks=tasks, storage=self.storage, project_tree=project_tree, symbol_table_cls=symbol_table_cls) return scheduler
def create(self, build_pattern=None, parser_cls=None, inline=False): symbol_table_cls = TestTable address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, build_pattern=build_pattern, parser_cls=parser_cls) tasks = create_graph_tasks(address_mapper, symbol_table_cls) project_tree = self.mk_fs_tree( os.path.join(os.path.dirname(__file__), 'examples')) scheduler, _ = self.mk_scheduler(tasks=tasks, storage=self.storage, project_tree=project_tree, symbol_table_cls=symbol_table_cls) return scheduler
def create(self, build_pattern=None, parser_cls=None, inline=False): subjects = Subjects() symbol_table_cls = TestTable project_tree_key = subjects.put( FileSystemProjectTree(self._build_root)) address_mapper_key = subjects.put( AddressMapper(symbol_table_cls=symbol_table_cls, build_pattern=build_pattern, parser_cls=parser_cls)) tasks = ( create_fs_tasks(project_tree_key) + create_graph_tasks(address_mapper_key, symbol_table_cls) ) return LocalScheduler(dict(), tasks, subjects, symbol_table_cls)
def setUp(self): # Set up a scheduler that supports address mapping. symbol_table_cls = TargetTable self.storage = Storage.create(in_memory=True) address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=JsonParser, build_pattern=r'.+\.BUILD.json$') tasks = create_graph_tasks(address_mapper, symbol_table_cls) build_root_src = os.path.join(os.path.dirname(__file__), 'examples/mapper_test') self.scheduler, _, self.build_root = self.mk_scheduler(tasks=tasks, build_root_src=build_root_src, storage=self.storage, symbol_table_cls=symbol_table_cls) self.a_b = Address.parse('a/b') self.a_b_target = Target(name='b', dependencies=['//d:e'], configurations=['//a', Struct(embedded='yes')], type_alias='target')
def setUp(self): self.work_dir = safe_mkdtemp() self.addCleanup(safe_rmtree, self.work_dir) self.build_root = os.path.join(self.work_dir, 'build_root') shutil.copytree( os.path.join(os.path.dirname(__file__), 'examples/mapper_test'), self.build_root) self._goal = 'list' symbol_table_cls = TargetTable self.address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=JsonParser, build_pattern=r'.+\.BUILD.json$') tasks = (create_fs_tasks(self.build_root) + create_graph_tasks(self.address_mapper, symbol_table_cls)) self.scheduler = LocalScheduler({self._goal: UnhydratedStruct}, symbol_table_cls, tasks) self.a_b = Address.parse('a/b') self.a_b_target = Target( name='b', dependencies=['//d:e'], configurations=['//a', Struct(embedded='yes')])
def setup_json_scheduler(build_root): """Return a build graph and scheduler configured for BLD.json files under the given build root. :rtype :class:`pants.engine.exp.scheduler.LocalScheduler` """ address_mapper = AddressMapper(build_root=build_root, symbol_table_cls=ExampleTable, build_pattern=r'^BLD.json$', parser_cls=JsonParser) # TODO(John Sirois): once the options system is plumbed, make the tool spec configurable. # It could also just be pointed at the scrooge jar at that point. scrooge_tool_address = Address.parse('src/scala/scrooge') # TODO: Placeholder for the SourceRoot subsystem. source_roots = SourceRoots(build_root, ('src/java',)) products_by_goal = { 'compile': [Classpath], # TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product. 'resolve': [Classpath], 'gen': [JavaSources, PythonSources, ResourceSources, ScalaSources], 'unpickleable': [UnpickleableResult], } tasks = [ # Codegen (JavaSources, [Select(ThriftSources), SelectVariant(ApacheThriftJavaConfiguration, 'thrift')], gen_apache_thrift), (PythonSources, [Select(ThriftSources), SelectVariant(ApacheThriftPythonConfiguration, 'thrift')], gen_apache_thrift), (ScalaSources, [Select(ThriftSources), SelectVariant(ScroogeScalaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath)], gen_scrooge_thrift), (JavaSources, [Select(ThriftSources), SelectVariant(ScroogeJavaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath)], gen_scrooge_thrift), ] + [ # scala dependency inference (ScalaSources, [Select(Address), Select(ScalaInferredDepsSources), SelectDependencies(Address, ImportedJVMPackages)], reify_scala_sources), (ImportedJVMPackages, [Select(Address), Select(ScalaInferredDepsSources), SelectLiteral(source_roots, SourceRoots)], extract_scala_imports), # TODO: The request for an AddressFamily for each member of a SearchPath will fail whenever # a member of the path doesn't exist. Need to allow for optional products and to then # request the AddressFamilies optionally here. (Address, [Select(JVMPackageName), SelectDependencies(AddressFamily, SearchPath)], select_package_address), (SearchPath, [Select(JVMPackageName), SelectLiteral(source_roots, SourceRoots)], calculate_package_search_path), ] + [ # Remote dependency resolution (Classpath, [Select(Jar)], ivy_resolve), (Jar, [Select(ManagedJar), SelectVariant(ManagedResolve, 'resolve')], select_rev), ] + [ # Compilers (Classpath, [Select(ResourceSources)], isolate_resources), (Classpath, [Select(BuildPropertiesConfiguration)], write_name_file), (Classpath, [Select(JavaSources), SelectDependencies(Classpath, JavaSources)], javac), (Classpath, [Select(ScalaSources), SelectDependencies(Classpath, ScalaSources)], scalac), ] + [ # TODO (UnpickleableOutput, [], unpickleable_output), (UnpickleableResult, [Select(UnpickleableOutput)], unpickleable_input), ] + ( create_graph_tasks(address_mapper) ) scheduler = LocalScheduler(products_by_goal, tasks) return scheduler
def setup_json_scheduler(build_root, debug=True): """Return a build graph and scheduler configured for BLD.json files under the given build root. :rtype A tuple of :class:`pants.engine.exp.scheduler.LocalScheduler`, :class:`pants.engine.exp.storage.Storage`. """ storage = Storage.create(debug=debug) symbol_table_cls = ExampleTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, build_pattern=r'^BLD.json$', parser_cls=JsonParser) source_roots = SourceRoots(('src/java','src/scala')) scrooge_tool_address = Address.parse('src/scala/scrooge') goals = { 'compile': Classpath, # TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product. 'resolve': Classpath, 'list': Address, GenGoal.name(): GenGoal, 'unpickleable': UnpickleableResult, 'ls': File, 'cat': FileContent, } tasks = [ # Codegen GenGoal.signature(), (JavaSources, [Select(ThriftSources), SelectVariant(ApacheThriftJavaConfiguration, 'thrift')], gen_apache_thrift), (PythonSources, [Select(ThriftSources), SelectVariant(ApacheThriftPythonConfiguration, 'thrift')], gen_apache_thrift), (ScalaSources, [Select(ThriftSources), SelectVariant(ScroogeScalaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath)], gen_scrooge_thrift), (JavaSources, [Select(ThriftSources), SelectVariant(ScroogeJavaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath)], gen_scrooge_thrift), ] + [ # scala dependency inference (ScalaSources, [Select(ScalaInferredDepsSources), SelectDependencies(Address, ImportedJVMPackages)], reify_scala_sources), (ImportedJVMPackages, [SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)], extract_scala_imports), (Address, [Select(JVMPackageName), SelectDependencies(AddressFamily, Dirs)], select_package_address), (PathGlobs, [Select(JVMPackageName), SelectLiteral(source_roots, SourceRoots)], calculate_package_search_path), ] + [ # Remote dependency resolution (Classpath, [Select(Jar)], ivy_resolve), (Jar, [Select(ManagedJar), SelectVariant(ManagedResolve, 'resolve')], select_rev), ] + [ # Compilers (Classpath, [Select(ResourceSources)], isolate_resources), (Classpath, [Select(BuildPropertiesConfiguration)], write_name_file), (Classpath, [Select(JavaSources), SelectDependencies(Classpath, JavaSources)], javac), (Classpath, [Select(ScalaSources), SelectDependencies(Classpath, ScalaSources)], scalac), ] + [ # TODO (UnpickleableOutput, [], unpickleable_output), (UnpickleableResult, [Select(UnpickleableOutput)], unpickleable_input), ] + ( create_graph_tasks(address_mapper, symbol_table_cls) ) + ( create_fs_tasks() ) project_tree = FileSystemProjectTree(build_root) return LocalScheduler(goals, tasks, storage, project_tree, None, GraphValidator(symbol_table_cls)), storage
def setup_json_scheduler(build_root): """Return a build graph and scheduler configured for BLD.json files under the given build root. :rtype :class:`pants.engine.exp.scheduler.LocalScheduler` """ address_mapper = AddressMapper(build_root=build_root, symbol_table_cls=ExampleTable, build_pattern=r'^BLD.json$', parser_cls=JsonParser) # TODO(John Sirois): once the options system is plumbed, make the tool spec configurable. # It could also just be pointed at the scrooge jar at that point. scrooge_tool_address = Address.parse('src/scala/scrooge') # TODO: Placeholder for the SourceRoot subsystem. source_roots = SourceRoots(build_root, ('src/java', )) products_by_goal = { 'compile': [Classpath], # TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product. 'resolve': [Classpath], 'gen': [JavaSources, PythonSources, ResourceSources, ScalaSources], 'unpickleable': [UnpickleableResult], } tasks = [ # Codegen (JavaSources, [ Select(ThriftSources), SelectVariant(ApacheThriftJavaConfiguration, 'thrift') ], gen_apache_thrift), (PythonSources, [ Select(ThriftSources), SelectVariant(ApacheThriftPythonConfiguration, 'thrift') ], gen_apache_thrift), (ScalaSources, [ Select(ThriftSources), SelectVariant(ScroogeScalaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath) ], gen_scrooge_thrift), (JavaSources, [ Select(ThriftSources), SelectVariant(ScroogeJavaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath) ], gen_scrooge_thrift), ] + [ # scala dependency inference (ScalaSources, [ Select(Address), Select(ScalaInferredDepsSources), SelectDependencies(Address, ImportedJVMPackages) ], reify_scala_sources), (ImportedJVMPackages, [ Select(Address), Select(ScalaInferredDepsSources), SelectLiteral(source_roots, SourceRoots) ], extract_scala_imports), # TODO: The request for an AddressFamily for each member of a SearchPath will fail whenever # a member of the path doesn't exist. Need to allow for optional products and to then # request the AddressFamilies optionally here. (Address, [ Select(JVMPackageName), SelectDependencies(AddressFamily, SearchPath) ], select_package_address), (SearchPath, [ Select(JVMPackageName), SelectLiteral(source_roots, SourceRoots) ], calculate_package_search_path), ] + [ # Remote dependency resolution (Classpath, [Select(Jar)], ivy_resolve), (Jar, [Select(ManagedJar), SelectVariant(ManagedResolve, 'resolve')], select_rev), ] + [ # Compilers (Classpath, [Select(ResourceSources)], isolate_resources), (Classpath, [Select(BuildPropertiesConfiguration)], write_name_file), (Classpath, [Select(JavaSources), SelectDependencies(Classpath, JavaSources)], javac), (Classpath, [Select(ScalaSources), SelectDependencies(Classpath, ScalaSources)], scalac), ] + [ # TODO (UnpickleableOutput, [], unpickleable_output), (UnpickleableResult, [Select(UnpickleableOutput)], unpickleable_input), ] + (create_graph_tasks(address_mapper)) scheduler = LocalScheduler(products_by_goal, tasks) return scheduler
def setup_json_scheduler(build_root): """Return a build graph and scheduler configured for BLD.json files under the given build root. :rtype :class:`pants.engine.exp.scheduler.LocalScheduler` """ symbol_table_cls = ExampleTable address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, build_pattern=r'^BLD.json$', parser_cls=JsonParser) # TODO(John Sirois): once the options system is plumbed, make the tool spec configurable. # It could also just be pointed at the scrooge jar at that point. scrooge_tool_address = Address.parse('src/scala/scrooge') # TODO: Placeholder for the SourceRoot subsystem. source_roots = SourceRoots(('src/java',)) goals = { 'compile': Classpath, # TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product. 'resolve': Classpath, 'list': Address, 'walk': Path, GenGoal.name(): GenGoal, 'unpickleable': UnpickleableResult, } tasks = [ # Codegen GenGoal.signature(), (JavaSources, [Select(ThriftSources), SelectVariant(ApacheThriftJavaConfiguration, 'thrift')], gen_apache_thrift), (PythonSources, [Select(ThriftSources), SelectVariant(ApacheThriftPythonConfiguration, 'thrift')], gen_apache_thrift), (ScalaSources, [Select(ThriftSources), SelectVariant(ScroogeScalaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath)], gen_scrooge_thrift), (JavaSources, [Select(ThriftSources), SelectVariant(ScroogeJavaConfiguration, 'thrift'), SelectLiteral(scrooge_tool_address, Classpath)], gen_scrooge_thrift), ] + [ # scala dependency inference (ScalaSources, [Select(ScalaInferredDepsSources), SelectDependencies(Address, ImportedJVMPackages)], reify_scala_sources), (ImportedJVMPackages, [SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)], extract_scala_imports), (Address, [Select(JVMPackageName), SelectDependencies(AddressFamily, Paths)], select_package_address), (Paths, [Select(JVMPackageName), SelectLiteral(source_roots, SourceRoots)], calculate_package_search_path), ] + [ # Remote dependency resolution (Classpath, [Select(Jar)], ivy_resolve), (Jar, [Select(ManagedJar), SelectVariant(ManagedResolve, 'resolve')], select_rev), ] + [ # Compilers (Classpath, [Select(ResourceSources)], isolate_resources), (Classpath, [Select(BuildPropertiesConfiguration)], write_name_file), (Classpath, [Select(JavaSources), SelectDependencies(Classpath, JavaSources)], javac), (Classpath, [Select(ScalaSources), SelectDependencies(Classpath, ScalaSources)], scalac), ] + [ # TODO (UnpickleableOutput, [], unpickleable_output), (UnpickleableResult, [Select(UnpickleableOutput)], unpickleable_input), ] + ( create_graph_tasks(address_mapper, symbol_table_cls) ) + ( create_fs_tasks(build_root) ) scheduler = LocalScheduler(goals, symbol_table_cls, tasks) return scheduler