def create(self, build_pattern=None, parser_cls=None, inline=False):
mapper = AddressMapper(build_root=os.path.dirname(__file__),
symbol_table_cls=TestTable,
build_pattern=build_pattern,
parser_cls=parser_cls)
return LocalScheduler({self._goal: [self._product]},
create_graph_tasks(mapper))
def setup(options=None):
if not options:
options, _ = OptionsInitializer(OptionsBootstrapper()).setup()
build_root = get_buildroot()
cmd_line_spec_parser = CmdLineSpecParser(build_root)
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs]
storage = Storage.create(debug=False)
# Ignore any dotfile below build_root except . itself
project_tree = FileSystemProjectTree(build_root, ['.*', 'build-support/*.venv/'])
symbol_table_cls = LegacyTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser)
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph
# will explicitly request the products it needs.
tasks = (
create_legacy_graph_tasks() +
create_fs_tasks() +
create_graph_tasks(address_mapper, symbol_table_cls)
)
return (
LocalScheduler(dict(), tasks, storage, project_tree),
storage,
options,
spec_roots,
symbol_table_cls
)
def list():
"""Lists all addresses under the current build root."""
build_root = get_buildroot()
symbol_table_cls = LegacyTable
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser)
# Create a Scheduler containing only the graph tasks, with a single installed goal that
# requests an Address.
goal = 'list'
tasks = create_fs_tasks(build_root) + create_graph_tasks(address_mapper, symbol_table_cls)
scheduler = LocalScheduler({goal: Address}, symbol_table_cls, tasks)
# Execute a request for the root.
build_request = BuildRequest(goals=[goal], subjects=[DescendantAddresses('')])
result = LocalSerialEngine(scheduler).execute(build_request)
if result.error:
raise result.error
# Render the output.
for state in result.root_products.values():
if type(state) is Throw:
raise state.exc
elif type(state) is not Return:
State.raise_unrecognized(dep_state)
for address in state.value:
print(address)
def setup(options=None):
if not options:
options, _ = OptionsInitializer(OptionsBootstrapper()).setup()
build_root = get_buildroot()
cmd_line_spec_parser = CmdLineSpecParser(build_root)
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs]
storage = Storage.create(debug=False)
project_tree = FileSystemProjectTree(build_root)
symbol_table_cls = LegacyTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser)
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph
# will explicitly request the products it needs.
tasks = (
create_legacy_graph_tasks() +
create_fs_tasks() +
create_graph_tasks(address_mapper, symbol_table_cls)
)
return (
LocalScheduler(dict(), tasks, storage, project_tree),
storage,
options,
spec_roots,
symbol_table_cls
)
def setUp(self):
self.work_dir = safe_mkdtemp()
self.addCleanup(safe_rmtree, self.work_dir)
self.build_root = os.path.join(self.work_dir, 'build_root')
shutil.copytree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'),
self.build_root)
subjects = Subjects()
self._goal = 'list'
symbol_table_cls = TargetTable
project_tree_key = subjects.put(
FileSystemProjectTree(self.build_root))
address_mapper_key = subjects.put(
AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=JsonParser,
build_pattern=r'.+\.BUILD.json$'))
tasks = (
create_fs_tasks(project_tree_key) +
create_graph_tasks(address_mapper_key, symbol_table_cls)
)
self.scheduler = LocalScheduler({self._goal: UnhydratedStruct},
tasks,
subjects,
symbol_table_cls)
self.a_b = Address.parse('a/b')
self.a_b_target = Target(name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')],
type_alias='target')
def create(self, build_pattern=None, parser_cls=None, inline=False):
symbol_table_cls = TestTable
mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=build_pattern,
parser_cls=parser_cls)
tasks = (create_fs_tasks(self._build_root) +
create_graph_tasks(mapper, symbol_table_cls))
return LocalScheduler({self._goal: self._product}, symbol_table_cls,
tasks)
def create(self, build_pattern=None, parser_cls=None, inline=False):
symbol_table_cls = TestTable
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=build_pattern,
parser_cls=parser_cls)
tasks = create_graph_tasks(address_mapper, symbol_table_cls)
project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples'))
scheduler, _ = self.mk_scheduler(tasks=tasks,
storage=self.storage,
project_tree=project_tree,
symbol_table_cls=symbol_table_cls)
return scheduler
def create(self, build_pattern=None, parser_cls=None, inline=False):
symbol_table_cls = TestTable
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=build_pattern,
parser_cls=parser_cls)
tasks = create_graph_tasks(address_mapper, symbol_table_cls)
project_tree = self.mk_fs_tree(
os.path.join(os.path.dirname(__file__), 'examples'))
scheduler, _ = self.mk_scheduler(tasks=tasks,
storage=self.storage,
project_tree=project_tree,
symbol_table_cls=symbol_table_cls)
return scheduler
def create(self, build_pattern=None, parser_cls=None, inline=False):
subjects = Subjects()
symbol_table_cls = TestTable
project_tree_key = subjects.put(
FileSystemProjectTree(self._build_root))
address_mapper_key = subjects.put(
AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=build_pattern,
parser_cls=parser_cls))
tasks = (
create_fs_tasks(project_tree_key) +
create_graph_tasks(address_mapper_key, symbol_table_cls)
)
return LocalScheduler(dict(),
tasks,
subjects,
symbol_table_cls)
def setUp(self):
# Set up a scheduler that supports address mapping.
symbol_table_cls = TargetTable
self.storage = Storage.create(in_memory=True)
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=JsonParser,
build_pattern=r'.+\.BUILD.json$')
tasks = create_graph_tasks(address_mapper, symbol_table_cls)
build_root_src = os.path.join(os.path.dirname(__file__), 'examples/mapper_test')
self.scheduler, _, self.build_root = self.mk_scheduler(tasks=tasks,
build_root_src=build_root_src,
storage=self.storage,
symbol_table_cls=symbol_table_cls)
self.a_b = Address.parse('a/b')
self.a_b_target = Target(name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')],
type_alias='target')
def setUp(self):
self.work_dir = safe_mkdtemp()
self.addCleanup(safe_rmtree, self.work_dir)
self.build_root = os.path.join(self.work_dir, 'build_root')
shutil.copytree(
os.path.join(os.path.dirname(__file__), 'examples/mapper_test'),
self.build_root)
self._goal = 'list'
symbol_table_cls = TargetTable
self.address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=JsonParser,
build_pattern=r'.+\.BUILD.json$')
tasks = (create_fs_tasks(self.build_root) +
create_graph_tasks(self.address_mapper, symbol_table_cls))
self.scheduler = LocalScheduler({self._goal: UnhydratedStruct},
symbol_table_cls, tasks)
self.a_b = Address.parse('a/b')
self.a_b_target = Target(
name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')])
def setup_json_scheduler(build_root):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype :class:`pants.engine.exp.scheduler.LocalScheduler`
"""
address_mapper = AddressMapper(build_root=build_root,
symbol_table_cls=ExampleTable,
build_pattern=r'^BLD.json$',
parser_cls=JsonParser)
# TODO(John Sirois): once the options system is plumbed, make the tool spec configurable.
# It could also just be pointed at the scrooge jar at that point.
scrooge_tool_address = Address.parse('src/scala/scrooge')
# TODO: Placeholder for the SourceRoot subsystem.
source_roots = SourceRoots(build_root, ('src/java',))
products_by_goal = {
'compile': [Classpath],
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': [Classpath],
'gen': [JavaSources, PythonSources, ResourceSources, ScalaSources],
'unpickleable': [UnpickleableResult],
}
tasks = [
# Codegen
(JavaSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftJavaConfiguration, 'thrift')],
gen_apache_thrift),
(PythonSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftPythonConfiguration, 'thrift')],
gen_apache_thrift),
(ScalaSources,
[Select(ThriftSources),
SelectVariant(ScroogeScalaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ScroogeJavaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
] + [
# scala dependency inference
(ScalaSources,
[Select(Address),
Select(ScalaInferredDepsSources),
SelectDependencies(Address, ImportedJVMPackages)],
reify_scala_sources),
(ImportedJVMPackages,
[Select(Address),
Select(ScalaInferredDepsSources),
SelectLiteral(source_roots, SourceRoots)],
extract_scala_imports),
# TODO: The request for an AddressFamily for each member of a SearchPath will fail whenever
# a member of the path doesn't exist. Need to allow for optional products and to then
# request the AddressFamilies optionally here.
(Address,
[Select(JVMPackageName),
SelectDependencies(AddressFamily, SearchPath)],
select_package_address),
(SearchPath,
[Select(JVMPackageName),
SelectLiteral(source_roots, SourceRoots)],
calculate_package_search_path),
] + [
# Remote dependency resolution
(Classpath,
[Select(Jar)],
ivy_resolve),
(Jar,
[Select(ManagedJar),
SelectVariant(ManagedResolve, 'resolve')],
select_rev),
] + [
# Compilers
(Classpath,
[Select(ResourceSources)],
isolate_resources),
(Classpath,
[Select(BuildPropertiesConfiguration)],
write_name_file),
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources)],
javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources)],
scalac),
] + [
# TODO
(UnpickleableOutput,
[],
unpickleable_output),
(UnpickleableResult,
[Select(UnpickleableOutput)],
unpickleable_input),
] + (
create_graph_tasks(address_mapper)
)
scheduler = LocalScheduler(products_by_goal, tasks)
return scheduler
def setup_json_scheduler(build_root, debug=True):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype A tuple of :class:`pants.engine.exp.scheduler.LocalScheduler`,
:class:`pants.engine.exp.storage.Storage`.
"""
storage = Storage.create(debug=debug)
symbol_table_cls = ExampleTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=r'^BLD.json$',
parser_cls=JsonParser)
source_roots = SourceRoots(('src/java','src/scala'))
scrooge_tool_address = Address.parse('src/scala/scrooge')
goals = {
'compile': Classpath,
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': Classpath,
'list': Address,
GenGoal.name(): GenGoal,
'unpickleable': UnpickleableResult,
'ls': File,
'cat': FileContent,
}
tasks = [
# Codegen
GenGoal.signature(),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftJavaConfiguration, 'thrift')],
gen_apache_thrift),
(PythonSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftPythonConfiguration, 'thrift')],
gen_apache_thrift),
(ScalaSources,
[Select(ThriftSources),
SelectVariant(ScroogeScalaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ScroogeJavaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
] + [
# scala dependency inference
(ScalaSources,
[Select(ScalaInferredDepsSources),
SelectDependencies(Address, ImportedJVMPackages)],
reify_scala_sources),
(ImportedJVMPackages,
[SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)],
extract_scala_imports),
(Address,
[Select(JVMPackageName),
SelectDependencies(AddressFamily, Dirs)],
select_package_address),
(PathGlobs,
[Select(JVMPackageName),
SelectLiteral(source_roots, SourceRoots)],
calculate_package_search_path),
] + [
# Remote dependency resolution
(Classpath,
[Select(Jar)],
ivy_resolve),
(Jar,
[Select(ManagedJar),
SelectVariant(ManagedResolve, 'resolve')],
select_rev),
] + [
# Compilers
(Classpath,
[Select(ResourceSources)],
isolate_resources),
(Classpath,
[Select(BuildPropertiesConfiguration)],
write_name_file),
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources)],
javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources)],
scalac),
] + [
# TODO
(UnpickleableOutput,
[],
unpickleable_output),
(UnpickleableResult,
[Select(UnpickleableOutput)],
unpickleable_input),
] + (
create_graph_tasks(address_mapper, symbol_table_cls)
) + (
create_fs_tasks()
)
project_tree = FileSystemProjectTree(build_root)
return LocalScheduler(goals, tasks, storage, project_tree, None, GraphValidator(symbol_table_cls)), storage
def create(self, build_pattern=None, parser_cls=None, inline=False):
mapper = AddressMapper(build_root=os.path.dirname(__file__),
symbol_table_cls=TestTable,
build_pattern=build_pattern,
parser_cls=parser_cls)
return LocalScheduler({self._goal: [self._product]}, create_graph_tasks(mapper))
def setup_json_scheduler(build_root):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype :class:`pants.engine.exp.scheduler.LocalScheduler`
"""
address_mapper = AddressMapper(build_root=build_root,
symbol_table_cls=ExampleTable,
build_pattern=r'^BLD.json$',
parser_cls=JsonParser)
# TODO(John Sirois): once the options system is plumbed, make the tool spec configurable.
# It could also just be pointed at the scrooge jar at that point.
scrooge_tool_address = Address.parse('src/scala/scrooge')
# TODO: Placeholder for the SourceRoot subsystem.
source_roots = SourceRoots(build_root, ('src/java', ))
products_by_goal = {
'compile': [Classpath],
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': [Classpath],
'gen': [JavaSources, PythonSources, ResourceSources, ScalaSources],
'unpickleable': [UnpickleableResult],
}
tasks = [
# Codegen
(JavaSources, [
Select(ThriftSources),
SelectVariant(ApacheThriftJavaConfiguration, 'thrift')
], gen_apache_thrift),
(PythonSources, [
Select(ThriftSources),
SelectVariant(ApacheThriftPythonConfiguration, 'thrift')
], gen_apache_thrift),
(ScalaSources, [
Select(ThriftSources),
SelectVariant(ScroogeScalaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)
], gen_scrooge_thrift),
(JavaSources, [
Select(ThriftSources),
SelectVariant(ScroogeJavaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)
], gen_scrooge_thrift),
] + [
# scala dependency inference
(ScalaSources, [
Select(Address),
Select(ScalaInferredDepsSources),
SelectDependencies(Address, ImportedJVMPackages)
], reify_scala_sources),
(ImportedJVMPackages, [
Select(Address),
Select(ScalaInferredDepsSources),
SelectLiteral(source_roots, SourceRoots)
], extract_scala_imports),
# TODO: The request for an AddressFamily for each member of a SearchPath will fail whenever
# a member of the path doesn't exist. Need to allow for optional products and to then
# request the AddressFamilies optionally here.
(Address, [
Select(JVMPackageName),
SelectDependencies(AddressFamily, SearchPath)
], select_package_address),
(SearchPath, [
Select(JVMPackageName),
SelectLiteral(source_roots, SourceRoots)
], calculate_package_search_path),
] + [
# Remote dependency resolution
(Classpath, [Select(Jar)], ivy_resolve),
(Jar, [Select(ManagedJar),
SelectVariant(ManagedResolve, 'resolve')], select_rev),
] + [
# Compilers
(Classpath, [Select(ResourceSources)], isolate_resources),
(Classpath, [Select(BuildPropertiesConfiguration)], write_name_file),
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources)], javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources)], scalac),
] + [
# TODO
(UnpickleableOutput, [], unpickleable_output),
(UnpickleableResult, [Select(UnpickleableOutput)], unpickleable_input),
] + (create_graph_tasks(address_mapper))
scheduler = LocalScheduler(products_by_goal, tasks)
return scheduler
def setup_json_scheduler(build_root):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype :class:`pants.engine.exp.scheduler.LocalScheduler`
"""
symbol_table_cls = ExampleTable
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=r'^BLD.json$',
parser_cls=JsonParser)
# TODO(John Sirois): once the options system is plumbed, make the tool spec configurable.
# It could also just be pointed at the scrooge jar at that point.
scrooge_tool_address = Address.parse('src/scala/scrooge')
# TODO: Placeholder for the SourceRoot subsystem.
source_roots = SourceRoots(('src/java',))
goals = {
'compile': Classpath,
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': Classpath,
'list': Address,
'walk': Path,
GenGoal.name(): GenGoal,
'unpickleable': UnpickleableResult,
}
tasks = [
# Codegen
GenGoal.signature(),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftJavaConfiguration, 'thrift')],
gen_apache_thrift),
(PythonSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftPythonConfiguration, 'thrift')],
gen_apache_thrift),
(ScalaSources,
[Select(ThriftSources),
SelectVariant(ScroogeScalaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ScroogeJavaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
] + [
# scala dependency inference
(ScalaSources,
[Select(ScalaInferredDepsSources),
SelectDependencies(Address, ImportedJVMPackages)],
reify_scala_sources),
(ImportedJVMPackages,
[SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)],
extract_scala_imports),
(Address,
[Select(JVMPackageName),
SelectDependencies(AddressFamily, Paths)],
select_package_address),
(Paths,
[Select(JVMPackageName),
SelectLiteral(source_roots, SourceRoots)],
calculate_package_search_path),
] + [
# Remote dependency resolution
(Classpath,
[Select(Jar)],
ivy_resolve),
(Jar,
[Select(ManagedJar),
SelectVariant(ManagedResolve, 'resolve')],
select_rev),
] + [
# Compilers
(Classpath,
[Select(ResourceSources)],
isolate_resources),
(Classpath,
[Select(BuildPropertiesConfiguration)],
write_name_file),
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources)],
javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources)],
scalac),
] + [
# TODO
(UnpickleableOutput,
[],
unpickleable_output),
(UnpickleableResult,
[Select(UnpickleableOutput)],
unpickleable_input),
] + (
create_graph_tasks(address_mapper, symbol_table_cls)
) + (
create_fs_tasks(build_root)
)
scheduler = LocalScheduler(goals, symbol_table_cls, tasks)
return scheduler
