def list():
"""Lists all addresses under the current build root subject to `--spec-excludes` constraints."""
build_root = get_buildroot()
options, build_config = OptionsInitializer().setup()
aliases = build_config.registered_aliases()
symbol_table = {alias: Target for alias in aliases.target_types}
object_table = aliases.objects
def per_path_symbol_factory(path, global_symbols):
per_path_symbols = {}
symbols = global_symbols.copy()
for alias, target_macro_factory in aliases.target_macro_factories.items(
):
for target_type in target_macro_factory.target_types:
symbols[
target_type] = lambda *args, **kwargs: per_path_symbols[
alias](*args, **kwargs)
parse_context = ParseContext(rel_path=os.path.relpath(
os.path.dirname(path), build_root),
type_aliases=symbols)
for alias, object_factory in aliases.context_aware_object_factories.items(
):
per_path_symbols[alias] = object_factory(parse_context)
for alias, target_macro_factory in aliases.target_macro_factories.items(
):
target_macro = target_macro_factory.target_macro(parse_context)
per_path_symbols[alias] = target_macro
for target_type in target_macro_factory.target_types:
per_path_symbols[target_type] = target_macro
return per_path_symbols
parser = legacy_python_callbacks_parser(
symbol_table,
object_table=object_table,
per_path_symbol_factory=per_path_symbol_factory)
mapper = AddressMapper(build_root, parser=parser)
# Should use build_ignore_patterns instead.
spec_excludes = None
for address, obj in mapper.walk_addressables(path_excludes=spec_excludes):
print(address.spec)
def setUp(self):
self.work_dir = safe_mkdtemp()
self.addCleanup(safe_rmtree, self.work_dir)
self.build_root = os.path.join(self.work_dir, 'build_root')
shutil.copytree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'),
self.build_root)
parser = partial(parse_json, symbol_table={'configuration': Configuration, 'target': Target})
self.address_mapper = AddressMapper(build_root=self.build_root,
build_pattern=r'.+\.BUILD.json$',
parser=parser)
self.a_b_target = Target(name='b',
dependencies=['//d:e'],
configurations=['//a', Configuration(embedded='yes')])
def setup(options=None):
if not options:
options, _ = OptionsInitializer(OptionsBootstrapper()).setup()
build_root = get_buildroot()
cmd_line_spec_parser = CmdLineSpecParser(build_root)
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs]
storage = Storage.create(debug=False)
project_tree = FileSystemProjectTree(build_root)
symbol_table_cls = LegacyTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=LegacyPythonCallbacksParser)
# Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph
# will explicitly request the products it needs.
tasks = (
create_legacy_graph_tasks() +
create_fs_tasks() +
create_graph_tasks(address_mapper, symbol_table_cls)
)
return (
LocalScheduler(dict(), tasks, storage, project_tree),
storage,
options,
spec_roots,
symbol_table_cls
)
def create(self, build_pattern=None, parser_cls=None, inline=False):
mapper = AddressMapper(build_root=os.path.dirname(__file__),
symbol_table_cls=TestTable,
build_pattern=build_pattern,
parser_cls=parser_cls)
return LocalScheduler({self._goal: [self._product]},
create_graph_tasks(mapper))
def setup_json_scheduler(build_root):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype tuple of (:class:`pants.engine.exp.graph.Graph`,
:class:`pants.engine.exp.scheduler.LocalScheduler`)
"""
symbol_table = {
'apache_thrift_configuration': ApacheThriftConfiguration,
'jar': Jar,
'requirement': Requirement,
'scrooge_configuration': ScroogeConfiguration,
'sources': AddressableSources,
'target': Target,
'build_properties': BuildPropertiesConfiguration
}
json_parser = functools.partial(parse_json, symbol_table=symbol_table)
graph = Graph(
AddressMapper(build_root=build_root,
build_pattern=r'^BLD.json$',
parser=json_parser))
planners = Planners([
ApacheThriftPlanner(),
BuildPropertiesPlanner(),
GlobalIvyResolvePlanner(),
JavacPlanner(),
ScalacPlanner(),
ScroogePlanner(),
UnpickleableInputsPlanner(),
UnpickleableResultPlanner()
])
scheduler = LocalScheduler(graph, planners)
return graph, scheduler
def setUp(self):
self.work_dir = safe_mkdtemp()
self.addCleanup(safe_rmtree, self.work_dir)
self.build_root = os.path.join(self.work_dir, 'build_root')
shutil.copytree(
os.path.join(os.path.dirname(__file__), 'examples/mapper_test'),
self.build_root)
self.address_mapper = AddressMapper(build_root=self.build_root,
symbol_table_cls=TargetTable,
parser_cls=JsonParser,
build_pattern=r'.+\.BUILD.json$')
self.a_b_target = Target(
name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')])
def create(self, build_pattern=None, parser_cls=None, inline=False):
symbol_table_cls = TestTable
mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=build_pattern,
parser_cls=parser_cls)
tasks = (create_fs_tasks(self._build_root) +
create_graph_tasks(mapper, symbol_table_cls))
return LocalScheduler({self._goal: self._product}, symbol_table_cls,
tasks)
def setUp(self):
self.work_dir = safe_mkdtemp()
self.addCleanup(safe_rmtree, self.work_dir)
self.build_root = os.path.join(self.work_dir, 'build_root')
shutil.copytree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'),
self.build_root)
parser = partial(parse_json, symbol_table={'configuration': Configuration, 'target': Target})
self.address_mapper = AddressMapper(build_root=self.build_root,
build_pattern=r'.+\.BUILD.json$',
parser=parser)
def list():
"""Lists all addresses under the current build root subject to `--spec-excludes` constraints."""
build_root = get_buildroot()
options, build_config = OptionsInitializer().setup()
aliases = build_config.registered_aliases()
symbol_table = {alias: Target for alias in aliases.target_types}
object_table = aliases.objects
def per_path_symbol_factory(path, global_symbols):
per_path_symbols = {}
symbols = global_symbols.copy()
for alias, target_macro_factory in aliases.target_macro_factories.items():
for target_type in target_macro_factory.target_types:
symbols[target_type] = lambda *args, **kwargs: per_path_symbols[alias](*args, **kwargs)
parse_context = ParseContext(rel_path=os.path.relpath(os.path.dirname(path), build_root),
type_aliases=symbols)
for alias, object_factory in aliases.context_aware_object_factories.items():
per_path_symbols[alias] = object_factory(parse_context)
for alias, target_macro_factory in aliases.target_macro_factories.items():
target_macro = target_macro_factory.target_macro(parse_context)
per_path_symbols[alias] = target_macro
for target_type in target_macro_factory.target_types:
per_path_symbols[target_type] = target_macro
return per_path_symbols
parser = legacy_python_callbacks_parser(symbol_table,
object_table=object_table,
per_path_symbol_factory=per_path_symbol_factory)
mapper = AddressMapper(build_root, parser=parser)
spec_excludes = options.for_global_scope().spec_excludes
for address, obj in mapper.walk_addressables(path_excludes=spec_excludes):
print(address.spec)
def create(self, build_pattern=None, parser_cls=None, inline=False):
symbol_table_cls = TestTable
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=build_pattern,
parser_cls=parser_cls)
tasks = create_graph_tasks(address_mapper, symbol_table_cls)
project_tree = self.mk_fs_tree(
os.path.join(os.path.dirname(__file__), 'examples'))
scheduler, _ = self.mk_scheduler(tasks=tasks,
storage=self.storage,
project_tree=project_tree,
symbol_table_cls=symbol_table_cls)
return scheduler
def setUp(self):
self.work_dir = safe_mkdtemp()
self.addCleanup(safe_rmtree, self.work_dir)
self.build_root = os.path.join(self.work_dir, 'build_root')
shutil.copytree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'),
self.build_root)
parser = partial(parse_json, symbol_table={'struct': Struct, 'target': Target})
self.address_mapper = AddressMapper(build_root=self.build_root,
build_pattern=r'.+\.BUILD.json$',
parser=parser)
self.a_b_target = Target(name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')])
def setUp(self):
# Set up a scheduler that supports address mapping.
symbol_table_cls = TargetTable
self.storage = Storage.create(in_memory=True)
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=JsonParser,
build_pattern=r'.+\.BUILD.json$')
tasks = create_graph_tasks(address_mapper, symbol_table_cls)
build_root_src = os.path.join(os.path.dirname(__file__), 'examples/mapper_test')
self.scheduler, _, self.build_root = self.mk_scheduler(tasks=tasks,
build_root_src=build_root_src,
storage=self.storage,
symbol_table_cls=symbol_table_cls)
self.a_b = Address.parse('a/b')
self.a_b_target = Target(name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')],
type_alias='target')
def setUp(self):
self.work_dir = safe_mkdtemp()
self.addCleanup(safe_rmtree, self.work_dir)
self.build_root = os.path.join(self.work_dir, 'build_root')
shutil.copytree(
os.path.join(os.path.dirname(__file__), 'examples/mapper_test'),
self.build_root)
self._goal = 'list'
symbol_table_cls = TargetTable
self.address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
parser_cls=JsonParser,
build_pattern=r'.+\.BUILD.json$')
tasks = (create_fs_tasks(self.build_root) +
create_graph_tasks(self.address_mapper, symbol_table_cls))
self.scheduler = LocalScheduler({self._goal: UnhydratedStruct},
symbol_table_cls, tasks)
self.a_b = Address.parse('a/b')
self.a_b_target = Target(
name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')])
def create_graph(self, build_pattern=None, parser=None, inline=False):
mapper = AddressMapper(build_root=os.path.dirname(__file__),
build_pattern=build_pattern,
parser=parser)
return Graph(mapper, inline=inline)
class AddressMapperTest(unittest.TestCase):
def setUp(self):
self.work_dir = safe_mkdtemp()
self.addCleanup(safe_rmtree, self.work_dir)
self.build_root = os.path.join(self.work_dir, 'build_root')
shutil.copytree(
os.path.join(os.path.dirname(__file__), 'examples/mapper_test'),
self.build_root)
parser = partial(parse_json,
symbol_table={
'struct': Struct,
'target': Target
})
self.address_mapper = AddressMapper(build_root=self.build_root,
build_pattern=r'.+\.BUILD.json$',
parser=parser)
self.a_b_target = Target(
name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')])
def test_no_family(self):
with self.assertRaises(ResolveError):
self.address_mapper.family('a/c')
# Errors are not cached.
with self.assertRaises(ResolveError):
self.address_mapper.family('a/c')
build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json')
touch(build_file)
address_family = self.address_mapper.family('a/c')
self.assertEqual({}, address_family.addressables)
# But success is cached.
self.assertIs(address_family, self.address_mapper.family('a/c'))
def test_no_address_no_family(self):
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/c'))
# Errors are not cached.
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/c'))
build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json')
with safe_open(build_file, 'w') as fp:
fp.write('{"type_alias": "struct", "name": "c"}')
resolved = self.address_mapper.resolve(Address.parse('a/c'))
self.assertEqual(Struct(name='c'), resolved)
# But success is cached.
self.assertIs(resolved,
self.address_mapper.resolve(Address.parse('a/c')))
def test_resolve(self):
resolved = self.address_mapper.resolve(Address.parse('a/b'))
self.assertEqual(self.a_b_target, resolved)
def test_invalidate_build_file_added(self):
address_family = self.address_mapper.family('a/b')
self.assertEqual({Address.parse('a/b'): self.a_b_target},
address_family.addressables)
with open(os.path.join(self.build_root, 'a/b/sibling.BUILD.json'),
'w') as fp:
fp.write('{"type_alias": "struct", "name": "c"}')
still_valid = self.address_mapper.family('a/b')
self.assertIs(address_family, still_valid)
self.address_mapper.invalidate_build_file('a/b/sibling.BUILD.json')
newly_formed = self.address_mapper.family('a/b')
self.assertIsNot(address_family, newly_formed)
self.assertEqual(
{
Address.parse('a/b'): self.a_b_target,
Address.parse('a/b:c'): Struct(name='c')
}, newly_formed.addressables)
def test_invalidate_build_file_changed(self):
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b:c'))
build_file = os.path.join(self.build_root, 'a/b/b.BUILD.json')
with safe_open(build_file, 'w+') as fp:
fp.write('{"type_alias": "struct", "name": "c"}')
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b:c'))
self.address_mapper.invalidate_build_file('a/b/b.BUILD.json')
resolved = self.address_mapper.resolve(Address.parse('a/b:c'))
self.assertEqual(Struct(name='c'), resolved)
# But success is cached.
self.assertIs(resolved,
self.address_mapper.resolve(Address.parse('a/b:c')))
def test_invalidate_build_file_removed(self):
resolved = self.address_mapper.resolve(Address.parse('a/b'))
self.assertEqual(self.a_b_target, resolved)
build_file = os.path.join(self.build_root, 'a/b/b.BUILD.json')
os.unlink(build_file)
self.assertIs(resolved,
self.address_mapper.resolve(Address.parse('a/b')))
self.address_mapper.invalidate_build_file(build_file)
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b'))
def test_invalidation_un_normalized(self):
resolved = self.address_mapper.resolve(Address.parse('a/b'))
self.assertEqual(self.a_b_target, resolved)
os.unlink(os.path.join(self.build_root, 'a/b/b.BUILD.json'))
self.assertIs(resolved,
self.address_mapper.resolve(Address.parse('a/b')))
un_normalized_build_root = os.path.join(self.work_dir,
'build_root_linked')
os.symlink(self.build_root, un_normalized_build_root)
un_normalized_build_file = os.path.join(un_normalized_build_root,
'a/b/b.BUILD.json')
self.address_mapper.invalidate_build_file(un_normalized_build_file)
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b'))
def test_invalidation_relative(self):
resolved = self.address_mapper.resolve(Address.parse('a/b'))
self.assertEqual(self.a_b_target, resolved)
build_file = os.path.join(self.build_root, 'a/b/b.BUILD.json')
os.unlink(build_file)
self.assertIs(resolved,
self.address_mapper.resolve(Address.parse('a/b')))
self.address_mapper.invalidate_build_file('a/b/b.BUILD.json')
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b'))
@staticmethod
def addr(spec):
return Address.parse(spec)
def test_walk_addressables(self):
self.assertEqual(
sorted([(self.addr('//:root'), Struct(name='root')),
(self.addr('a/b:b'), self.a_b_target),
(self.addr('a/d:d'), Target(name='d')),
(self.addr('a/d/e:e'), Target(name='e')),
(self.addr('a/d/e:e-prime'), Struct(name='e-prime'))]),
sorted(self.address_mapper.walk_addressables()))
def test_walk_addressables_rel_path(self):
self.assertEqual(
sorted([(self.addr('a/d:d'), Target(name='d')),
(self.addr('a/d/e:e'), Target(name='e')),
(self.addr('a/d/e:e-prime'), Struct(name='e-prime'))]),
sorted(self.address_mapper.walk_addressables(rel_path='a/d')))
def test_walk_addressables_path_excludes(self):
self.assertEqual([(self.addr('//:root'), Struct(name='root')),
(self.addr('a/d:d'), Target(name='d'))],
list(
self.address_mapper.walk_addressables(
path_excludes=['a/b', 'a/d/e'])))
def setup_json_scheduler(build_root, debug=True):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype A tuple of :class:`pants.engine.exp.scheduler.LocalScheduler`,
:class:`pants.engine.exp.storage.Storage`.
"""
storage = Storage.create(debug=debug)
symbol_table_cls = ExampleTable
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls,
build_pattern=r'^BLD.json$',
parser_cls=JsonParser)
source_roots = SourceRoots(('src/java','src/scala'))
scrooge_tool_address = Address.parse('src/scala/scrooge')
goals = {
'compile': Classpath,
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': Classpath,
'list': Address,
GenGoal.name(): GenGoal,
'unpickleable': UnpickleableResult,
'ls': File,
'cat': FileContent,
}
tasks = [
# Codegen
GenGoal.signature(),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftJavaConfiguration, 'thrift')],
gen_apache_thrift),
(PythonSources,
[Select(ThriftSources),
SelectVariant(ApacheThriftPythonConfiguration, 'thrift')],
gen_apache_thrift),
(ScalaSources,
[Select(ThriftSources),
SelectVariant(ScroogeScalaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
(JavaSources,
[Select(ThriftSources),
SelectVariant(ScroogeJavaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)],
gen_scrooge_thrift),
] + [
# scala dependency inference
(ScalaSources,
[Select(ScalaInferredDepsSources),
SelectDependencies(Address, ImportedJVMPackages)],
reify_scala_sources),
(ImportedJVMPackages,
[SelectProjection(FilesContent, PathGlobs, ('path_globs',), ScalaInferredDepsSources)],
extract_scala_imports),
(Address,
[Select(JVMPackageName),
SelectDependencies(AddressFamily, Dirs)],
select_package_address),
(PathGlobs,
[Select(JVMPackageName),
SelectLiteral(source_roots, SourceRoots)],
calculate_package_search_path),
] + [
# Remote dependency resolution
(Classpath,
[Select(Jar)],
ivy_resolve),
(Jar,
[Select(ManagedJar),
SelectVariant(ManagedResolve, 'resolve')],
select_rev),
] + [
# Compilers
(Classpath,
[Select(ResourceSources)],
isolate_resources),
(Classpath,
[Select(BuildPropertiesConfiguration)],
write_name_file),
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources)],
javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources)],
scalac),
] + [
# TODO
(UnpickleableOutput,
[],
unpickleable_output),
(UnpickleableResult,
[Select(UnpickleableOutput)],
unpickleable_input),
] + (
create_graph_tasks(address_mapper, symbol_table_cls)
) + (
create_fs_tasks()
)
project_tree = FileSystemProjectTree(build_root)
return LocalScheduler(goals, tasks, storage, project_tree, None, GraphValidator(symbol_table_cls)), storage
class AddressMapperTest(unittest.TestCase):
def setUp(self):
self.work_dir = safe_mkdtemp()
self.addCleanup(safe_rmtree, self.work_dir)
self.build_root = os.path.join(self.work_dir, 'build_root')
shutil.copytree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'),
self.build_root)
self.address_mapper = AddressMapper(build_root=self.build_root,
symbol_table_cls=TargetTable,
parser_cls=JsonParser,
build_pattern=r'.+\.BUILD.json$')
self.a_b_target = Target(name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')])
def test_no_family(self):
with self.assertRaises(ResolveError):
self.address_mapper.family('a/c')
# Errors are not cached.
with self.assertRaises(ResolveError):
self.address_mapper.family('a/c')
build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json')
touch(build_file)
address_family = self.address_mapper.family('a/c')
self.assertEqual({}, address_family.addressables)
def test_no_address_no_family(self):
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/c'))
# Errors are not cached.
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/c'))
build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json')
with safe_open(build_file, 'w') as fp:
fp.write('{"type_alias": "struct", "name": "c"}')
resolved = self.address_mapper.resolve(Address.parse('a/c'))
self.assertEqual(Struct(name='c'), resolved)
def test_resolve(self):
resolved = self.address_mapper.resolve(Address.parse('a/b'))
self.assertEqual(self.a_b_target, resolved)
@staticmethod
def addr(spec):
return Address.parse(spec)
def test_walk_addressables(self):
self.assertEqual(sorted([(self.addr('//:root'), Struct(name='root')),
(self.addr('a/b:b'), self.a_b_target),
(self.addr('a/d:d'), Target(name='d')),
(self.addr('a/d/e:e'), Target(name='e')),
(self.addr('a/d/e:e-prime'), Struct(name='e-prime'))]),
sorted(self.address_mapper.walk_addressables()))
def test_walk_addressables_rel_path(self):
self.assertEqual(sorted([(self.addr('a/d:d'), Target(name='d')),
(self.addr('a/d/e:e'), Target(name='e')),
(self.addr('a/d/e:e-prime'), Struct(name='e-prime'))]),
sorted(self.address_mapper.walk_addressables(rel_path='a/d')))
def test_walk_addressables_path_excludes(self):
self.assertEqual([(self.addr('//:root'), Struct(name='root')),
(self.addr('a/d:d'), Target(name='d'))],
list(self.address_mapper.walk_addressables(path_excludes=['a/b', 'a/d/e'])))
def setup_json_scheduler(build_root):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype tuple of (:class:`pants.engine.exp.graph.Graph`,
:class:`pants.engine.exp.scheduler.LocalScheduler`)
"""
symbol_table = {
'apache_thrift_java_configuration': ApacheThriftJavaConfiguration,
'apache_thrift_python_configuration': ApacheThriftPythonConfiguration,
'jar': Jar,
'managed_jar': ManagedJar,
'managed_resolve': ManagedResolve,
'requirement': Requirement,
'scrooge_java_configuration': ScroogeJavaConfiguration,
'scrooge_scala_configuration': ScroogeScalaConfiguration,
'java': JavaSources,
'python': PythonSources,
'resources': ResourceSources,
'scala': ScalaSources,
'thrift': ThriftSources,
'target': Target,
'build_properties': BuildPropertiesConfiguration
}
json_parser = functools.partial(parse_json, symbol_table=symbol_table)
graph = Graph(
AddressMapper(build_root=build_root,
build_pattern=r'^BLD.json$',
parser=json_parser))
# TODO(John Sirois): once the options system is plumbed, make the tool spec configurable.
# It could also just be pointed at the scrooge jar at that point.
scrooge_tool_address = Address.parse('src/scala/scrooge')
products_by_goal = {
'compile': [Classpath],
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': [Classpath],
'gen': [JavaSources, PythonSources, ResourceSources, ScalaSources],
'unpickleable': [UnpickleableResult],
}
tasks = [
(JavaSources, [
Select(ThriftSources),
SelectVariant('thrift', ApacheThriftJavaConfiguration)
], gen_apache_thrift),
(PythonSources, [
Select(ThriftSources),
SelectVariant('thrift', ApacheThriftPythonConfiguration)
], gen_apache_thrift),
(ScalaSources, [
Select(ThriftSources),
SelectVariant('thrift', ScroogeScalaConfiguration),
SelectAddress(scrooge_tool_address, Classpath)
], gen_scrooge_thrift),
(JavaSources, [
Select(ThriftSources),
SelectVariant('thrift', ScroogeJavaConfiguration),
SelectAddress(scrooge_tool_address, Classpath)
], gen_scrooge_thrift),
(Classpath, [Select(Jar)], ivy_resolve),
(Jar, [Select(ManagedJar),
SelectVariant('resolve', ManagedResolve)], select_rev),
(Classpath, [Select(ResourceSources)], isolate_resources),
(Classpath, [Select(BuildPropertiesConfiguration)], write_name_file),
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources)], javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources)], scalac),
(UnpickleableOutput, [], unpickleable_output),
(UnpickleableResult, [Select(UnpickleableOutput)], unpickleable_input),
]
scheduler = LocalScheduler(graph, products_by_goal, tasks)
return graph, scheduler
def setup_json_scheduler(build_root):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype :class:`pants.engine.exp.scheduler.LocalScheduler`
"""
address_mapper = AddressMapper(build_root=build_root,
symbol_table_cls=ExampleTable,
build_pattern=r'^BLD.json$',
parser_cls=JsonParser)
# TODO(John Sirois): once the options system is plumbed, make the tool spec configurable.
# It could also just be pointed at the scrooge jar at that point.
scrooge_tool_address = Address.parse('src/scala/scrooge')
# TODO: Placeholder for the SourceRoot subsystem.
source_roots = SourceRoots(build_root, ('src/java', ))
products_by_goal = {
'compile': [Classpath],
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': [Classpath],
'gen': [JavaSources, PythonSources, ResourceSources, ScalaSources],
'unpickleable': [UnpickleableResult],
}
tasks = [
# Codegen
(JavaSources, [
Select(ThriftSources),
SelectVariant(ApacheThriftJavaConfiguration, 'thrift')
], gen_apache_thrift),
(PythonSources, [
Select(ThriftSources),
SelectVariant(ApacheThriftPythonConfiguration, 'thrift')
], gen_apache_thrift),
(ScalaSources, [
Select(ThriftSources),
SelectVariant(ScroogeScalaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)
], gen_scrooge_thrift),
(JavaSources, [
Select(ThriftSources),
SelectVariant(ScroogeJavaConfiguration, 'thrift'),
SelectLiteral(scrooge_tool_address, Classpath)
], gen_scrooge_thrift),
] + [
# scala dependency inference
(ScalaSources, [
Select(Address),
Select(ScalaInferredDepsSources),
SelectDependencies(Address, ImportedJVMPackages)
], reify_scala_sources),
(ImportedJVMPackages, [
Select(Address),
Select(ScalaInferredDepsSources),
SelectLiteral(source_roots, SourceRoots)
], extract_scala_imports),
# TODO: The request for an AddressFamily for each member of a SearchPath will fail whenever
# a member of the path doesn't exist. Need to allow for optional products and to then
# request the AddressFamilies optionally here.
(Address, [
Select(JVMPackageName),
SelectDependencies(AddressFamily, SearchPath)
], select_package_address),
(SearchPath, [
Select(JVMPackageName),
SelectLiteral(source_roots, SourceRoots)
], calculate_package_search_path),
] + [
# Remote dependency resolution
(Classpath, [Select(Jar)], ivy_resolve),
(Jar, [Select(ManagedJar),
SelectVariant(ManagedResolve, 'resolve')], select_rev),
] + [
# Compilers
(Classpath, [Select(ResourceSources)], isolate_resources),
(Classpath, [Select(BuildPropertiesConfiguration)], write_name_file),
(Classpath,
[Select(JavaSources),
SelectDependencies(Classpath, JavaSources)], javac),
(Classpath,
[Select(ScalaSources),
SelectDependencies(Classpath, ScalaSources)], scalac),
] + [
# TODO
(UnpickleableOutput, [], unpickleable_output),
(UnpickleableResult, [Select(UnpickleableOutput)], unpickleable_input),
] + (create_graph_tasks(address_mapper))
scheduler = LocalScheduler(products_by_goal, tasks)
return scheduler
class AddressMapperTest(unittest.TestCase):
def setUp(self):
self.work_dir = safe_mkdtemp()
self.addCleanup(safe_rmtree, self.work_dir)
self.build_root = os.path.join(self.work_dir, 'build_root')
shutil.copytree(
os.path.join(os.path.dirname(__file__), 'examples/mapper_test'),
self.build_root)
self.address_mapper = AddressMapper(build_root=self.build_root,
symbol_table_cls=TargetTable,
parser_cls=JsonParser,
build_pattern=r'.+\.BUILD.json$')
self.a_b_target = Target(
name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')])
def test_no_family(self):
with self.assertRaises(ResolveError):
self.address_mapper.family('a/c')
# Errors are not cached.
with self.assertRaises(ResolveError):
self.address_mapper.family('a/c')
build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json')
touch(build_file)
address_family = self.address_mapper.family('a/c')
self.assertEqual({}, address_family.addressables)
def test_no_address_no_family(self):
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/c'))
# Errors are not cached.
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/c'))
build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json')
with safe_open(build_file, 'w') as fp:
fp.write('{"type_alias": "struct", "name": "c"}')
resolved = self.address_mapper.resolve(Address.parse('a/c'))
self.assertEqual(Struct(name='c'), resolved)
def test_resolve(self):
resolved = self.address_mapper.resolve(Address.parse('a/b'))
self.assertEqual(self.a_b_target, resolved)
@staticmethod
def addr(spec):
return Address.parse(spec)
def test_walk_addressables(self):
self.assertEqual(
sorted([(self.addr('//:root'), Struct(name='root')),
(self.addr('a/b:b'), self.a_b_target),
(self.addr('a/d:d'), Target(name='d')),
(self.addr('a/d/e:e'), Target(name='e')),
(self.addr('a/d/e:e-prime'), Struct(name='e-prime'))]),
sorted(self.address_mapper.walk_addressables()))
def test_walk_addressables_rel_path(self):
self.assertEqual(
sorted([(self.addr('a/d:d'), Target(name='d')),
(self.addr('a/d/e:e'), Target(name='e')),
(self.addr('a/d/e:e-prime'), Struct(name='e-prime'))]),
sorted(self.address_mapper.walk_addressables(rel_path='a/d')))
def test_walk_addressables_path_excludes(self):
self.assertEqual([(self.addr('//:root'), Struct(name='root')),
(self.addr('a/d:d'), Target(name='d'))],
list(
self.address_mapper.walk_addressables(
path_excludes=['a/b', 'a/d/e'])))
class AddressMapperTest(unittest.TestCase):
def setUp(self):
self.work_dir = safe_mkdtemp()
self.addCleanup(safe_rmtree, self.work_dir)
self.build_root = os.path.join(self.work_dir, 'build_root')
shutil.copytree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'),
self.build_root)
parser = partial(parse_json, symbol_table={'configuration': Configuration, 'target': Target})
self.address_mapper = AddressMapper(build_root=self.build_root,
build_pattern=r'.+\.BUILD.json$',
parser=parser)
self.a_b_target = Target(name='b',
dependencies=['//d:e'],
configurations=['//a', Configuration(embedded='yes')])
def test_no_family(self):
with self.assertRaises(ResolveError):
self.address_mapper.family('a/c')
# Errors are not cached.
with self.assertRaises(ResolveError):
self.address_mapper.family('a/c')
build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json')
touch(build_file)
address_family = self.address_mapper.family('a/c')
self.assertEqual({}, address_family.addressables)
# But success is cached.
self.assertIs(address_family, self.address_mapper.family('a/c'))
def test_no_address_no_family(self):
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/c'))
# Errors are not cached.
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/c'))
build_file = os.path.join(self.build_root, 'a/c/c.BUILD.json')
with safe_open(build_file, 'w') as fp:
fp.write('{"type_alias": "configuration", "name": "c"}')
resolved = self.address_mapper.resolve(Address.parse('a/c'))
self.assertEqual(Configuration(name='c'), resolved)
# But success is cached.
self.assertIs(resolved, self.address_mapper.resolve(Address.parse('a/c')))
def test_resolve(self):
resolved = self.address_mapper.resolve(Address.parse('a/b'))
self.assertEqual(self.a_b_target, resolved)
def test_invalidate_build_file_added(self):
address_family = self.address_mapper.family('a/b')
self.assertEqual({Address.parse('a/b'): self.a_b_target},
address_family.addressables)
with open(os.path.join(self.build_root, 'a/b/sibling.BUILD.json'), 'w') as fp:
fp.write('{"type_alias": "configuration", "name": "c"}')
still_valid = self.address_mapper.family('a/b')
self.assertIs(address_family, still_valid)
self.address_mapper.invalidate_build_file('a/b/sibling.BUILD.json')
newly_formed = self.address_mapper.family('a/b')
self.assertIsNot(address_family, newly_formed)
self.assertEqual({Address.parse('a/b'): self.a_b_target,
Address.parse('a/b:c'): Configuration(name='c')},
newly_formed.addressables)
def test_invalidate_build_file_changed(self):
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b:c'))
build_file = os.path.join(self.build_root, 'a/b/b.BUILD.json')
with safe_open(build_file, 'w+') as fp:
fp.write('{"type_alias": "configuration", "name": "c"}')
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b:c'))
self.address_mapper.invalidate_build_file('a/b/b.BUILD.json')
resolved = self.address_mapper.resolve(Address.parse('a/b:c'))
self.assertEqual(Configuration(name='c'), resolved)
# But success is cached.
self.assertIs(resolved, self.address_mapper.resolve(Address.parse('a/b:c')))
def test_invalidate_build_file_removed(self):
resolved = self.address_mapper.resolve(Address.parse('a/b'))
self.assertEqual(self.a_b_target, resolved)
build_file = os.path.join(self.build_root, 'a/b/b.BUILD.json')
os.unlink(build_file)
self.assertIs(resolved, self.address_mapper.resolve(Address.parse('a/b')))
self.address_mapper.invalidate_build_file(build_file)
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b'))
def test_invalidation_un_normalized(self):
resolved = self.address_mapper.resolve(Address.parse('a/b'))
self.assertEqual(self.a_b_target, resolved)
os.unlink(os.path.join(self.build_root, 'a/b/b.BUILD.json'))
self.assertIs(resolved, self.address_mapper.resolve(Address.parse('a/b')))
un_normalized_build_root = os.path.join(self.work_dir, 'build_root_linked')
os.symlink(self.build_root, un_normalized_build_root)
un_normalized_build_file = os.path.join(un_normalized_build_root, 'a/b/b.BUILD.json')
self.address_mapper.invalidate_build_file(un_normalized_build_file)
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b'))
def test_invalidation_relative(self):
resolved = self.address_mapper.resolve(Address.parse('a/b'))
self.assertEqual(self.a_b_target, resolved)
build_file = os.path.join(self.build_root, 'a/b/b.BUILD.json')
os.unlink(build_file)
self.assertIs(resolved, self.address_mapper.resolve(Address.parse('a/b')))
self.address_mapper.invalidate_build_file('a/b/b.BUILD.json')
with self.assertRaises(ResolveError):
self.address_mapper.resolve(Address.parse('a/b'))
@staticmethod
def addr(spec):
return Address.parse(spec)
def test_walk_addressables(self):
self.assertEqual(sorted([(self.addr('//:root'), Configuration(name='root')),
(self.addr('a/b:b'), self.a_b_target),
(self.addr('a/d:d'), Target(name='d')),
(self.addr('a/d/e:e'), Target(name='e')),
(self.addr('a/d/e:e-prime'), Configuration(name='e-prime'))]),
sorted(self.address_mapper.walk_addressables()))
def test_walk_addressables_rel_path(self):
self.assertEqual(sorted([(self.addr('a/d:d'), Target(name='d')),
(self.addr('a/d/e:e'), Target(name='e')),
(self.addr('a/d/e:e-prime'), Configuration(name='e-prime'))]),
sorted(self.address_mapper.walk_addressables(rel_path='a/d')))
def test_walk_addressables_path_excludes(self):
self.assertEqual([(self.addr('//:root'), Configuration(name='root')),
(self.addr('a/d:d'), Target(name='d'))],
list(self.address_mapper.walk_addressables(path_excludes=['a/b', 'a/d/e'])))
