def setup_json_scheduler(build_root, native):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype :class:`pants.engine.scheduler.LocalScheduler`
"""
symbol_table = ExampleTable()
# Register "literal" subjects required for these tasks.
# TODO: Replace with `Subsystems`.
address_mapper = AddressMapper(build_patterns=('BLD.json',),
parser=JsonParser(symbol_table))
work_dir = os_path_join(build_root, '.pants.d')
project_tree = FileSystemProjectTree(build_root)
goals = {
'compile': Classpath,
# TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product.
'resolve': Classpath,
'list': BuildFileAddresses,
GenGoal.name(): GenGoal,
'ls': Snapshot,
'cat': FilesContent,
}
tasks = [
# Codegen
GenGoal.rule(),
gen_apache_java_thrift,
gen_apache_python_thrift,
gen_scrooge_scala_thrift,
gen_scrooge_java_thrift,
SingletonRule(Scrooge, Scrooge(Address.parse('src/scala/scrooge')))
] + [
# scala dependency inference
reify_scala_sources,
select_package_address,
calculate_package_search_path,
SingletonRule(SourceRoots, SourceRoots(('src/java','src/scala'))),
] + [
# Remote dependency resolution
ivy_resolve,
select_rev,
] + [
# Compilers
isolate_resources,
write_name_file,
javac,
scalac,
] + (
create_graph_rules(address_mapper, symbol_table)
) + (
create_fs_rules()
)
return LocalScheduler(work_dir,
goals,
tasks,
project_tree,
native)
def test_full_graph_for_planner_example(self):
symbol_table = TargetTable()
address_mapper = AddressMapper(JsonParser(symbol_table),
'*.BUILD.json')
rules = create_graph_rules(address_mapper,
symbol_table) + create_fs_rules()
fullgraph_str = self.create_full_graph(rules)
print('---diagnostic------')
print(fullgraph_str)
print('/---diagnostic------')
in_root_rules = False
in_all_rules = False
all_rules = []
root_rule_lines = []
for line in fullgraph_str.splitlines():
if line.startswith(' // root subject types:'):
pass
elif line.startswith(' // root entries'):
in_root_rules = True
elif line.startswith(' // internal entries'):
in_all_rules = True
elif in_all_rules:
all_rules.append(line)
elif in_root_rules:
root_rule_lines.append(line)
else:
pass
self.assertTrue(6 < len(all_rules))
self.assertTrue(12 < len(root_rule_lines)) # 2 lines per entry
def test_full_graph_for_planner_example(self):
address_mapper = AddressMapper(JsonParser(TARGET_TABLE), "*.BUILD.json")
rules = create_graph_rules(address_mapper) + create_fs_rules()
fullgraph_str = self.create_full_graph(rules)
print("---diagnostic------")
print(fullgraph_str)
print("/---diagnostic------")
in_root_rules = False
in_all_rules = False
all_rules = []
root_rule_lines = []
for line in fullgraph_str.splitlines():
if line.startswith(" // root subject types:"):
pass
elif line.startswith(" // root entries"):
in_root_rules = True
elif line.startswith(" // internal entries"):
in_all_rules = True
elif in_all_rules:
all_rules.append(line)
elif in_root_rules:
root_rule_lines.append(line)
else:
pass
self.assertTrue(6 < len(all_rules))
self.assertTrue(12 < len(root_rule_lines)) # 2 lines per entry
def setUp(self):
# Set up a scheduler that supports address mapping.
symbol_table = TargetTable()
address_mapper = AddressMapper(parser=JsonParser(symbol_table),
build_patterns=('*.BUILD.json',))
rules = create_fs_rules() + create_graph_rules(address_mapper, symbol_table)
# TODO handle updating the rule graph when passed unexpected root selectors.
# Adding the following task allows us to get around the fact that SelectDependencies
# requests are not currently supported.
rules.append(TaskRule(UnhydratedStructs,
[SelectDependencies(UnhydratedStruct,
BuildFileAddresses,
field_types=(Address,),
field='addresses')],
UnhydratedStructs))
project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'))
self.build_root = project_tree.build_root
self.scheduler = self.mk_scheduler(rules=rules, project_tree=project_tree)
self.a_b = Address.parse('a/b')
self.a_b_target = Target(name='b',
dependencies=['//d:e'],
configurations=['//a', Struct(embedded='yes')],
type_alias='target')
def test_empty(self):
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(JsonParser(TestTable()))
af = run_rule(parse_address_family, address_mapper, Dir('/dev/null'), {
(FilesContent, PathGlobs): lambda _: FilesContent([FileContent('/dev/null/BUILD', '')])
})
self.assertEquals(len(af.objects_by_name), 0)
def test_empty(self):
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(JsonParser(TestTable()))
af = run_rule(parse_address_family, address_mapper, Dir('/dev/null'), {
(Snapshot, PathGlobs): lambda _: Snapshot(DirectoryDigest('abc', 10), (File('/dev/null/BUILD'),)),
(FilesContent, DirectoryDigest): lambda _: FilesContent([FileContent('/dev/null/BUILD', b'')]),
})
self.assertEqual(len(af.objects_by_name), 0)
def setup_json_scheduler(build_root, native):
"""Return a build graph and scheduler configured for BLD.json files under the given build root.
:rtype :class:`pants.engine.scheduler.SchedulerSession`
"""
symbol_table = ExampleTable()
# Register "literal" subjects required for these rules.
address_mapper = AddressMapper(build_patterns=('BLD.json',),
parser=JsonParser(symbol_table))
work_dir = os_path_join(build_root, '.pants.d')
project_tree = FileSystemProjectTree(build_root)
rules = [
# Codegen
GenGoal.rule(),
gen_apache_java_thrift,
gen_apache_python_thrift,
gen_scrooge_scala_thrift,
gen_scrooge_java_thrift,
SingletonRule(Scrooge, Scrooge(Address.parse('src/scala/scrooge')))
] + [
# scala dependency inference
reify_scala_sources,
select_package_address,
calculate_package_search_path,
SingletonRule(SourceRoots, SourceRoots(('src/java','src/scala'))),
] + [
# Remote dependency resolution
ivy_resolve,
select_rev,
] + [
# Compilers
isolate_resources,
write_name_file,
javac,
scalac,
] + (
create_graph_rules(address_mapper, symbol_table)
) + (
create_fs_rules()
)
scheduler = Scheduler(native,
project_tree,
work_dir,
rules,
DEFAULT_EXECUTION_OPTIONS,
None,
None)
return scheduler.new_session()
def test_duplicated(self):
"""Test that matching the same Spec twice succeeds."""
address = SingleAddress('a', 'a')
address_mapper = AddressMapper(JsonParser(TestTable()))
snapshot = Snapshot(DirectoryDigest(str('xx'), 2), (Path('a/BUILD', File('a/BUILD')),))
address_family = AddressFamily('a', {'a': ('a/BUILD', 'this is an object!')})
bfas = run_rule(addresses_from_address_families, address_mapper, Specs([address, address]), {
(Snapshot, PathGlobs): lambda _: snapshot,
(AddressFamily, Dir): lambda _: address_family,
})
self.assertEquals(len(bfas.dependencies), 1)
self.assertEquals(bfas.dependencies[0].spec, 'a:a')
class AddressMapTest(unittest.TestCase):
_parser = JsonParser(SymbolTable({"thing": Thing}))
@contextmanager
def parse_address_map(self, json):
path = "/dev/null"
address_map = AddressMap.parse(path, json, self._parser)
self.assertEqual(path, address_map.path)
yield address_map
def test_parse(self) -> None:
with self.parse_address_map(
dedent(
"""
{
"type_alias": "thing",
"name": "one",
"age": 42
}
{
"type_alias": "thing",
"name": "two",
"age": 37
}
"""
)
) as address_map:
self.assertEqual(
{"one": Thing(name="one", age=42), "two": Thing(name="two", age=37)},
address_map.objects_by_name,
)
def test_not_serializable(self) -> None:
with self.assertRaises(UnaddressableObjectError):
with self.parse_address_map("{}"):
self.fail()
def test_not_named(self) -> None:
with self.assertRaises(UnaddressableObjectError):
with self.parse_address_map('{"type_alias": "thing"}'):
self.fail()
def test_duplicate_names(self) -> None:
with self.assertRaises(DuplicateNameError):
with self.parse_address_map(
'{"type_alias": "thing", "name": "one"}' '{"type_alias": "thing", "name": "one"}'
):
self.fail()
class AddressMapTest(unittest.TestCase):
_symbol_table = ThingTable()
_parser = JsonParser(_symbol_table)
@contextmanager
def parse_address_map(self, json):
path = '/dev/null'
address_map = AddressMap.parse(path, json, self._parser)
self.assertEqual(path, address_map.path)
yield address_map
def test_parse(self):
with self.parse_address_map(
dedent("""
{
"type_alias": "thing",
"name": "one",
"age": 42
}
{
"type_alias": "thing",
"name": "two",
"age": 37
}
""")) as address_map:
self.assertEqual(
{
'one': Thing(name='one', age=42),
'two': Thing(name='two', age=37)
}, address_map.objects_by_name)
def test_not_serializable(self):
with self.assertRaises(UnaddressableObjectError):
with self.parse_address_map('{}'):
self.fail()
def test_not_named(self):
with self.assertRaises(UnaddressableObjectError):
with self.parse_address_map('{"type_alias": "thing"}'):
self.fail()
def test_duplicate_names(self):
with self.assertRaises(DuplicateNameError):
with self.parse_address_map(
'{"type_alias": "thing", "name": "one"}'
'{"type_alias": "thing", "name": "one"}'):
self.fail()
def test_exclude_pattern_with_single_address(self):
"""Test that single address targets are filtered based on exclude patterns."""
spec = SingleAddress('root', 'not_me')
address_mapper = AddressMapper(JsonParser(TestTable()))
snapshot = Snapshot(DirectoryDigest('xx', 2),
(Path('root/BUILD', File('root/BUILD')),))
address_family = AddressFamily('root',
{
'not_me': ('root/BUILD', TargetAdaptor()),
}
)
targets = run_rule(
addresses_from_address_families, address_mapper, Specs([spec], exclude_patterns=tuple(['root.*'])),{
(Snapshot, PathGlobs): lambda _: snapshot,
(AddressFamily, Dir): lambda _: address_family,
})
self.assertEqual(len(targets.dependencies), 0)
def setUp(self) -> None:
# Set up a scheduler that supports address mapping.
address_mapper = AddressMapper(parser=JsonParser(TARGET_TABLE),
build_patterns=('*.BUILD.json',))
# We add the `unhydrated_structs` rule because it is otherwise not used in the core engine.
rules = [
unhydrated_structs
] + create_fs_rules() + create_graph_rules(address_mapper)
project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test'))
self.build_root = project_tree.build_root
self.scheduler = self.mk_scheduler(rules=rules, project_tree=project_tree)
self.a_b = Address.parse('a/b')
self.a_b_target = Target(address=self.a_b,
dependencies=['//a/d/e'],
configurations=['//a/d/e', Struct(embedded='yes')],
type_alias='target')
def test_exclude_pattern(self):
"""Test that targets are filtered based on exclude patterns."""
spec = SiblingAddresses('root')
address_mapper = AddressMapper(JsonParser(TestTable()))
snapshot = Snapshot(DirectoryDigest(text_type('xx'), 2),
(Path('root/BUILD', File('root/BUILD')), ))
address_family = AddressFamily(
'root', {
'exclude_me': ('root/BUILD', TargetAdaptor()),
'not_me': ('root/BUILD', TargetAdaptor()),
})
targets = run_rule(
addresses_from_address_families, address_mapper,
Specs([spec], exclude_patterns=tuple(['.exclude*'])), {
(Snapshot, PathGlobs): lambda _: snapshot,
(AddressFamily, Dir): lambda _: address_family,
})
self.assertEquals(len(targets.dependencies), 1)
self.assertEquals(targets.dependencies[0].spec, 'root:not_me')
def test_tag_filter(self):
"""Test that targets are filtered based on `tags`."""
spec = SiblingAddresses('root')
address_mapper = AddressMapper(JsonParser(TestTable()))
snapshot = Snapshot(DirectoryDigest(str('xx'), 2), (Path('root/BUILD', File('root/BUILD')),))
address_family = AddressFamily('root',
{'a': ('root/BUILD', TargetAdaptor()),
'b': ('root/BUILD', TargetAdaptor(tags={'integration'})),
'c': ('root/BUILD', TargetAdaptor(tags={'not_integration'}))
}
)
targets = run_rule(
addresses_from_address_families, address_mapper, Specs([spec], tags=['+integration']), {
(Snapshot, PathGlobs): lambda _: snapshot,
(AddressFamily, Dir): lambda _: address_family,
})
self.assertEquals(len(targets.dependencies), 1)
self.assertEquals(targets.dependencies[0].spec, 'root:b')
def test_empty(self):
"""Test that parsing an empty BUILD file results in an empty AddressFamily."""
address_mapper = AddressMapper(JsonParser(TEST_TABLE))
af = run_rule(
parse_address_family,
rule_args=[address_mapper, Dir('/dev/null')],
mock_gets=[
MockGet(
product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: Snapshot(Digest('abc', 10),
('/dev/null/BUILD', ), ()),
),
MockGet(
product_type=FilesContent,
subject_type=Digest,
mock=lambda _: FilesContent(
[FileContent(path='/dev/null/BUILD', content=b'')]),
),
],
)
self.assertEqual(len(af.objects_by_name), 0)
def create_json(self):
return self.create(build_patterns=('*.BUILD.json',), parser=JsonParser(TestTable()))
def _address_mapper(self): return AddressMapper(JsonParser(TestTable()))
def _address_mapper(self):
return AddressMapper(JsonParser(TEST_TABLE))
def create_json(self):
return self.create(build_patterns=('*.BUILD.json', ),
parser=JsonParser(TEST_TABLE))
def create_json(self) -> SchedulerSession:
return self.create(build_patterns=('*.BUILD.json', ),
parser=JsonParser(TEST_TABLE))