def setup_json_scheduler(build_root, native): """Return a build graph and scheduler configured for BLD.json files under the given build root. :rtype :class:`pants.engine.scheduler.LocalScheduler` """ symbol_table = ExampleTable() # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(build_patterns=('BLD.json',), parser=JsonParser(symbol_table)) work_dir = os_path_join(build_root, '.pants.d') project_tree = FileSystemProjectTree(build_root) goals = { 'compile': Classpath, # TODO: to allow for running resolve alone, should split out a distinct 'IvyReport' product. 'resolve': Classpath, 'list': BuildFileAddresses, GenGoal.name(): GenGoal, 'ls': Snapshot, 'cat': FilesContent, } tasks = [ # Codegen GenGoal.rule(), gen_apache_java_thrift, gen_apache_python_thrift, gen_scrooge_scala_thrift, gen_scrooge_java_thrift, SingletonRule(Scrooge, Scrooge(Address.parse('src/scala/scrooge'))) ] + [ # scala dependency inference reify_scala_sources, select_package_address, calculate_package_search_path, SingletonRule(SourceRoots, SourceRoots(('src/java','src/scala'))), ] + [ # Remote dependency resolution ivy_resolve, select_rev, ] + [ # Compilers isolate_resources, write_name_file, javac, scalac, ] + ( create_graph_rules(address_mapper, symbol_table) ) + ( create_fs_rules() ) return LocalScheduler(work_dir, goals, tasks, project_tree, native)
def test_full_graph_for_planner_example(self): symbol_table = TargetTable() address_mapper = AddressMapper(JsonParser(symbol_table), '*.BUILD.json') rules = create_graph_rules(address_mapper, symbol_table) + create_fs_rules() fullgraph_str = self.create_full_graph(rules) print('---diagnostic------') print(fullgraph_str) print('/---diagnostic------') in_root_rules = False in_all_rules = False all_rules = [] root_rule_lines = [] for line in fullgraph_str.splitlines(): if line.startswith(' // root subject types:'): pass elif line.startswith(' // root entries'): in_root_rules = True elif line.startswith(' // internal entries'): in_all_rules = True elif in_all_rules: all_rules.append(line) elif in_root_rules: root_rule_lines.append(line) else: pass self.assertTrue(6 < len(all_rules)) self.assertTrue(12 < len(root_rule_lines)) # 2 lines per entry
def test_full_graph_for_planner_example(self): address_mapper = AddressMapper(JsonParser(TARGET_TABLE), "*.BUILD.json") rules = create_graph_rules(address_mapper) + create_fs_rules() fullgraph_str = self.create_full_graph(rules) print("---diagnostic------") print(fullgraph_str) print("/---diagnostic------") in_root_rules = False in_all_rules = False all_rules = [] root_rule_lines = [] for line in fullgraph_str.splitlines(): if line.startswith(" // root subject types:"): pass elif line.startswith(" // root entries"): in_root_rules = True elif line.startswith(" // internal entries"): in_all_rules = True elif in_all_rules: all_rules.append(line) elif in_root_rules: root_rule_lines.append(line) else: pass self.assertTrue(6 < len(all_rules)) self.assertTrue(12 < len(root_rule_lines)) # 2 lines per entry
def setUp(self): # Set up a scheduler that supports address mapping. symbol_table = TargetTable() address_mapper = AddressMapper(parser=JsonParser(symbol_table), build_patterns=('*.BUILD.json',)) rules = create_fs_rules() + create_graph_rules(address_mapper, symbol_table) # TODO handle updating the rule graph when passed unexpected root selectors. # Adding the following task allows us to get around the fact that SelectDependencies # requests are not currently supported. rules.append(TaskRule(UnhydratedStructs, [SelectDependencies(UnhydratedStruct, BuildFileAddresses, field_types=(Address,), field='addresses')], UnhydratedStructs)) project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test')) self.build_root = project_tree.build_root self.scheduler = self.mk_scheduler(rules=rules, project_tree=project_tree) self.a_b = Address.parse('a/b') self.a_b_target = Target(name='b', dependencies=['//d:e'], configurations=['//a', Struct(embedded='yes')], type_alias='target')
def test_empty(self): """Test that parsing an empty BUILD file results in an empty AddressFamily.""" address_mapper = AddressMapper(JsonParser(TestTable())) af = run_rule(parse_address_family, address_mapper, Dir('/dev/null'), { (FilesContent, PathGlobs): lambda _: FilesContent([FileContent('/dev/null/BUILD', '')]) }) self.assertEquals(len(af.objects_by_name), 0)
def test_empty(self): """Test that parsing an empty BUILD file results in an empty AddressFamily.""" address_mapper = AddressMapper(JsonParser(TestTable())) af = run_rule(parse_address_family, address_mapper, Dir('/dev/null'), { (Snapshot, PathGlobs): lambda _: Snapshot(DirectoryDigest('abc', 10), (File('/dev/null/BUILD'),)), (FilesContent, DirectoryDigest): lambda _: FilesContent([FileContent('/dev/null/BUILD', b'')]), }) self.assertEqual(len(af.objects_by_name), 0)
def setup_json_scheduler(build_root, native): """Return a build graph and scheduler configured for BLD.json files under the given build root. :rtype :class:`pants.engine.scheduler.SchedulerSession` """ symbol_table = ExampleTable() # Register "literal" subjects required for these rules. address_mapper = AddressMapper(build_patterns=('BLD.json',), parser=JsonParser(symbol_table)) work_dir = os_path_join(build_root, '.pants.d') project_tree = FileSystemProjectTree(build_root) rules = [ # Codegen GenGoal.rule(), gen_apache_java_thrift, gen_apache_python_thrift, gen_scrooge_scala_thrift, gen_scrooge_java_thrift, SingletonRule(Scrooge, Scrooge(Address.parse('src/scala/scrooge'))) ] + [ # scala dependency inference reify_scala_sources, select_package_address, calculate_package_search_path, SingletonRule(SourceRoots, SourceRoots(('src/java','src/scala'))), ] + [ # Remote dependency resolution ivy_resolve, select_rev, ] + [ # Compilers isolate_resources, write_name_file, javac, scalac, ] + ( create_graph_rules(address_mapper, symbol_table) ) + ( create_fs_rules() ) scheduler = Scheduler(native, project_tree, work_dir, rules, DEFAULT_EXECUTION_OPTIONS, None, None) return scheduler.new_session()
def test_duplicated(self): """Test that matching the same Spec twice succeeds.""" address = SingleAddress('a', 'a') address_mapper = AddressMapper(JsonParser(TestTable())) snapshot = Snapshot(DirectoryDigest(str('xx'), 2), (Path('a/BUILD', File('a/BUILD')),)) address_family = AddressFamily('a', {'a': ('a/BUILD', 'this is an object!')}) bfas = run_rule(addresses_from_address_families, address_mapper, Specs([address, address]), { (Snapshot, PathGlobs): lambda _: snapshot, (AddressFamily, Dir): lambda _: address_family, }) self.assertEquals(len(bfas.dependencies), 1) self.assertEquals(bfas.dependencies[0].spec, 'a:a')
class AddressMapTest(unittest.TestCase): _parser = JsonParser(SymbolTable({"thing": Thing})) @contextmanager def parse_address_map(self, json): path = "/dev/null" address_map = AddressMap.parse(path, json, self._parser) self.assertEqual(path, address_map.path) yield address_map def test_parse(self) -> None: with self.parse_address_map( dedent( """ { "type_alias": "thing", "name": "one", "age": 42 } { "type_alias": "thing", "name": "two", "age": 37 } """ ) ) as address_map: self.assertEqual( {"one": Thing(name="one", age=42), "two": Thing(name="two", age=37)}, address_map.objects_by_name, ) def test_not_serializable(self) -> None: with self.assertRaises(UnaddressableObjectError): with self.parse_address_map("{}"): self.fail() def test_not_named(self) -> None: with self.assertRaises(UnaddressableObjectError): with self.parse_address_map('{"type_alias": "thing"}'): self.fail() def test_duplicate_names(self) -> None: with self.assertRaises(DuplicateNameError): with self.parse_address_map( '{"type_alias": "thing", "name": "one"}' '{"type_alias": "thing", "name": "one"}' ): self.fail()
class AddressMapTest(unittest.TestCase): _symbol_table = ThingTable() _parser = JsonParser(_symbol_table) @contextmanager def parse_address_map(self, json): path = '/dev/null' address_map = AddressMap.parse(path, json, self._parser) self.assertEqual(path, address_map.path) yield address_map def test_parse(self): with self.parse_address_map( dedent(""" { "type_alias": "thing", "name": "one", "age": 42 } { "type_alias": "thing", "name": "two", "age": 37 } """)) as address_map: self.assertEqual( { 'one': Thing(name='one', age=42), 'two': Thing(name='two', age=37) }, address_map.objects_by_name) def test_not_serializable(self): with self.assertRaises(UnaddressableObjectError): with self.parse_address_map('{}'): self.fail() def test_not_named(self): with self.assertRaises(UnaddressableObjectError): with self.parse_address_map('{"type_alias": "thing"}'): self.fail() def test_duplicate_names(self): with self.assertRaises(DuplicateNameError): with self.parse_address_map( '{"type_alias": "thing", "name": "one"}' '{"type_alias": "thing", "name": "one"}'): self.fail()
def test_exclude_pattern_with_single_address(self): """Test that single address targets are filtered based on exclude patterns.""" spec = SingleAddress('root', 'not_me') address_mapper = AddressMapper(JsonParser(TestTable())) snapshot = Snapshot(DirectoryDigest('xx', 2), (Path('root/BUILD', File('root/BUILD')),)) address_family = AddressFamily('root', { 'not_me': ('root/BUILD', TargetAdaptor()), } ) targets = run_rule( addresses_from_address_families, address_mapper, Specs([spec], exclude_patterns=tuple(['root.*'])),{ (Snapshot, PathGlobs): lambda _: snapshot, (AddressFamily, Dir): lambda _: address_family, }) self.assertEqual(len(targets.dependencies), 0)
def setUp(self) -> None: # Set up a scheduler that supports address mapping. address_mapper = AddressMapper(parser=JsonParser(TARGET_TABLE), build_patterns=('*.BUILD.json',)) # We add the `unhydrated_structs` rule because it is otherwise not used in the core engine. rules = [ unhydrated_structs ] + create_fs_rules() + create_graph_rules(address_mapper) project_tree = self.mk_fs_tree(os.path.join(os.path.dirname(__file__), 'examples/mapper_test')) self.build_root = project_tree.build_root self.scheduler = self.mk_scheduler(rules=rules, project_tree=project_tree) self.a_b = Address.parse('a/b') self.a_b_target = Target(address=self.a_b, dependencies=['//a/d/e'], configurations=['//a/d/e', Struct(embedded='yes')], type_alias='target')
def test_exclude_pattern(self): """Test that targets are filtered based on exclude patterns.""" spec = SiblingAddresses('root') address_mapper = AddressMapper(JsonParser(TestTable())) snapshot = Snapshot(DirectoryDigest(text_type('xx'), 2), (Path('root/BUILD', File('root/BUILD')), )) address_family = AddressFamily( 'root', { 'exclude_me': ('root/BUILD', TargetAdaptor()), 'not_me': ('root/BUILD', TargetAdaptor()), }) targets = run_rule( addresses_from_address_families, address_mapper, Specs([spec], exclude_patterns=tuple(['.exclude*'])), { (Snapshot, PathGlobs): lambda _: snapshot, (AddressFamily, Dir): lambda _: address_family, }) self.assertEquals(len(targets.dependencies), 1) self.assertEquals(targets.dependencies[0].spec, 'root:not_me')
def test_tag_filter(self): """Test that targets are filtered based on `tags`.""" spec = SiblingAddresses('root') address_mapper = AddressMapper(JsonParser(TestTable())) snapshot = Snapshot(DirectoryDigest(str('xx'), 2), (Path('root/BUILD', File('root/BUILD')),)) address_family = AddressFamily('root', {'a': ('root/BUILD', TargetAdaptor()), 'b': ('root/BUILD', TargetAdaptor(tags={'integration'})), 'c': ('root/BUILD', TargetAdaptor(tags={'not_integration'})) } ) targets = run_rule( addresses_from_address_families, address_mapper, Specs([spec], tags=['+integration']), { (Snapshot, PathGlobs): lambda _: snapshot, (AddressFamily, Dir): lambda _: address_family, }) self.assertEquals(len(targets.dependencies), 1) self.assertEquals(targets.dependencies[0].spec, 'root:b')
def test_empty(self): """Test that parsing an empty BUILD file results in an empty AddressFamily.""" address_mapper = AddressMapper(JsonParser(TEST_TABLE)) af = run_rule( parse_address_family, rule_args=[address_mapper, Dir('/dev/null')], mock_gets=[ MockGet( product_type=Snapshot, subject_type=PathGlobs, mock=lambda _: Snapshot(Digest('abc', 10), ('/dev/null/BUILD', ), ()), ), MockGet( product_type=FilesContent, subject_type=Digest, mock=lambda _: FilesContent( [FileContent(path='/dev/null/BUILD', content=b'')]), ), ], ) self.assertEqual(len(af.objects_by_name), 0)
def create_json(self): return self.create(build_patterns=('*.BUILD.json',), parser=JsonParser(TestTable()))
def _address_mapper(self): return AddressMapper(JsonParser(TestTable()))
def _address_mapper(self): return AddressMapper(JsonParser(TEST_TABLE))
def create_json(self): return self.create(build_patterns=('*.BUILD.json', ), parser=JsonParser(TEST_TABLE))
def create_json(self) -> SchedulerSession: return self.create(build_patterns=('*.BUILD.json', ), parser=JsonParser(TEST_TABLE))