def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root) self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse( 'src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse( 'src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse( 'src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse( '3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse( '3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps')
def _open_scheduler(*args, **kwargs): scheduler, storage, options, spec_roots, symbol_table_cls = setup(*args, **kwargs) engine = LocalSerialEngine(scheduler, storage) engine.start() try: yield scheduler, engine, symbol_table_cls, spec_roots maybe_launch_pantsd(options, scheduler) finally: logger.debug('Cache stats: {}'.format(engine._cache.get_stats())) engine.close()
def _open_graph(): scheduler, storage, spec_roots, symbol_table_cls = setup() # Populate the graph for the given request, and print the resulting Addresses. engine = LocalSerialEngine(scheduler, storage) engine.start() try: graph = ExpGraph(scheduler, engine, symbol_table_cls) addresses = tuple(graph.inject_specs_closure(spec_roots)) yield graph, addresses finally: print('Cache stats: {}'.format(engine._cache.get_stats()), file=sys.stderr) engine.close()
def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.graph, self.scheduler = setup_json_scheduler(build_root) self.engine = LocalSerialEngine(self.scheduler) self.guava = self.graph.resolve(Address.parse('3rdparty/jvm:guava')) self.thrift = self.graph.resolve(Address.parse('src/thrift/codegen/simple')) self.java = self.graph.resolve(Address.parse('src/java/codegen/simple')) self.java_multi = self.graph.resolve(Address.parse('src/java/multiple_classpath_entries')) self.unconfigured_thrift = self.graph.resolve(Address.parse('src/thrift/codegen/unconfigured')) self.resources = self.graph.resolve(Address.parse('src/resources/simple')) self.consumes_resources = self.graph.resolve(Address.parse('src/java/consumes_resources')) self.consumes_managed_thirdparty = self.graph.resolve(Address.parse('src/java/managed_thirdparty')) self.managed_guava = self.graph.resolve(Address.parse('3rdparty/jvm/managed:guava')) self.managed_hadoop = self.graph.resolve(Address.parse('3rdparty/jvm/managed:hadoop-common'))
def execute_request(self, scheduler, storage, product, *subjects): """Creates, runs, and returns an ExecutionRequest for the given product and subjects.""" request = scheduler.execution_request([product], subjects) res = LocalSerialEngine(scheduler, storage).execute(request) if res.error: raise res.error return request
def _populate(self, scheduler, address): """Perform an ExecutionRequest to parse the given Address into a Struct.""" request = scheduler.execution_request([self._product], [address]) LocalSerialEngine(scheduler, self.storage).reduce(request) root_entries = scheduler.root_entries(request).items() self.assertEquals(1, len(root_entries)) return root_entries[0]
def visualize_build_request(build_root, goals, subjects): scheduler, storage = setup_json_scheduler(build_root) execution_request = scheduler.build_request(goals, subjects) # NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it. engine = LocalSerialEngine(scheduler, storage) engine.start() try: engine.reduce(execution_request) visualize_execution_graph(scheduler, storage, execution_request) finally: engine.close()
def resolve(self, spec): request = BuildRequest(goals=[self._goal], subjects=[spec]) result = LocalSerialEngine(self.scheduler).execute(request) if result.error: raise result.error # Expect a single root. state, = result.root_products.values() if type(state) is Throw: raise state.exc return state.value
def resolve(self, spec): request = self.scheduler.execution_request([UnhydratedStruct], [spec]) result = LocalSerialEngine(self.scheduler, self.storage).execute(request) if result.error: raise result.error # Expect a single root. state, = result.root_products.values() if type(state) is Throw: raise state.exc return state.value
def setUp(self): build_root = os.path.join(os.path.dirname(__file__), "examples", "scheduler_inputs") self.graph, self.scheduler = setup_json_scheduler(build_root) self.engine = LocalSerialEngine(self.scheduler) self.guava = self.graph.resolve(Address.parse("3rdparty/jvm:guava")) self.thrift = self.graph.resolve(Address.parse("src/thrift/codegen/simple")) self.java = self.graph.resolve(Address.parse("src/java/codegen/simple")) self.java_multi = self.graph.resolve(Address.parse("src/java/multiple_classpath_entries")) self.unconfigured_thrift = self.graph.resolve(Address.parse("src/thrift/codegen/unconfigured")) self.resources = self.graph.resolve(Address.parse("src/resources/simple")) self.consumes_resources = self.graph.resolve(Address.parse("src/java/consumes_resources")) self.consumes_managed_thirdparty = self.graph.resolve(Address.parse("src/java/managed_thirdparty")) self.managed_guava = self.graph.resolve(Address.parse("3rdparty/jvm/managed:guava")) self.managed_hadoop = self.graph.resolve(Address.parse("3rdparty/jvm/managed:hadoop-common"))
def _open_scheduler(*args, **kwargs): scheduler, storage, options, spec_roots, symbol_table_cls = setup(*args, **kwargs) engine = LocalSerialEngine(scheduler, storage) engine.start() try: yield scheduler, engine, symbol_table_cls, spec_roots maybe_launch_pantsd(options, scheduler) finally: print('Cache stats: {}'.format(engine._cache.get_stats()), file=sys.stderr) engine.close()
def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.scheduler = setup_json_scheduler(build_root) self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common') self.inferred_deps = Address.parse('src/scala/inferred_deps')
def setUp(self): build_root = os.path.join(os.path.dirname(__file__), "examples", "scheduler_inputs") self.spec_parser = CmdLineSpecParser(build_root) self.scheduler, storage = setup_json_scheduler(build_root) self.storage = storage self.engine = LocalSerialEngine(self.scheduler, storage) self.guava = Address.parse("3rdparty/jvm:guava") self.thrift = Address.parse("src/thrift/codegen/simple") self.java = Address.parse("src/java/codegen/simple") self.java_simple = Address.parse("src/java/simple") self.java_multi = Address.parse("src/java/multiple_classpath_entries") self.no_variant_thrift = Address.parse("src/java/codegen/selector:conflict") self.unconfigured_thrift = Address.parse("src/thrift/codegen/unconfigured") self.resources = Address.parse("src/resources/simple") self.consumes_resources = Address.parse("src/java/consumes_resources") self.consumes_managed_thirdparty = Address.parse("src/java/managed_thirdparty") self.managed_guava = Address.parse("3rdparty/jvm/managed:guava") self.managed_hadoop = Address.parse("3rdparty/jvm/managed:hadoop-common") self.managed_resolve_latest = Address.parse("3rdparty/jvm/managed:latest-hadoop") self.inferred_deps = Address.parse("src/scala/inferred_deps")
class SchedulerTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.graph, self.scheduler = setup_json_scheduler(build_root) self.engine = LocalSerialEngine(self.scheduler) self.guava = self.graph.resolve(Address.parse('3rdparty/jvm:guava')) self.thrift = self.graph.resolve(Address.parse('src/thrift/codegen/simple')) self.java = self.graph.resolve(Address.parse('src/java/codegen/simple')) self.java_multi = self.graph.resolve(Address.parse('src/java/multiple_classpath_entries')) self.unconfigured_thrift = self.graph.resolve(Address.parse('src/thrift/codegen/unconfigured')) self.resources = self.graph.resolve(Address.parse('src/resources/simple')) self.consumes_resources = self.graph.resolve(Address.parse('src/java/consumes_resources')) self.consumes_managed_thirdparty = self.graph.resolve(Address.parse('src/java/managed_thirdparty')) self.managed_guava = self.graph.resolve(Address.parse('3rdparty/jvm/managed:guava')) self.managed_hadoop = self.graph.resolve(Address.parse('3rdparty/jvm/managed:hadoop-common')) def assert_product_for_subjects(self, walk, product, subjects, variants=None): variants = tuple(variants.items()) if variants else None self.assertEqual({SelectNode(subject, product, variants) for subject in subjects}, {node for (node, _), _ in walk if node.product == product and isinstance(node, SelectNode) and node.variants == variants}) def build_and_walk(self, build_request): """Build and then walk the given build_request, returning the walked graph as a list.""" result = self.engine.execute(build_request) self.assertIsNone(result.error) walk = list(self.scheduler.walk_product_graph()) for entry in walk: print('>>> {}'.format(entry)) return walk def assert_resolve_only(self, goals, root_specs, jars): build_request = BuildRequest(goals=goals, addressable_roots=[Address.parse(spec) for spec in root_specs]) walk = self.build_and_walk(build_request) # Expect a SelectNode for each of the Jar and Classpath, and a TaskNode and NativeNode. self.assertEqual(4 * len(jars), len(walk)) self.assert_product_for_subjects(walk, Jar, jars) self.assert_product_for_subjects(walk, Classpath, jars) def test_resolve(self): self.assert_resolve_only(goals=['resolve'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_compile_only_3rdparty(self): self.assert_resolve_only(goals=['compile'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_gen_noop(self): # TODO(John Sirois): Ask around - is this OK? # This is different than today. There is a gen'able target reachable from the java target, but # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's # handed is not thrift. build_request = BuildRequest(goals=['gen'], addressable_roots=[self.java.address]) walk = self.build_and_walk(build_request) self.assertEqual(2, len(walk)) self.assert_product_for_subjects(walk, JavaSources, [self.java]) def test_gen(self): build_request = BuildRequest(goals=['gen'], addressable_roots=[self.thrift.address]) walk = self.build_and_walk(build_request) # Root: expect JavaSources. root_entry = walk[0][0] self.assertEqual(SelectNode(self.thrift, JavaSources, None), root_entry[0]) self.assertIsInstance(root_entry[1], Return) # Expect an ApacheThriftJavaConfiguration to have been used. product = ApacheThriftJavaConfiguration self.assertEqual({NativeNode(self.thrift, product, None)}, {node for (node, _), _ in walk if node.product == product and isinstance(node, NativeNode)}) def test_codegen_simple(self): build_request = BuildRequest(goals=['compile'], addressable_roots=[self.java.address]) walk = self.build_and_walk(build_request) subjects = [self.guava, Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2'), Jar(org='commons-lang', name='commons-lang', rev='2.5'), self.graph.resolve(Address.parse('src/thrift:slf4j-api')), self.java, self.thrift] # Root: expect compilation via javac. self.assertEqual((SelectNode(self.java, Classpath, None), Return(Classpath(creator='javac'))), walk[0][0]) # Confirm that exactly the expected subjects got Classpaths. self.assert_product_for_subjects(walk, Classpath, subjects) def test_consumes_resources(self): build_request = BuildRequest(goals=['compile'], addressable_roots=[self.consumes_resources.address]) walk = self.build_and_walk(build_request) # Validate the root. self.assertEqual((SelectNode(self.consumes_resources, Classpath, None), Return(Classpath(creator='javac'))), walk[0][0]) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_product_for_subjects(walk, Classpath, subjects) def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = BuildRequest(goals=['compile'], addressable_roots=[self.consumes_managed_thirdparty.address]) walk = self.build_and_walk(build_request) # Validate the root. self.assertEqual((SelectNode(self.consumes_managed_thirdparty, Classpath, None), Return(Classpath(creator='javac'))), walk[0][0]) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_product_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty]) self.assert_product_for_subjects(walk, Classpath, managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0')}, {ret.value for (node, ret), _ in walk if node.product == Jar and isinstance(node, SelectNode)}) @pytest.mark.xfail(raises=ConflictingProducersError) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = BuildRequest(goals=['compile'], addressable_roots=[self.java_multi.address]) walk = self.build_and_walk(build_request) @pytest.mark.xfail(raises=PartiallyConsumedInputsError) def test_no_configured_thrift_planner(self): """Even though the BuildPropertiesPlanner is able to produce a Classpath, we still fail when a target with thrift sources doesn't have a thrift config. """ build_request = BuildRequest(goals=['compile'], addressable_roots=[self.unconfigured_thrift.address]) walk = self.build_and_walk(build_request)
class SchedulerTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), "examples", "scheduler_inputs") self.spec_parser = CmdLineSpecParser(build_root) self.scheduler, storage = setup_json_scheduler(build_root) self.storage = storage self.engine = LocalSerialEngine(self.scheduler, storage) self.guava = Address.parse("3rdparty/jvm:guava") self.thrift = Address.parse("src/thrift/codegen/simple") self.java = Address.parse("src/java/codegen/simple") self.java_simple = Address.parse("src/java/simple") self.java_multi = Address.parse("src/java/multiple_classpath_entries") self.no_variant_thrift = Address.parse("src/java/codegen/selector:conflict") self.unconfigured_thrift = Address.parse("src/thrift/codegen/unconfigured") self.resources = Address.parse("src/resources/simple") self.consumes_resources = Address.parse("src/java/consumes_resources") self.consumes_managed_thirdparty = Address.parse("src/java/managed_thirdparty") self.managed_guava = Address.parse("3rdparty/jvm/managed:guava") self.managed_hadoop = Address.parse("3rdparty/jvm/managed:hadoop-common") self.managed_resolve_latest = Address.parse("3rdparty/jvm/managed:latest-hadoop") self.inferred_deps = Address.parse("src/scala/inferred_deps") def assert_select_for_subjects(self, walk, product, subjects, variants=None, variant_key=None): node_type = SelectNode variants = tuple(variants.items()) if variants else None self.assertEqual( {node_type(subject, product, variants, variant_key) for subject in subjects}, { node for (node, _), _ in walk if node.product == product and isinstance(node, node_type) and node.variants == variants }, ) def build_and_walk(self, build_request, failures=False): """Build and then walk the given build_request, returning the walked graph as a list.""" predicate = (lambda _: True) if failures else None result = self.engine.execute(build_request) self.assertIsNone(result.error) return list(self.scheduler.product_graph.walk(build_request.roots, predicate=predicate)) def request(self, goals, *addresses): return self.request_specs(goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses]) def request_specs(self, goals, *specs): return self.scheduler.build_request(goals=goals, subjects=specs) def assert_resolve_only(self, goals, root_specs, jars): build_request = self.request(goals, *root_specs) walk = self.build_and_walk(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Jar, jars) self.assert_select_for_subjects(walk, Classpath, jars) def assert_root(self, walk, node, return_value): """Asserts that the first Node in a walk was a DependenciesNode with the single given result.""" ((root, root_state), dependencies) = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Return([return_value]), root_state) self.assertIn((node, Return(return_value)), dependencies) def assert_root_failed(self, walk, node, thrown_type): """Asserts that the first Node in a walk was a DependenciesNode with a Throw result.""" ((root, root_state), dependencies) = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Throw, type(root_state)) self.assertIn((node, thrown_type), [(k, type(v.exc)) for k, v in dependencies if type(v) is Throw]) def test_resolve(self): self.assert_resolve_only(goals=["resolve"], root_specs=["3rdparty/jvm:guava"], jars=[self.guava]) def test_compile_only_3rdparty(self): self.assert_resolve_only(goals=["compile"], root_specs=["3rdparty/jvm:guava"], jars=[self.guava]) def test_gen_noop(self): # TODO(John Sirois): Ask around - is this OK? # This is different than today. There is a gen'able target reachable from the java target, but # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's # handed is not thrift. build_request = self.request(["gen"], self.java) walk = self.build_and_walk(build_request) self.assert_select_for_subjects(walk, JavaSources, [self.java]) def test_gen(self): build_request = self.request(["gen"], self.thrift) walk = self.build_and_walk(build_request) # Root: expect the synthetic GenGoal product. self.assert_root( walk, SelectNode(self.thrift, GenGoal, None, None), GenGoal("non-empty input to satisfy the Goal constructor"), ) variants = {"thrift": "apache_java"} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, ThriftSources, [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects( walk, ApacheThriftJavaConfiguration, [self.thrift], variants=variants, variant_key="thrift" ) def test_codegen_simple(self): build_request = self.request(["compile"], self.java) walk = self.build_and_walk(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [self.guava, self.java, self.thrift] variant_subjects = [ Jar(org="org.apache.thrift", name="libthrift", rev="0.9.2", type_alias="jar"), Jar(org="commons-lang", name="commons-lang", rev="2.5", type_alias="jar"), Address.parse("src/thrift:slf4j-api"), ] # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. self.assert_root(walk, SelectNode(self.java, Classpath, None, None), Classpath(creator="javac")) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Classpath, subjects) self.assert_select_for_subjects(walk, Classpath, variant_subjects, variants={"thrift": "apache_java"}) def test_consumes_resources(self): build_request = self.request(["compile"], self.consumes_resources) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.consumes_resources, Classpath, None, None), Classpath(creator="javac")) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Classpath, subjects) def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request(["compile"], self.consumes_managed_thirdparty) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root( walk, SelectNode(self.consumes_managed_thirdparty, Classpath, None, None), Classpath(creator="javac") ) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Classpath, managed_jars, variants={"resolve": "latest-hadoop"}) # Confirm that the produced jars had the appropriate versions. self.assertEquals( {Jar("org.apache.hadoop", "hadoop-common", "2.7.0"), Jar("com.google.guava", "guava", "18.0")}, {ret.value for (node, ret), _ in walk if node.product == Jar and isinstance(node, SelectNode)}, ) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(["compile"], self.inferred_deps) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.inferred_deps, Classpath, None, None), Classpath(creator="scalac")) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Classpath, [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request(["compile"], self.java_multi) walk = self.build_and_walk(build_request, failures=True) # Validate that the root failed. self.assert_root_failed(walk, SelectNode(self.java_multi, Classpath, None, None), ConflictingProducersError) def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec("3rdparty/jvm::") build_request = self.request_specs(["list"], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0][0] root_value = root_state.value self.assertEqual(DependenciesNode(spec, Address, None, Addresses, None), root) self.assertEqual(list, type(root_value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root_value) self.assertIn(self.managed_guava, root_value) self.assertIn(self.managed_resolve_latest, root_value) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec("3rdparty/jvm:") build_request = self.request_specs(["list"], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0][0] root_value = root_state.value self.assertEqual(DependenciesNode(spec, Address, None, Addresses, None), root) self.assertEqual(list, type(root_value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root_value) # And that an subdirectory address is not. self.assertNotIn(self.managed_guava, root_value) def test_scheduler_visualize(self): spec = self.spec_parser.parse_spec("3rdparty/jvm:") build_request = self.request_specs(["list"], spec) self.build_and_walk(build_request) graphviz_output = "\n".join(self.scheduler.product_graph.visualize(build_request.roots)) with temporary_dir() as td: output_path = os.path.join(td, "output.dot") self.scheduler.visualize_graph_to_file(build_request.roots, output_path) with open(output_path, "rb") as fh: graphviz_disk_output = fh.read().strip() self.assertEqual(graphviz_output, graphviz_disk_output) self.assertIn("digraph", graphviz_output) self.assertIn(" -> ", graphviz_output)
class SchedulerTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.scheduler = setup_json_scheduler(build_root) self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common') self.inferred_deps = Address.parse('src/scala/inferred_deps') def assert_select_for_subjects(self, walk, product, subjects, variants=None, variant_key=None): node_type = SelectNode variants = tuple(variants.items()) if variants else None self.assertEqual({node_type(subject, product, variants, variant_key) for subject in subjects}, {node for (node, _), _ in walk if node.product == product and isinstance(node, node_type) and node.variants == variants}) def build_and_walk(self, build_request): """Build and then walk the given build_request, returning the walked graph as a list.""" result = self.engine.execute(build_request) self.assertIsNone(result.error) return list(self.scheduler.walk_product_graph()) def assert_resolve_only(self, goals, root_specs, jars): build_request = BuildRequest(goals=goals, addressable_roots=[Address.parse(spec) for spec in root_specs]) walk = self.build_and_walk(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Jar, jars) self.assert_select_for_subjects(walk, Classpath, jars) def test_resolve(self): self.assert_resolve_only(goals=['resolve'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_compile_only_3rdparty(self): self.assert_resolve_only(goals=['compile'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_gen_noop(self): # TODO(John Sirois): Ask around - is this OK? # This is different than today. There is a gen'able target reachable from the java target, but # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's # handed is not thrift. build_request = BuildRequest(goals=['gen'], addressable_roots=[self.java]) walk = self.build_and_walk(build_request) self.assert_select_for_subjects(walk, JavaSources, [self.java]) def test_gen(self): build_request = BuildRequest(goals=['gen'], addressable_roots=[self.thrift]) walk = self.build_and_walk(build_request) # Root: expect JavaSources. root_entry = walk[0][0] self.assertEqual(SelectNode(self.thrift, JavaSources, None, None), root_entry[0]) self.assertIsInstance(root_entry[1], Return) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, ApacheThriftJavaConfiguration, [self.thrift], variants={'thrift': 'apache_java'}, variant_key='thrift') def test_codegen_simple(self): build_request = BuildRequest(goals=['compile'], addressable_roots=[self.java]) walk = self.build_and_walk(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [ self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2'), Jar(org='commons-lang', name='commons-lang', rev='2.5'), Address.parse('src/thrift:slf4j-api')] # Root: expect compilation via javac. self.assertEqual((SelectNode(self.java, Classpath, None, None), Return(Classpath(creator='javac'))), walk[0][0]) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Classpath, subjects) self.assert_select_for_subjects(walk, Classpath, variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = BuildRequest(goals=['compile'], addressable_roots=[self.consumes_resources]) walk = self.build_and_walk(build_request) # Validate the root. self.assertEqual((SelectNode(self.consumes_resources, Classpath, None, None), Return(Classpath(creator='javac'))), walk[0][0]) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Classpath, subjects) def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = BuildRequest(goals=['compile'], addressable_roots=[self.consumes_managed_thirdparty]) walk = self.build_and_walk(build_request) # Validate the root. self.assertEqual((SelectNode(self.consumes_managed_thirdparty, Classpath, None, None), Return(Classpath(creator='javac'))), walk[0][0]) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Classpath, managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0')}, {ret.value for (node, ret), _ in walk if node.product == Jar and isinstance(node, SelectNode)}) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = BuildRequest(goals=['compile'], addressable_roots=[self.inferred_deps]) walk = self.build_and_walk(build_request) # Validate the root. self.assertEqual((SelectNode(self.inferred_deps, Classpath, None, None), Return(Classpath(creator='scalac'))), walk[0][0]) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Classpath, [self.inferred_deps, self.java_simple]) @pytest.mark.xfail(raises=ConflictingProducersError) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = BuildRequest(goals=['compile'], addressable_roots=[self.java_multi]) walk = self.build_and_walk(build_request) @pytest.mark.xfail(raises=PartiallyConsumedInputsError) def test_no_configured_thrift_planner(self): """Even though the BuildPropertiesPlanner is able to produce a Classpath, we still fail when a target with thrift sources doesn't have a thrift config. """ build_request = BuildRequest(goals=['compile'], addressable_roots=[self.unconfigured_thrift]) walk = self.build_and_walk(build_request)
class SchedulerTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root) self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse( 'src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse( 'src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse( 'src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse( '3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse( '3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps') def assert_select_for_subjects(self, walk, product, subjects, variants=None, variant_key=None): node_type = SelectNode variants = tuple(variants.items()) if variants else None self.assertEqual( { node_type(subject, product, variants, variant_key) for subject in subjects }, { node for (node, _), _ in walk if node.product == product and isinstance(node, node_type) and node.variants == variants }) def build_and_walk(self, build_request, failures=False): """Build and then walk the given build_request, returning the walked graph as a list.""" predicate = (lambda _: True) if failures else None result = self.engine.execute(build_request) self.assertIsNone(result.error) return list( self.scheduler.walk_product_graph(build_request, predicate=predicate)) def request(self, goals, *addresses): return self.request_specs( goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses]) def request_specs(self, goals, *specs): return BuildRequest(goals=goals, subjects=specs) def assert_resolve_only(self, goals, root_specs, jars): build_request = self.request(goals, *root_specs) walk = self.build_and_walk(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Jar, jars) self.assert_select_for_subjects(walk, Classpath, jars) def test_resolve(self): self.assert_resolve_only(goals=['resolve'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_compile_only_3rdparty(self): self.assert_resolve_only(goals=['compile'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_gen_noop(self): # TODO(John Sirois): Ask around - is this OK? # This is different than today. There is a gen'able target reachable from the java target, but # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's # handed is not thrift. build_request = self.request(['gen'], self.java) walk = self.build_and_walk(build_request) self.assert_select_for_subjects(walk, JavaSources, [self.java]) def test_gen(self): build_request = self.request(['gen'], self.thrift) walk = self.build_and_walk(build_request) # Root: expect the synthetic GenGoal product. root_entry = walk[0][0] self.assertEqual(SelectNode(self.thrift, GenGoal, None, None), root_entry[0]) self.assertIsInstance(root_entry[1], Return) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, ThriftSources, [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, ApacheThriftJavaConfiguration, [self.thrift], variants=variants, variant_key='thrift') def test_codegen_simple(self): build_request = self.request(['compile'], self.java) walk = self.build_and_walk(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2'), Jar(org='commons-lang', name='commons-lang', rev='2.5'), Address.parse('src/thrift:slf4j-api') ] # Root: expect compilation via javac. self.assertEqual( (SelectNode(self.java, Classpath, None, None), Return(Classpath(creator='javac'))), walk[0][0]) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Classpath, subjects) self.assert_select_for_subjects(walk, Classpath, variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = self.request(['compile'], self.consumes_resources) walk = self.build_and_walk(build_request) # Validate the root. self.assertEqual( (SelectNode(self.consumes_resources, Classpath, None, None), Return(Classpath(creator='javac'))), walk[0][0]) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Classpath, subjects) def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request(['compile'], self.consumes_managed_thirdparty) walk = self.build_and_walk(build_request) # Validate the root. self.assertEqual( (SelectNode(self.consumes_managed_thirdparty, Classpath, None, None), Return(Classpath(creator='javac'))), walk[0][0]) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Classpath, managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals( { Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0') }, { ret.value for (node, ret), _ in walk if node.product == Jar and isinstance(node, SelectNode) }) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) walk = self.build_and_walk(build_request) # Validate the root. self.assertEqual( (SelectNode(self.inferred_deps, Classpath, None, None), Return(Classpath(creator='scalac'))), walk[0][0]) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Classpath, [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request(['compile'], self.java_multi) walk = self.build_and_walk(build_request, failures=True) # Validate that the root failed. root_node, root_state = walk[0][0] self.assertEqual(SelectNode(self.java_multi, Classpath, None, None), root_node) self.assertEqual(Throw, type(root_state)) def test_no_variant_thrift(self): """No `thrift` variant is configured, and so no configuration is selected.""" build_request = self.request(['compile'], self.no_variant_thrift) with self.assertRaises(PartiallyConsumedInputsError): self.build_and_walk(build_request) def test_unconfigured_thrift(self): """The BuildPropertiesPlanner is able to produce a Classpath, but we should still fail. A target with ThriftSources doesn't have a thrift config: that input is partially consumed. """ build_request = self.request(['compile'], self.unconfigured_thrift) with self.assertRaises(PartiallyConsumedInputsError): self.build_and_walk(build_request) def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm::') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0][0] root_value = root_state.value self.assertEqual( DependenciesNode(spec, Address, None, Addresses, None), root) self.assertEqual(list, type(root_value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root_value) self.assertIn(self.managed_guava, root_value) self.assertIn(self.managed_resolve_latest, root_value) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0][0] root_value = root_state.value self.assertEqual( DependenciesNode(spec, Address, None, Addresses, None), root) self.assertEqual(list, type(root_value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root_value) # And that an subdirectory address is not. self.assertNotIn(self.managed_guava, root_value)
def visualize_build_request(build_root, build_request): scheduler = setup_json_scheduler(build_root) LocalSerialEngine(scheduler).reduce(build_request) visualize_execution_graph(scheduler)
def test_serial_engine_simple(self): engine = LocalSerialEngine(self.scheduler) self.assert_engine(engine)
class SchedulerTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root) self.subjects = self.scheduler._subjects self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps') def key(self, subject): return self.subjects.put(subject) def assert_select_for_subjects(self, walk, product, subjects, variants=None, variant_key=None): node_type = SelectNode variants = tuple(variants.items()) if variants else None self.assertEqual({node_type(self.key(subject), product, variants, variant_key) for subject in subjects}, {node for (node, _), _ in walk if node.product == product and isinstance(node, node_type) and node.variants == variants}) def build_and_walk(self, build_request, failures=False): """Build and then walk the given build_request, returning the walked graph as a list.""" predicate = (lambda _: True) if failures else None result = self.engine.execute(build_request) self.assertIsNone(result.error) return list(self.scheduler.product_graph.walk(build_request.roots, predicate=predicate)) def request(self, goals, *addresses): return self.request_specs(goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses]) def request_specs(self, goals, *specs): return self.scheduler.build_request(goals=goals, subjects=specs) def assert_resolve_only(self, goals, root_specs, jars): build_request = self.request(goals, *root_specs) walk = self.build_and_walk(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Jar, jars) self.assert_select_for_subjects(walk, Classpath, jars) def assert_root(self, walk, node, return_value): """Asserts that the first Node in a walk was a DependenciesNode with the single given result.""" ((root, root_state), dependencies) = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Return([return_value]), root_state) self.assertIn((node, Return(return_value)), dependencies) def assert_root_failed(self, walk, node, thrown_type): """Asserts that the first Node in a walk was a DependenciesNode with a Throw result.""" ((root, root_state), dependencies) = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Throw, type(root_state)) self.assertIn((node, thrown_type), [(k, type(v.exc)) for k, v in dependencies if type(v) is Throw]) def test_resolve(self): self.assert_resolve_only(goals=['resolve'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_compile_only_3rdparty(self): self.assert_resolve_only(goals=['compile'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_gen_noop(self): # TODO(John Sirois): Ask around - is this OK? # This is different than today. There is a gen'able target reachable from the java target, but # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's # handed is not thrift. build_request = self.request(['gen'], self.java) walk = self.build_and_walk(build_request) self.assert_select_for_subjects(walk, JavaSources, [self.java]) def test_gen(self): build_request = self.request(['gen'], self.thrift) walk = self.build_and_walk(build_request) # Root: expect the synthetic GenGoal product. self.assert_root(walk, SelectNode(self.key(self.thrift), GenGoal, None, None), GenGoal("non-empty input to satisfy the Goal constructor")) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, ThriftSources, [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, ApacheThriftJavaConfiguration, [self.thrift], variants=variants, variant_key='thrift') def test_codegen_simple(self): build_request = self.request(['compile'], self.java) walk = self.build_and_walk(build_request) # TODO: Utter insanity. Pickle only encodes a unique object (ie, `if A is B`) a single # time on the wire. Because the copy of this object that we're comparing to is coming # from a file, the string will be encoded twice. Thus, to match (with pickle) we need # to ensure that `(cl1 is not cl2)` here. See: # https://github.com/pantsbuild/pants/issues/2969 cl1 = 'commons-lang' cl2 = 'commons' + '-lang' # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [ self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'), Jar(org=cl1, name=cl2, rev='2.5', type_alias='jar'), Address.parse('src/thrift:slf4j-api')] # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. self.assert_root(walk, SelectNode(self.key(self.java), Classpath, None, None), Classpath(creator='javac')) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Classpath, subjects) self.assert_select_for_subjects(walk, Classpath, variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = self.request(['compile'], self.consumes_resources) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.key(self.consumes_resources), Classpath, None, None), Classpath(creator='javac')) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Classpath, subjects) def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request(['compile'], self.consumes_managed_thirdparty) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.key(self.consumes_managed_thirdparty), Classpath, None, None), Classpath(creator='javac')) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Classpath, managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0')}, {ret.value for (node, ret), _ in walk if node.product == Jar and isinstance(node, SelectNode)}) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.key(self.inferred_deps), Classpath, None, None), Classpath(creator='scalac')) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Classpath, [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request(['compile'], self.java_multi) walk = self.build_and_walk(build_request, failures=True) # Validate that the root failed. self.assert_root_failed(walk, SelectNode(self.key(self.java_multi), Classpath, None, None), ConflictingProducersError) def test_no_variant_thrift(self): """No `thrift` variant is configured, and so no configuration is selected.""" build_request = self.request(['compile'], self.no_variant_thrift) with self.assertRaises(PartiallyConsumedInputsError): self.build_and_walk(build_request) def test_unconfigured_thrift(self): """The BuildPropertiesPlanner is able to produce a Classpath, but we should still fail. A target with ThriftSources doesn't have a thrift config: that input is partially consumed. """ build_request = self.request(['compile'], self.unconfigured_thrift) with self.assertRaises(PartiallyConsumedInputsError): self.build_and_walk(build_request) def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm::') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0][0] root_value = root_state.value self.assertEqual(DependenciesNode(self.key(spec), Address, None, Addresses, None), root) self.assertEqual(list, type(root_value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root_value) self.assertIn(self.managed_guava, root_value) self.assertIn(self.managed_resolve_latest, root_value) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0][0] root_value = root_state.value self.assertEqual(DependenciesNode(self.key(spec), Address, None, Addresses, None), root) self.assertEqual(list, type(root_value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root_value) # And that an subdirectory address is not. self.assertNotIn(self.managed_guava, root_value)
def _populate(self, scheduler, address): """Make a BuildRequest to parse the given Address into a Struct.""" spec = self._cmd_line_spec_parser.parse_spec(str(address)) request = BuildRequest(goals=[self._goal], subjects=[spec]) LocalSerialEngine(scheduler).reduce(request) return self._select(address)
def _populate(self, scheduler, address): """Make a BuildRequest to parse the given Address into a Struct.""" request = BuildRequest(goals=[self._goal], addressable_roots=[address]) LocalSerialEngine(scheduler).reduce(request) return self._select(address)
def test_serial_engine_simple(self): engine = LocalSerialEngine(self.scheduler, self.storage, self.cache) self.assert_engine(engine)
def visualize_build_request(build_root, build_request): scheduler = setup_json_scheduler(build_root) # NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it. LocalSerialEngine(scheduler).reduce(build_request) visualize_execution_graph(scheduler, build_request) scheduler.validate()
class SchedulerTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), "examples", "scheduler_inputs") self.graph, self.scheduler = setup_json_scheduler(build_root) self.engine = LocalSerialEngine(self.scheduler) self.guava = self.graph.resolve(Address.parse("3rdparty/jvm:guava")) self.thrift = self.graph.resolve(Address.parse("src/thrift/codegen/simple")) self.java = self.graph.resolve(Address.parse("src/java/codegen/simple")) self.java_multi = self.graph.resolve(Address.parse("src/java/multiple_classpath_entries")) self.unconfigured_thrift = self.graph.resolve(Address.parse("src/thrift/codegen/unconfigured")) self.resources = self.graph.resolve(Address.parse("src/resources/simple")) self.consumes_resources = self.graph.resolve(Address.parse("src/java/consumes_resources")) self.consumes_managed_thirdparty = self.graph.resolve(Address.parse("src/java/managed_thirdparty")) self.managed_guava = self.graph.resolve(Address.parse("3rdparty/jvm/managed:guava")) self.managed_hadoop = self.graph.resolve(Address.parse("3rdparty/jvm/managed:hadoop-common")) def assert_product_for_subjects(self, walk, product, subjects, variants=None): variants = tuple(variants.items()) if variants else None self.assertEqual( {SelectNode(subject, product, variants) for subject in subjects}, { node for (node, _), _ in walk if node.product == product and isinstance(node, SelectNode) and node.variants == variants }, ) def build_and_walk(self, build_request): """Build and then walk the given build_request, returning the walked graph as a list.""" result = self.engine.execute(build_request) self.assertIsNone(result.error) walk = list(self.scheduler.walk_product_graph()) for entry in walk: print(">>> {}".format(entry)) return walk def assert_resolve_only(self, goals, root_specs, jars): build_request = BuildRequest(goals=goals, addressable_roots=[Address.parse(spec) for spec in root_specs]) walk = self.build_and_walk(build_request) # Expect a SelectNode for each of the Jar and Classpath, and a TaskNode and NativeNode. self.assertEqual(4 * len(jars), len(walk)) self.assert_product_for_subjects(walk, Jar, jars) self.assert_product_for_subjects(walk, Classpath, jars) def test_resolve(self): self.assert_resolve_only(goals=["resolve"], root_specs=["3rdparty/jvm:guava"], jars=[self.guava]) def test_compile_only_3rdparty(self): self.assert_resolve_only(goals=["compile"], root_specs=["3rdparty/jvm:guava"], jars=[self.guava]) def test_gen_noop(self): # TODO(John Sirois): Ask around - is this OK? # This is different than today. There is a gen'able target reachable from the java target, but # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's # handed is not thrift. build_request = BuildRequest(goals=["gen"], addressable_roots=[self.java.address]) walk = self.build_and_walk(build_request) self.assertEqual(2, len(walk)) self.assert_product_for_subjects(walk, JavaSources, [self.java]) def test_gen(self): build_request = BuildRequest(goals=["gen"], addressable_roots=[self.thrift.address]) walk = self.build_and_walk(build_request) # Root: expect JavaSources. root_entry = walk[0][0] self.assertEqual(SelectNode(self.thrift, JavaSources, None), root_entry[0]) self.assertIsInstance(root_entry[1], Return) # Expect an ApacheThriftJavaConfiguration to have been used. product = ApacheThriftJavaConfiguration self.assertEqual( {NativeNode(self.thrift, product, None)}, {node for (node, _), _ in walk if node.product == product and isinstance(node, NativeNode)}, ) def test_codegen_simple(self): build_request = BuildRequest(goals=["compile"], addressable_roots=[self.java.address]) walk = self.build_and_walk(build_request) subjects = [ self.guava, Jar(org="org.apache.thrift", name="libthrift", rev="0.9.2"), Jar(org="commons-lang", name="commons-lang", rev="2.5"), self.graph.resolve(Address.parse("src/thrift:slf4j-api")), self.java, self.thrift, ] # Root: expect compilation via javac. self.assertEqual((SelectNode(self.java, Classpath, None), Return(Classpath(creator="javac"))), walk[0][0]) # Confirm that exactly the expected subjects got Classpaths. self.assert_product_for_subjects(walk, Classpath, subjects) def test_consumes_resources(self): build_request = BuildRequest(goals=["compile"], addressable_roots=[self.consumes_resources.address]) walk = self.build_and_walk(build_request) # Validate the root. self.assertEqual( (SelectNode(self.consumes_resources, Classpath, None), Return(Classpath(creator="javac"))), walk[0][0] ) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_product_for_subjects(walk, Classpath, subjects) def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = BuildRequest(goals=["compile"], addressable_roots=[self.consumes_managed_thirdparty.address]) walk = self.build_and_walk(build_request) # Validate the root. self.assertEqual( (SelectNode(self.consumes_managed_thirdparty, Classpath, None), Return(Classpath(creator="javac"))), walk[0][0], ) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_product_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty]) self.assert_product_for_subjects(walk, Classpath, managed_jars, variants={"resolve": "latest-hadoop"}) # Confirm that the produced jars had the appropriate versions. self.assertEquals( {Jar("org.apache.hadoop", "hadoop-common", "2.7.0"), Jar("com.google.guava", "guava", "18.0")}, {ret.value for (node, ret), _ in walk if node.product == Jar and isinstance(node, SelectNode)}, ) @pytest.mark.xfail(raises=ConflictingProducersError) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = BuildRequest(goals=["compile"], addressable_roots=[self.java_multi.address]) walk = self.build_and_walk(build_request) @pytest.mark.xfail(raises=PartiallyConsumedInputsError) def test_no_configured_thrift_planner(self): """Even though the BuildPropertiesPlanner is able to produce a Classpath, we still fail when a target with thrift sources doesn't have a thrift config. """ build_request = BuildRequest(goals=["compile"], addressable_roots=[self.unconfigured_thrift.address]) walk = self.build_and_walk(build_request)
def test_serial_engine_fail_slow(self): engine = LocalSerialEngine(self.scheduler) self.assert_engine_fail_slow(engine)