def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples',
'scheduler_inputs')
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler = setup_json_scheduler(build_root)
self.engine = LocalSerialEngine(self.scheduler)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.no_variant_thrift = Address.parse(
'src/java/codegen/selector:conflict')
self.unconfigured_thrift = Address.parse(
'src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse(
'src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse(
'3rdparty/jvm/managed:hadoop-common')
self.managed_resolve_latest = Address.parse(
'3rdparty/jvm/managed:latest-hadoop')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def _open_scheduler(*args, **kwargs):
scheduler, storage, options, spec_roots, symbol_table_cls = setup(*args, **kwargs)
engine = LocalSerialEngine(scheduler, storage)
engine.start()
try:
yield scheduler, engine, symbol_table_cls, spec_roots
maybe_launch_pantsd(options, scheduler)
finally:
logger.debug('Cache stats: {}'.format(engine._cache.get_stats()))
engine.close()
def _open_graph():
scheduler, storage, spec_roots, symbol_table_cls = setup()
# Populate the graph for the given request, and print the resulting Addresses.
engine = LocalSerialEngine(scheduler, storage)
engine.start()
try:
graph = ExpGraph(scheduler, engine, symbol_table_cls)
addresses = tuple(graph.inject_specs_closure(spec_roots))
yield graph, addresses
finally:
print('Cache stats: {}'.format(engine._cache.get_stats()), file=sys.stderr)
engine.close()
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
self.graph, self.scheduler = setup_json_scheduler(build_root)
self.engine = LocalSerialEngine(self.scheduler)
self.guava = self.graph.resolve(Address.parse('3rdparty/jvm:guava'))
self.thrift = self.graph.resolve(Address.parse('src/thrift/codegen/simple'))
self.java = self.graph.resolve(Address.parse('src/java/codegen/simple'))
self.java_multi = self.graph.resolve(Address.parse('src/java/multiple_classpath_entries'))
self.unconfigured_thrift = self.graph.resolve(Address.parse('src/thrift/codegen/unconfigured'))
self.resources = self.graph.resolve(Address.parse('src/resources/simple'))
self.consumes_resources = self.graph.resolve(Address.parse('src/java/consumes_resources'))
self.consumes_managed_thirdparty = self.graph.resolve(Address.parse('src/java/managed_thirdparty'))
self.managed_guava = self.graph.resolve(Address.parse('3rdparty/jvm/managed:guava'))
self.managed_hadoop = self.graph.resolve(Address.parse('3rdparty/jvm/managed:hadoop-common'))
def execute_request(self, scheduler, storage, product, *subjects):
"""Creates, runs, and returns an ExecutionRequest for the given product and subjects."""
request = scheduler.execution_request([product], subjects)
res = LocalSerialEngine(scheduler, storage).execute(request)
if res.error:
raise res.error
return request
def _populate(self, scheduler, address):
"""Perform an ExecutionRequest to parse the given Address into a Struct."""
request = scheduler.execution_request([self._product], [address])
LocalSerialEngine(scheduler, self.storage).reduce(request)
root_entries = scheduler.root_entries(request).items()
self.assertEquals(1, len(root_entries))
return root_entries[0]
def visualize_build_request(build_root, goals, subjects):
scheduler, storage = setup_json_scheduler(build_root)
execution_request = scheduler.build_request(goals, subjects)
# NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it.
engine = LocalSerialEngine(scheduler, storage)
engine.start()
try:
engine.reduce(execution_request)
visualize_execution_graph(scheduler, storage, execution_request)
finally:
engine.close()
def resolve(self, spec):
request = BuildRequest(goals=[self._goal], subjects=[spec])
result = LocalSerialEngine(self.scheduler).execute(request)
if result.error:
raise result.error
# Expect a single root.
state, = result.root_products.values()
if type(state) is Throw:
raise state.exc
return state.value
def resolve(self, spec):
request = self.scheduler.execution_request([UnhydratedStruct], [spec])
result = LocalSerialEngine(self.scheduler, self.storage).execute(request)
if result.error:
raise result.error
# Expect a single root.
state, = result.root_products.values()
if type(state) is Throw:
raise state.exc
return state.value
def visualize_build_request(build_root, goals, subjects):
scheduler, storage = setup_json_scheduler(build_root)
execution_request = scheduler.build_request(goals, subjects)
# NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it.
engine = LocalSerialEngine(scheduler, storage)
engine.start()
try:
engine.reduce(execution_request)
visualize_execution_graph(scheduler, storage, execution_request)
finally:
engine.close()
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), "examples", "scheduler_inputs")
self.graph, self.scheduler = setup_json_scheduler(build_root)
self.engine = LocalSerialEngine(self.scheduler)
self.guava = self.graph.resolve(Address.parse("3rdparty/jvm:guava"))
self.thrift = self.graph.resolve(Address.parse("src/thrift/codegen/simple"))
self.java = self.graph.resolve(Address.parse("src/java/codegen/simple"))
self.java_multi = self.graph.resolve(Address.parse("src/java/multiple_classpath_entries"))
self.unconfigured_thrift = self.graph.resolve(Address.parse("src/thrift/codegen/unconfigured"))
self.resources = self.graph.resolve(Address.parse("src/resources/simple"))
self.consumes_resources = self.graph.resolve(Address.parse("src/java/consumes_resources"))
self.consumes_managed_thirdparty = self.graph.resolve(Address.parse("src/java/managed_thirdparty"))
self.managed_guava = self.graph.resolve(Address.parse("3rdparty/jvm/managed:guava"))
self.managed_hadoop = self.graph.resolve(Address.parse("3rdparty/jvm/managed:hadoop-common"))
def _open_scheduler(*args, **kwargs):
scheduler, storage, options, spec_roots, symbol_table_cls = setup(*args, **kwargs)
engine = LocalSerialEngine(scheduler, storage)
engine.start()
try:
yield scheduler, engine, symbol_table_cls, spec_roots
maybe_launch_pantsd(options, scheduler)
finally:
print('Cache stats: {}'.format(engine._cache.get_stats()), file=sys.stderr)
engine.close()
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
self.scheduler = setup_json_scheduler(build_root)
self.engine = LocalSerialEngine(self.scheduler)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), "examples", "scheduler_inputs")
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler, storage = setup_json_scheduler(build_root)
self.storage = storage
self.engine = LocalSerialEngine(self.scheduler, storage)
self.guava = Address.parse("3rdparty/jvm:guava")
self.thrift = Address.parse("src/thrift/codegen/simple")
self.java = Address.parse("src/java/codegen/simple")
self.java_simple = Address.parse("src/java/simple")
self.java_multi = Address.parse("src/java/multiple_classpath_entries")
self.no_variant_thrift = Address.parse("src/java/codegen/selector:conflict")
self.unconfigured_thrift = Address.parse("src/thrift/codegen/unconfigured")
self.resources = Address.parse("src/resources/simple")
self.consumes_resources = Address.parse("src/java/consumes_resources")
self.consumes_managed_thirdparty = Address.parse("src/java/managed_thirdparty")
self.managed_guava = Address.parse("3rdparty/jvm/managed:guava")
self.managed_hadoop = Address.parse("3rdparty/jvm/managed:hadoop-common")
self.managed_resolve_latest = Address.parse("3rdparty/jvm/managed:latest-hadoop")
self.inferred_deps = Address.parse("src/scala/inferred_deps")
class SchedulerTest(unittest.TestCase):
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
self.graph, self.scheduler = setup_json_scheduler(build_root)
self.engine = LocalSerialEngine(self.scheduler)
self.guava = self.graph.resolve(Address.parse('3rdparty/jvm:guava'))
self.thrift = self.graph.resolve(Address.parse('src/thrift/codegen/simple'))
self.java = self.graph.resolve(Address.parse('src/java/codegen/simple'))
self.java_multi = self.graph.resolve(Address.parse('src/java/multiple_classpath_entries'))
self.unconfigured_thrift = self.graph.resolve(Address.parse('src/thrift/codegen/unconfigured'))
self.resources = self.graph.resolve(Address.parse('src/resources/simple'))
self.consumes_resources = self.graph.resolve(Address.parse('src/java/consumes_resources'))
self.consumes_managed_thirdparty = self.graph.resolve(Address.parse('src/java/managed_thirdparty'))
self.managed_guava = self.graph.resolve(Address.parse('3rdparty/jvm/managed:guava'))
self.managed_hadoop = self.graph.resolve(Address.parse('3rdparty/jvm/managed:hadoop-common'))
def assert_product_for_subjects(self, walk, product, subjects, variants=None):
variants = tuple(variants.items()) if variants else None
self.assertEqual({SelectNode(subject, product, variants) for subject in subjects},
{node for (node, _), _ in walk
if node.product == product and isinstance(node, SelectNode) and node.variants == variants})
def build_and_walk(self, build_request):
"""Build and then walk the given build_request, returning the walked graph as a list."""
result = self.engine.execute(build_request)
self.assertIsNone(result.error)
walk = list(self.scheduler.walk_product_graph())
for entry in walk:
print('>>> {}'.format(entry))
return walk
def assert_resolve_only(self, goals, root_specs, jars):
build_request = BuildRequest(goals=goals,
addressable_roots=[Address.parse(spec) for spec in root_specs])
walk = self.build_and_walk(build_request)
# Expect a SelectNode for each of the Jar and Classpath, and a TaskNode and NativeNode.
self.assertEqual(4 * len(jars), len(walk))
self.assert_product_for_subjects(walk, Jar, jars)
self.assert_product_for_subjects(walk, Classpath, jars)
def test_resolve(self):
self.assert_resolve_only(goals=['resolve'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_compile_only_3rdparty(self):
self.assert_resolve_only(goals=['compile'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = BuildRequest(goals=['gen'], addressable_roots=[self.java.address])
walk = self.build_and_walk(build_request)
self.assertEqual(2, len(walk))
self.assert_product_for_subjects(walk, JavaSources, [self.java])
def test_gen(self):
build_request = BuildRequest(goals=['gen'], addressable_roots=[self.thrift.address])
walk = self.build_and_walk(build_request)
# Root: expect JavaSources.
root_entry = walk[0][0]
self.assertEqual(SelectNode(self.thrift, JavaSources, None), root_entry[0])
self.assertIsInstance(root_entry[1], Return)
# Expect an ApacheThriftJavaConfiguration to have been used.
product = ApacheThriftJavaConfiguration
self.assertEqual({NativeNode(self.thrift, product, None)},
{node for (node, _), _ in walk
if node.product == product and isinstance(node, NativeNode)})
def test_codegen_simple(self):
build_request = BuildRequest(goals=['compile'], addressable_roots=[self.java.address])
walk = self.build_and_walk(build_request)
subjects = [self.guava,
Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2'),
Jar(org='commons-lang', name='commons-lang', rev='2.5'),
self.graph.resolve(Address.parse('src/thrift:slf4j-api')),
self.java,
self.thrift]
# Root: expect compilation via javac.
self.assertEqual((SelectNode(self.java, Classpath, None), Return(Classpath(creator='javac'))),
walk[0][0])
# Confirm that exactly the expected subjects got Classpaths.
self.assert_product_for_subjects(walk, Classpath, subjects)
def test_consumes_resources(self):
build_request = BuildRequest(goals=['compile'], addressable_roots=[self.consumes_resources.address])
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual((SelectNode(self.consumes_resources, Classpath, None),
Return(Classpath(creator='javac'))),
walk[0][0])
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources,
self.consumes_resources,
self.guava]
self.assert_product_for_subjects(walk, Classpath, subjects)
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = BuildRequest(goals=['compile'],
addressable_roots=[self.consumes_managed_thirdparty.address])
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual((SelectNode(self.consumes_managed_thirdparty, Classpath, None),
Return(Classpath(creator='javac'))),
walk[0][0])
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava,
self.managed_hadoop]
self.assert_product_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty])
self.assert_product_for_subjects(walk, Classpath, managed_jars, variants={'resolve': 'latest-hadoop'})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
Jar('com.google.guava', 'guava', '18.0')},
{ret.value for (node, ret), _ in walk
if node.product == Jar and isinstance(node, SelectNode)})
@pytest.mark.xfail(raises=ConflictingProducersError)
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = BuildRequest(goals=['compile'], addressable_roots=[self.java_multi.address])
walk = self.build_and_walk(build_request)
@pytest.mark.xfail(raises=PartiallyConsumedInputsError)
def test_no_configured_thrift_planner(self):
"""Even though the BuildPropertiesPlanner is able to produce a Classpath,
we still fail when a target with thrift sources doesn't have a thrift config.
"""
build_request = BuildRequest(goals=['compile'],
addressable_roots=[self.unconfigured_thrift.address])
walk = self.build_and_walk(build_request)
class SchedulerTest(unittest.TestCase):
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), "examples", "scheduler_inputs")
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler, storage = setup_json_scheduler(build_root)
self.storage = storage
self.engine = LocalSerialEngine(self.scheduler, storage)
self.guava = Address.parse("3rdparty/jvm:guava")
self.thrift = Address.parse("src/thrift/codegen/simple")
self.java = Address.parse("src/java/codegen/simple")
self.java_simple = Address.parse("src/java/simple")
self.java_multi = Address.parse("src/java/multiple_classpath_entries")
self.no_variant_thrift = Address.parse("src/java/codegen/selector:conflict")
self.unconfigured_thrift = Address.parse("src/thrift/codegen/unconfigured")
self.resources = Address.parse("src/resources/simple")
self.consumes_resources = Address.parse("src/java/consumes_resources")
self.consumes_managed_thirdparty = Address.parse("src/java/managed_thirdparty")
self.managed_guava = Address.parse("3rdparty/jvm/managed:guava")
self.managed_hadoop = Address.parse("3rdparty/jvm/managed:hadoop-common")
self.managed_resolve_latest = Address.parse("3rdparty/jvm/managed:latest-hadoop")
self.inferred_deps = Address.parse("src/scala/inferred_deps")
def assert_select_for_subjects(self, walk, product, subjects, variants=None, variant_key=None):
node_type = SelectNode
variants = tuple(variants.items()) if variants else None
self.assertEqual(
{node_type(subject, product, variants, variant_key) for subject in subjects},
{
node
for (node, _), _ in walk
if node.product == product and isinstance(node, node_type) and node.variants == variants
},
)
def build_and_walk(self, build_request, failures=False):
"""Build and then walk the given build_request, returning the walked graph as a list."""
predicate = (lambda _: True) if failures else None
result = self.engine.execute(build_request)
self.assertIsNone(result.error)
return list(self.scheduler.product_graph.walk(build_request.roots, predicate=predicate))
def request(self, goals, *addresses):
return self.request_specs(goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses])
def request_specs(self, goals, *specs):
return self.scheduler.build_request(goals=goals, subjects=specs)
def assert_resolve_only(self, goals, root_specs, jars):
build_request = self.request(goals, *root_specs)
walk = self.build_and_walk(build_request)
# Expect a SelectNode for each of the Jar/Classpath.
self.assert_select_for_subjects(walk, Jar, jars)
self.assert_select_for_subjects(walk, Classpath, jars)
def assert_root(self, walk, node, return_value):
"""Asserts that the first Node in a walk was a DependenciesNode with the single given result."""
((root, root_state), dependencies) = walk[0]
self.assertEquals(type(root), DependenciesNode)
self.assertEquals(Return([return_value]), root_state)
self.assertIn((node, Return(return_value)), dependencies)
def assert_root_failed(self, walk, node, thrown_type):
"""Asserts that the first Node in a walk was a DependenciesNode with a Throw result."""
((root, root_state), dependencies) = walk[0]
self.assertEquals(type(root), DependenciesNode)
self.assertEquals(Throw, type(root_state))
self.assertIn((node, thrown_type), [(k, type(v.exc)) for k, v in dependencies if type(v) is Throw])
def test_resolve(self):
self.assert_resolve_only(goals=["resolve"], root_specs=["3rdparty/jvm:guava"], jars=[self.guava])
def test_compile_only_3rdparty(self):
self.assert_resolve_only(goals=["compile"], root_specs=["3rdparty/jvm:guava"], jars=[self.guava])
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = self.request(["gen"], self.java)
walk = self.build_and_walk(build_request)
self.assert_select_for_subjects(walk, JavaSources, [self.java])
def test_gen(self):
build_request = self.request(["gen"], self.thrift)
walk = self.build_and_walk(build_request)
# Root: expect the synthetic GenGoal product.
self.assert_root(
walk,
SelectNode(self.thrift, GenGoal, None, None),
GenGoal("non-empty input to satisfy the Goal constructor"),
)
variants = {"thrift": "apache_java"}
# Expect ThriftSources to have been selected.
self.assert_select_for_subjects(walk, ThriftSources, [self.thrift], variants=variants)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(
walk, ApacheThriftJavaConfiguration, [self.thrift], variants=variants, variant_key="thrift"
)
def test_codegen_simple(self):
build_request = self.request(["compile"], self.java)
walk = self.build_and_walk(build_request)
# The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
subjects = [self.guava, self.java, self.thrift]
variant_subjects = [
Jar(org="org.apache.thrift", name="libthrift", rev="0.9.2", type_alias="jar"),
Jar(org="commons-lang", name="commons-lang", rev="2.5", type_alias="jar"),
Address.parse("src/thrift:slf4j-api"),
]
# Root: expect a DependenciesNode depending on a SelectNode with compilation via javac.
self.assert_root(walk, SelectNode(self.java, Classpath, None, None), Classpath(creator="javac"))
# Confirm that exactly the expected subjects got Classpaths.
self.assert_select_for_subjects(walk, Classpath, subjects)
self.assert_select_for_subjects(walk, Classpath, variant_subjects, variants={"thrift": "apache_java"})
def test_consumes_resources(self):
build_request = self.request(["compile"], self.consumes_resources)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(walk, SelectNode(self.consumes_resources, Classpath, None, None), Classpath(creator="javac"))
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources, self.consumes_resources, self.guava]
self.assert_select_for_subjects(walk, Classpath, subjects)
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = self.request(["compile"], self.consumes_managed_thirdparty)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(
walk, SelectNode(self.consumes_managed_thirdparty, Classpath, None, None), Classpath(creator="javac")
)
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava, self.managed_hadoop]
self.assert_select_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty])
self.assert_select_for_subjects(walk, Classpath, managed_jars, variants={"resolve": "latest-hadoop"})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals(
{Jar("org.apache.hadoop", "hadoop-common", "2.7.0"), Jar("com.google.guava", "guava", "18.0")},
{ret.value for (node, ret), _ in walk if node.product == Jar and isinstance(node, SelectNode)},
)
def test_dependency_inference(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request(["compile"], self.inferred_deps)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(walk, SelectNode(self.inferred_deps, Classpath, None, None), Classpath(creator="scalac"))
# Confirm that we requested a classpath for the root and inferred targets.
self.assert_select_for_subjects(walk, Classpath, [self.inferred_deps, self.java_simple])
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = self.request(["compile"], self.java_multi)
walk = self.build_and_walk(build_request, failures=True)
# Validate that the root failed.
self.assert_root_failed(walk, SelectNode(self.java_multi, Classpath, None, None), ConflictingProducersError)
def test_descendant_specs(self):
"""Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec("3rdparty/jvm::")
build_request = self.request_specs(["list"], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0][0]
root_value = root_state.value
self.assertEqual(DependenciesNode(spec, Address, None, Addresses, None), root)
self.assertEqual(list, type(root_value))
# Confirm that a few expected addresses are in the list.
self.assertIn(self.guava, root_value)
self.assertIn(self.managed_guava, root_value)
self.assertIn(self.managed_resolve_latest, root_value)
def test_sibling_specs(self):
"""Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec("3rdparty/jvm:")
build_request = self.request_specs(["list"], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0][0]
root_value = root_state.value
self.assertEqual(DependenciesNode(spec, Address, None, Addresses, None), root)
self.assertEqual(list, type(root_value))
# Confirm that an expected address is in the list.
self.assertIn(self.guava, root_value)
# And that an subdirectory address is not.
self.assertNotIn(self.managed_guava, root_value)
def test_scheduler_visualize(self):
spec = self.spec_parser.parse_spec("3rdparty/jvm:")
build_request = self.request_specs(["list"], spec)
self.build_and_walk(build_request)
graphviz_output = "\n".join(self.scheduler.product_graph.visualize(build_request.roots))
with temporary_dir() as td:
output_path = os.path.join(td, "output.dot")
self.scheduler.visualize_graph_to_file(build_request.roots, output_path)
with open(output_path, "rb") as fh:
graphviz_disk_output = fh.read().strip()
self.assertEqual(graphviz_output, graphviz_disk_output)
self.assertIn("digraph", graphviz_output)
self.assertIn(" -> ", graphviz_output)
class SchedulerTest(unittest.TestCase):
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
self.scheduler = setup_json_scheduler(build_root)
self.engine = LocalSerialEngine(self.scheduler)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def assert_select_for_subjects(self, walk, product, subjects, variants=None, variant_key=None):
node_type = SelectNode
variants = tuple(variants.items()) if variants else None
self.assertEqual({node_type(subject, product, variants, variant_key) for subject in subjects},
{node for (node, _), _ in walk
if node.product == product and isinstance(node, node_type) and node.variants == variants})
def build_and_walk(self, build_request):
"""Build and then walk the given build_request, returning the walked graph as a list."""
result = self.engine.execute(build_request)
self.assertIsNone(result.error)
return list(self.scheduler.walk_product_graph())
def assert_resolve_only(self, goals, root_specs, jars):
build_request = BuildRequest(goals=goals,
addressable_roots=[Address.parse(spec) for spec in root_specs])
walk = self.build_and_walk(build_request)
# Expect a SelectNode for each of the Jar/Classpath.
self.assert_select_for_subjects(walk, Jar, jars)
self.assert_select_for_subjects(walk, Classpath, jars)
def test_resolve(self):
self.assert_resolve_only(goals=['resolve'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_compile_only_3rdparty(self):
self.assert_resolve_only(goals=['compile'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = BuildRequest(goals=['gen'], addressable_roots=[self.java])
walk = self.build_and_walk(build_request)
self.assert_select_for_subjects(walk, JavaSources, [self.java])
def test_gen(self):
build_request = BuildRequest(goals=['gen'], addressable_roots=[self.thrift])
walk = self.build_and_walk(build_request)
# Root: expect JavaSources.
root_entry = walk[0][0]
self.assertEqual(SelectNode(self.thrift, JavaSources, None, None), root_entry[0])
self.assertIsInstance(root_entry[1], Return)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(walk, ApacheThriftJavaConfiguration, [self.thrift],
variants={'thrift': 'apache_java'}, variant_key='thrift')
def test_codegen_simple(self):
build_request = BuildRequest(goals=['compile'], addressable_roots=[self.java])
walk = self.build_and_walk(build_request)
# The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
subjects = [
self.guava,
self.java,
self.thrift]
variant_subjects = [
Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2'),
Jar(org='commons-lang', name='commons-lang', rev='2.5'),
Address.parse('src/thrift:slf4j-api')]
# Root: expect compilation via javac.
self.assertEqual((SelectNode(self.java, Classpath, None, None), Return(Classpath(creator='javac'))),
walk[0][0])
# Confirm that exactly the expected subjects got Classpaths.
self.assert_select_for_subjects(walk, Classpath, subjects)
self.assert_select_for_subjects(walk, Classpath, variant_subjects,
variants={'thrift': 'apache_java'})
def test_consumes_resources(self):
build_request = BuildRequest(goals=['compile'], addressable_roots=[self.consumes_resources])
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual((SelectNode(self.consumes_resources, Classpath, None, None),
Return(Classpath(creator='javac'))),
walk[0][0])
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources,
self.consumes_resources,
self.guava]
self.assert_select_for_subjects(walk, Classpath, subjects)
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = BuildRequest(goals=['compile'],
addressable_roots=[self.consumes_managed_thirdparty])
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual((SelectNode(self.consumes_managed_thirdparty, Classpath, None, None),
Return(Classpath(creator='javac'))),
walk[0][0])
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava,
self.managed_hadoop]
self.assert_select_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty])
self.assert_select_for_subjects(walk, Classpath, managed_jars, variants={'resolve': 'latest-hadoop'})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
Jar('com.google.guava', 'guava', '18.0')},
{ret.value for (node, ret), _ in walk
if node.product == Jar and isinstance(node, SelectNode)})
def test_dependency_inference(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = BuildRequest(goals=['compile'],
addressable_roots=[self.inferred_deps])
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual((SelectNode(self.inferred_deps, Classpath, None, None),
Return(Classpath(creator='scalac'))),
walk[0][0])
# Confirm that we requested a classpath for the root and inferred targets.
self.assert_select_for_subjects(walk, Classpath, [self.inferred_deps, self.java_simple])
@pytest.mark.xfail(raises=ConflictingProducersError)
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = BuildRequest(goals=['compile'], addressable_roots=[self.java_multi])
walk = self.build_and_walk(build_request)
@pytest.mark.xfail(raises=PartiallyConsumedInputsError)
def test_no_configured_thrift_planner(self):
"""Even though the BuildPropertiesPlanner is able to produce a Classpath,
we still fail when a target with thrift sources doesn't have a thrift config.
"""
build_request = BuildRequest(goals=['compile'],
addressable_roots=[self.unconfigured_thrift])
walk = self.build_and_walk(build_request)
class SchedulerTest(unittest.TestCase):
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples',
'scheduler_inputs')
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler = setup_json_scheduler(build_root)
self.engine = LocalSerialEngine(self.scheduler)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.no_variant_thrift = Address.parse(
'src/java/codegen/selector:conflict')
self.unconfigured_thrift = Address.parse(
'src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse(
'src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse(
'3rdparty/jvm/managed:hadoop-common')
self.managed_resolve_latest = Address.parse(
'3rdparty/jvm/managed:latest-hadoop')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def assert_select_for_subjects(self,
walk,
product,
subjects,
variants=None,
variant_key=None):
node_type = SelectNode
variants = tuple(variants.items()) if variants else None
self.assertEqual(
{
node_type(subject, product, variants, variant_key)
for subject in subjects
}, {
node
for (node, _), _ in walk if node.product == product
and isinstance(node, node_type) and node.variants == variants
})
def build_and_walk(self, build_request, failures=False):
"""Build and then walk the given build_request, returning the walked graph as a list."""
predicate = (lambda _: True) if failures else None
result = self.engine.execute(build_request)
self.assertIsNone(result.error)
return list(
self.scheduler.walk_product_graph(build_request,
predicate=predicate))
def request(self, goals, *addresses):
return self.request_specs(
goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses])
def request_specs(self, goals, *specs):
return BuildRequest(goals=goals, subjects=specs)
def assert_resolve_only(self, goals, root_specs, jars):
build_request = self.request(goals, *root_specs)
walk = self.build_and_walk(build_request)
# Expect a SelectNode for each of the Jar/Classpath.
self.assert_select_for_subjects(walk, Jar, jars)
self.assert_select_for_subjects(walk, Classpath, jars)
def test_resolve(self):
self.assert_resolve_only(goals=['resolve'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_compile_only_3rdparty(self):
self.assert_resolve_only(goals=['compile'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = self.request(['gen'], self.java)
walk = self.build_and_walk(build_request)
self.assert_select_for_subjects(walk, JavaSources, [self.java])
def test_gen(self):
build_request = self.request(['gen'], self.thrift)
walk = self.build_and_walk(build_request)
# Root: expect the synthetic GenGoal product.
root_entry = walk[0][0]
self.assertEqual(SelectNode(self.thrift, GenGoal, None, None),
root_entry[0])
self.assertIsInstance(root_entry[1], Return)
variants = {'thrift': 'apache_java'}
# Expect ThriftSources to have been selected.
self.assert_select_for_subjects(walk,
ThriftSources, [self.thrift],
variants=variants)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(walk,
ApacheThriftJavaConfiguration,
[self.thrift],
variants=variants,
variant_key='thrift')
def test_codegen_simple(self):
build_request = self.request(['compile'], self.java)
walk = self.build_and_walk(build_request)
# The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
subjects = [self.guava, self.java, self.thrift]
variant_subjects = [
Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2'),
Jar(org='commons-lang', name='commons-lang', rev='2.5'),
Address.parse('src/thrift:slf4j-api')
]
# Root: expect compilation via javac.
self.assertEqual(
(SelectNode(self.java, Classpath, None,
None), Return(Classpath(creator='javac'))), walk[0][0])
# Confirm that exactly the expected subjects got Classpaths.
self.assert_select_for_subjects(walk, Classpath, subjects)
self.assert_select_for_subjects(walk,
Classpath,
variant_subjects,
variants={'thrift': 'apache_java'})
def test_consumes_resources(self):
build_request = self.request(['compile'], self.consumes_resources)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual(
(SelectNode(self.consumes_resources, Classpath, None,
None), Return(Classpath(creator='javac'))), walk[0][0])
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources, self.consumes_resources, self.guava]
self.assert_select_for_subjects(walk, Classpath, subjects)
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = self.request(['compile'],
self.consumes_managed_thirdparty)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual(
(SelectNode(self.consumes_managed_thirdparty, Classpath, None,
None), Return(Classpath(creator='javac'))), walk[0][0])
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava, self.managed_hadoop]
self.assert_select_for_subjects(walk, Classpath,
[self.consumes_managed_thirdparty])
self.assert_select_for_subjects(walk,
Classpath,
managed_jars,
variants={'resolve': 'latest-hadoop'})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals(
{
Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
Jar('com.google.guava', 'guava', '18.0')
}, {
ret.value
for (node, ret), _ in walk
if node.product == Jar and isinstance(node, SelectNode)
})
def test_dependency_inference(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request(['compile'], self.inferred_deps)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual(
(SelectNode(self.inferred_deps, Classpath, None, None),
Return(Classpath(creator='scalac'))), walk[0][0])
# Confirm that we requested a classpath for the root and inferred targets.
self.assert_select_for_subjects(walk, Classpath,
[self.inferred_deps, self.java_simple])
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = self.request(['compile'], self.java_multi)
walk = self.build_and_walk(build_request, failures=True)
# Validate that the root failed.
root_node, root_state = walk[0][0]
self.assertEqual(SelectNode(self.java_multi, Classpath, None, None),
root_node)
self.assertEqual(Throw, type(root_state))
def test_no_variant_thrift(self):
"""No `thrift` variant is configured, and so no configuration is selected."""
build_request = self.request(['compile'], self.no_variant_thrift)
with self.assertRaises(PartiallyConsumedInputsError):
self.build_and_walk(build_request)
def test_unconfigured_thrift(self):
"""The BuildPropertiesPlanner is able to produce a Classpath, but we should still fail.
A target with ThriftSources doesn't have a thrift config: that input is partially consumed.
"""
build_request = self.request(['compile'], self.unconfigured_thrift)
with self.assertRaises(PartiallyConsumedInputsError):
self.build_and_walk(build_request)
def test_descendant_specs(self):
"""Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm::')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0][0]
root_value = root_state.value
self.assertEqual(
DependenciesNode(spec, Address, None, Addresses, None), root)
self.assertEqual(list, type(root_value))
# Confirm that a few expected addresses are in the list.
self.assertIn(self.guava, root_value)
self.assertIn(self.managed_guava, root_value)
self.assertIn(self.managed_resolve_latest, root_value)
def test_sibling_specs(self):
"""Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm:')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0][0]
root_value = root_state.value
self.assertEqual(
DependenciesNode(spec, Address, None, Addresses, None), root)
self.assertEqual(list, type(root_value))
# Confirm that an expected address is in the list.
self.assertIn(self.guava, root_value)
# And that an subdirectory address is not.
self.assertNotIn(self.managed_guava, root_value)
def visualize_build_request(build_root, build_request): scheduler = setup_json_scheduler(build_root) LocalSerialEngine(scheduler).reduce(build_request) visualize_execution_graph(scheduler)
def test_serial_engine_simple(self): engine = LocalSerialEngine(self.scheduler) self.assert_engine(engine)
class SchedulerTest(unittest.TestCase):
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs')
self.spec_parser = CmdLineSpecParser(build_root)
self.scheduler = setup_json_scheduler(build_root)
self.subjects = self.scheduler._subjects
self.engine = LocalSerialEngine(self.scheduler)
self.guava = Address.parse('3rdparty/jvm:guava')
self.thrift = Address.parse('src/thrift/codegen/simple')
self.java = Address.parse('src/java/codegen/simple')
self.java_simple = Address.parse('src/java/simple')
self.java_multi = Address.parse('src/java/multiple_classpath_entries')
self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict')
self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured')
self.resources = Address.parse('src/resources/simple')
self.consumes_resources = Address.parse('src/java/consumes_resources')
self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty')
self.managed_guava = Address.parse('3rdparty/jvm/managed:guava')
self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common')
self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop')
self.inferred_deps = Address.parse('src/scala/inferred_deps')
def key(self, subject):
return self.subjects.put(subject)
def assert_select_for_subjects(self, walk, product, subjects, variants=None, variant_key=None):
node_type = SelectNode
variants = tuple(variants.items()) if variants else None
self.assertEqual({node_type(self.key(subject), product, variants, variant_key) for subject in subjects},
{node for (node, _), _ in walk
if node.product == product and isinstance(node, node_type) and node.variants == variants})
def build_and_walk(self, build_request, failures=False):
"""Build and then walk the given build_request, returning the walked graph as a list."""
predicate = (lambda _: True) if failures else None
result = self.engine.execute(build_request)
self.assertIsNone(result.error)
return list(self.scheduler.product_graph.walk(build_request.roots, predicate=predicate))
def request(self, goals, *addresses):
return self.request_specs(goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses])
def request_specs(self, goals, *specs):
return self.scheduler.build_request(goals=goals, subjects=specs)
def assert_resolve_only(self, goals, root_specs, jars):
build_request = self.request(goals, *root_specs)
walk = self.build_and_walk(build_request)
# Expect a SelectNode for each of the Jar/Classpath.
self.assert_select_for_subjects(walk, Jar, jars)
self.assert_select_for_subjects(walk, Classpath, jars)
def assert_root(self, walk, node, return_value):
"""Asserts that the first Node in a walk was a DependenciesNode with the single given result."""
((root, root_state), dependencies) = walk[0]
self.assertEquals(type(root), DependenciesNode)
self.assertEquals(Return([return_value]), root_state)
self.assertIn((node, Return(return_value)), dependencies)
def assert_root_failed(self, walk, node, thrown_type):
"""Asserts that the first Node in a walk was a DependenciesNode with a Throw result."""
((root, root_state), dependencies) = walk[0]
self.assertEquals(type(root), DependenciesNode)
self.assertEquals(Throw, type(root_state))
self.assertIn((node, thrown_type), [(k, type(v.exc)) for k, v in dependencies
if type(v) is Throw])
def test_resolve(self):
self.assert_resolve_only(goals=['resolve'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_compile_only_3rdparty(self):
self.assert_resolve_only(goals=['compile'],
root_specs=['3rdparty/jvm:guava'],
jars=[self.guava])
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = self.request(['gen'], self.java)
walk = self.build_and_walk(build_request)
self.assert_select_for_subjects(walk, JavaSources, [self.java])
def test_gen(self):
build_request = self.request(['gen'], self.thrift)
walk = self.build_and_walk(build_request)
# Root: expect the synthetic GenGoal product.
self.assert_root(walk,
SelectNode(self.key(self.thrift), GenGoal, None, None),
GenGoal("non-empty input to satisfy the Goal constructor"))
variants = {'thrift': 'apache_java'}
# Expect ThriftSources to have been selected.
self.assert_select_for_subjects(walk, ThriftSources, [self.thrift], variants=variants)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(walk, ApacheThriftJavaConfiguration, [self.thrift],
variants=variants, variant_key='thrift')
def test_codegen_simple(self):
build_request = self.request(['compile'], self.java)
walk = self.build_and_walk(build_request)
# TODO: Utter insanity. Pickle only encodes a unique object (ie, `if A is B`) a single
# time on the wire. Because the copy of this object that we're comparing to is coming
# from a file, the string will be encoded twice. Thus, to match (with pickle) we need
# to ensure that `(cl1 is not cl2)` here. See:
# https://github.com/pantsbuild/pants/issues/2969
cl1 = 'commons-lang'
cl2 = 'commons' + '-lang'
# The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
subjects = [
self.guava,
self.java,
self.thrift]
variant_subjects = [
Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'),
Jar(org=cl1, name=cl2, rev='2.5', type_alias='jar'),
Address.parse('src/thrift:slf4j-api')]
# Root: expect a DependenciesNode depending on a SelectNode with compilation via javac.
self.assert_root(walk,
SelectNode(self.key(self.java), Classpath, None, None),
Classpath(creator='javac'))
# Confirm that exactly the expected subjects got Classpaths.
self.assert_select_for_subjects(walk, Classpath, subjects)
self.assert_select_for_subjects(walk, Classpath, variant_subjects,
variants={'thrift': 'apache_java'})
def test_consumes_resources(self):
build_request = self.request(['compile'], self.consumes_resources)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(walk,
SelectNode(self.key(self.consumes_resources), Classpath, None, None),
Classpath(creator='javac'))
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources,
self.consumes_resources,
self.guava]
self.assert_select_for_subjects(walk, Classpath, subjects)
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = self.request(['compile'], self.consumes_managed_thirdparty)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(walk,
SelectNode(self.key(self.consumes_managed_thirdparty), Classpath, None, None),
Classpath(creator='javac'))
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava,
self.managed_hadoop]
self.assert_select_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty])
self.assert_select_for_subjects(walk, Classpath, managed_jars, variants={'resolve': 'latest-hadoop'})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
Jar('com.google.guava', 'guava', '18.0')},
{ret.value for (node, ret), _ in walk
if node.product == Jar and isinstance(node, SelectNode)})
def test_dependency_inference(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = self.request(['compile'], self.inferred_deps)
walk = self.build_and_walk(build_request)
# Validate the root.
self.assert_root(walk,
SelectNode(self.key(self.inferred_deps), Classpath, None, None),
Classpath(creator='scalac'))
# Confirm that we requested a classpath for the root and inferred targets.
self.assert_select_for_subjects(walk, Classpath, [self.inferred_deps, self.java_simple])
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = self.request(['compile'], self.java_multi)
walk = self.build_and_walk(build_request, failures=True)
# Validate that the root failed.
self.assert_root_failed(walk,
SelectNode(self.key(self.java_multi), Classpath, None, None),
ConflictingProducersError)
def test_no_variant_thrift(self):
"""No `thrift` variant is configured, and so no configuration is selected."""
build_request = self.request(['compile'], self.no_variant_thrift)
with self.assertRaises(PartiallyConsumedInputsError):
self.build_and_walk(build_request)
def test_unconfigured_thrift(self):
"""The BuildPropertiesPlanner is able to produce a Classpath, but we should still fail.
A target with ThriftSources doesn't have a thrift config: that input is partially consumed.
"""
build_request = self.request(['compile'], self.unconfigured_thrift)
with self.assertRaises(PartiallyConsumedInputsError):
self.build_and_walk(build_request)
def test_descendant_specs(self):
"""Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm::')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0][0]
root_value = root_state.value
self.assertEqual(DependenciesNode(self.key(spec), Address, None, Addresses, None), root)
self.assertEqual(list, type(root_value))
# Confirm that a few expected addresses are in the list.
self.assertIn(self.guava, root_value)
self.assertIn(self.managed_guava, root_value)
self.assertIn(self.managed_resolve_latest, root_value)
def test_sibling_specs(self):
"""Test that sibling Addresses are parsed in the 3rdparty/jvm directory."""
spec = self.spec_parser.parse_spec('3rdparty/jvm:')
build_request = self.request_specs(['list'], spec)
walk = self.build_and_walk(build_request)
# Validate the root.
root, root_state = walk[0][0]
root_value = root_state.value
self.assertEqual(DependenciesNode(self.key(spec), Address, None, Addresses, None), root)
self.assertEqual(list, type(root_value))
# Confirm that an expected address is in the list.
self.assertIn(self.guava, root_value)
# And that an subdirectory address is not.
self.assertNotIn(self.managed_guava, root_value)
def _populate(self, scheduler, address):
"""Make a BuildRequest to parse the given Address into a Struct."""
spec = self._cmd_line_spec_parser.parse_spec(str(address))
request = BuildRequest(goals=[self._goal], subjects=[spec])
LocalSerialEngine(scheduler).reduce(request)
return self._select(address)
def _populate(self, scheduler, address):
"""Make a BuildRequest to parse the given Address into a Struct."""
request = BuildRequest(goals=[self._goal], addressable_roots=[address])
LocalSerialEngine(scheduler).reduce(request)
return self._select(address)
def test_serial_engine_simple(self): engine = LocalSerialEngine(self.scheduler, self.storage, self.cache) self.assert_engine(engine)
def visualize_build_request(build_root, build_request):
scheduler = setup_json_scheduler(build_root)
# NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it.
LocalSerialEngine(scheduler).reduce(build_request)
visualize_execution_graph(scheduler, build_request)
scheduler.validate()
class SchedulerTest(unittest.TestCase):
def setUp(self):
build_root = os.path.join(os.path.dirname(__file__), "examples", "scheduler_inputs")
self.graph, self.scheduler = setup_json_scheduler(build_root)
self.engine = LocalSerialEngine(self.scheduler)
self.guava = self.graph.resolve(Address.parse("3rdparty/jvm:guava"))
self.thrift = self.graph.resolve(Address.parse("src/thrift/codegen/simple"))
self.java = self.graph.resolve(Address.parse("src/java/codegen/simple"))
self.java_multi = self.graph.resolve(Address.parse("src/java/multiple_classpath_entries"))
self.unconfigured_thrift = self.graph.resolve(Address.parse("src/thrift/codegen/unconfigured"))
self.resources = self.graph.resolve(Address.parse("src/resources/simple"))
self.consumes_resources = self.graph.resolve(Address.parse("src/java/consumes_resources"))
self.consumes_managed_thirdparty = self.graph.resolve(Address.parse("src/java/managed_thirdparty"))
self.managed_guava = self.graph.resolve(Address.parse("3rdparty/jvm/managed:guava"))
self.managed_hadoop = self.graph.resolve(Address.parse("3rdparty/jvm/managed:hadoop-common"))
def assert_product_for_subjects(self, walk, product, subjects, variants=None):
variants = tuple(variants.items()) if variants else None
self.assertEqual(
{SelectNode(subject, product, variants) for subject in subjects},
{
node
for (node, _), _ in walk
if node.product == product and isinstance(node, SelectNode) and node.variants == variants
},
)
def build_and_walk(self, build_request):
"""Build and then walk the given build_request, returning the walked graph as a list."""
result = self.engine.execute(build_request)
self.assertIsNone(result.error)
walk = list(self.scheduler.walk_product_graph())
for entry in walk:
print(">>> {}".format(entry))
return walk
def assert_resolve_only(self, goals, root_specs, jars):
build_request = BuildRequest(goals=goals, addressable_roots=[Address.parse(spec) for spec in root_specs])
walk = self.build_and_walk(build_request)
# Expect a SelectNode for each of the Jar and Classpath, and a TaskNode and NativeNode.
self.assertEqual(4 * len(jars), len(walk))
self.assert_product_for_subjects(walk, Jar, jars)
self.assert_product_for_subjects(walk, Classpath, jars)
def test_resolve(self):
self.assert_resolve_only(goals=["resolve"], root_specs=["3rdparty/jvm:guava"], jars=[self.guava])
def test_compile_only_3rdparty(self):
self.assert_resolve_only(goals=["compile"], root_specs=["3rdparty/jvm:guava"], jars=[self.guava])
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = BuildRequest(goals=["gen"], addressable_roots=[self.java.address])
walk = self.build_and_walk(build_request)
self.assertEqual(2, len(walk))
self.assert_product_for_subjects(walk, JavaSources, [self.java])
def test_gen(self):
build_request = BuildRequest(goals=["gen"], addressable_roots=[self.thrift.address])
walk = self.build_and_walk(build_request)
# Root: expect JavaSources.
root_entry = walk[0][0]
self.assertEqual(SelectNode(self.thrift, JavaSources, None), root_entry[0])
self.assertIsInstance(root_entry[1], Return)
# Expect an ApacheThriftJavaConfiguration to have been used.
product = ApacheThriftJavaConfiguration
self.assertEqual(
{NativeNode(self.thrift, product, None)},
{node for (node, _), _ in walk if node.product == product and isinstance(node, NativeNode)},
)
def test_codegen_simple(self):
build_request = BuildRequest(goals=["compile"], addressable_roots=[self.java.address])
walk = self.build_and_walk(build_request)
subjects = [
self.guava,
Jar(org="org.apache.thrift", name="libthrift", rev="0.9.2"),
Jar(org="commons-lang", name="commons-lang", rev="2.5"),
self.graph.resolve(Address.parse("src/thrift:slf4j-api")),
self.java,
self.thrift,
]
# Root: expect compilation via javac.
self.assertEqual((SelectNode(self.java, Classpath, None), Return(Classpath(creator="javac"))), walk[0][0])
# Confirm that exactly the expected subjects got Classpaths.
self.assert_product_for_subjects(walk, Classpath, subjects)
def test_consumes_resources(self):
build_request = BuildRequest(goals=["compile"], addressable_roots=[self.consumes_resources.address])
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual(
(SelectNode(self.consumes_resources, Classpath, None), Return(Classpath(creator="javac"))), walk[0][0]
)
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources, self.consumes_resources, self.guava]
self.assert_product_for_subjects(walk, Classpath, subjects)
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = BuildRequest(goals=["compile"], addressable_roots=[self.consumes_managed_thirdparty.address])
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual(
(SelectNode(self.consumes_managed_thirdparty, Classpath, None), Return(Classpath(creator="javac"))),
walk[0][0],
)
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava, self.managed_hadoop]
self.assert_product_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty])
self.assert_product_for_subjects(walk, Classpath, managed_jars, variants={"resolve": "latest-hadoop"})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals(
{Jar("org.apache.hadoop", "hadoop-common", "2.7.0"), Jar("com.google.guava", "guava", "18.0")},
{ret.value for (node, ret), _ in walk if node.product == Jar and isinstance(node, SelectNode)},
)
@pytest.mark.xfail(raises=ConflictingProducersError)
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = BuildRequest(goals=["compile"], addressable_roots=[self.java_multi.address])
walk = self.build_and_walk(build_request)
@pytest.mark.xfail(raises=PartiallyConsumedInputsError)
def test_no_configured_thrift_planner(self):
"""Even though the BuildPropertiesPlanner is able to produce a Classpath,
we still fail when a target with thrift sources doesn't have a thrift config.
"""
build_request = BuildRequest(goals=["compile"], addressable_roots=[self.unconfigured_thrift.address])
walk = self.build_and_walk(build_request)
def test_serial_engine_fail_slow(self):
engine = LocalSerialEngine(self.scheduler)
self.assert_engine_fail_slow(engine)
