def test_multiprocess_unpickleable(self):
build_request = BuildRequest(goals=['unpickleable'],
addressable_roots=[self.java.address])
with self.multiprocessing_engine() as engine:
with self.assertRaises(SerializationError):
engine.execute(build_request)
def assert_engine_fail_slow(self, engine):
build_request = BuildRequest(
goals=['compile'],
addressable_roots=[self.java.address, self.java_fail_slow.address])
result = engine.execute(build_request, fail_slow=True)
self.assertEqual({Promise(Classpath, self.java): Javac.fake_product()},
result.root_products)
self.assertIsInstance(result.error, Engine.PartialFailureError)
self.assertEqual(1, len(result.error.failed_to_produce))
failed_promise = Promise(Classpath, self.java_fail_slow)
failed_to_produce = result.error.failed_to_produce[failed_promise]
failing_configuration = self.failing_thrift.select_configuration(
'failing')
self.assertEqual([
Promise(Sources.of('.java'), self.failing_thrift,
failing_configuration),
Promise(Classpath, self.failing_thrift, failing_configuration),
Promise(Classpath, self.java_fail_slow)
], [ftp.promise for ftp in failed_to_produce.walk(postorder=True)])
errors = [ftp.error for ftp in failed_to_produce.walk(postorder=True)]
self.assertEqual(3, len(errors))
root_error = errors[0]
self.assertIsInstance(root_error, ApacheThriftError)
self.assertEqual([None, None], errors[1:])
def main():
def usage(error_message):
print(error_message, file=sys.stderr)
print(dedent("""
{} [build root path] [goal]+ [address spec]*
""".format(sys.argv[0])),
file=sys.stderr)
sys.exit(1)
args = sys.argv[1:]
if len(args) < 2:
usage('Must supply at least the build root path and one goal.')
build_root = args.pop(0)
if not os.path.isdir(build_root):
usage(
'First argument must be a valid build root, {} is not a directory.'
.format(build_root))
goals = [arg for arg in args if os.path.sep not in arg]
if not goals:
usage('Must supply at least one goal.')
build_request = BuildRequest(goals=goals,
addressable_roots=[
Address.parse(spec) for spec in args
if os.path.sep in spec
])
visualize_build_request(build_root, build_request)
def assert_engine(self, engine):
build_request = BuildRequest(goals=['compile'],
addressable_roots=[self.java.address])
result = engine.execute(build_request)
self.assertEqual({Promise(Classpath, self.java): Javac.fake_product()},
result.root_products)
self.assertIsNone(result.error)
def test_managed_resolve(self):
"""A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars."""
build_request = BuildRequest(
goals=['compile'],
addressable_roots=[self.consumes_managed_thirdparty])
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual(
(SelectNode(self.consumes_managed_thirdparty, Classpath, None,
None), Return(Classpath(creator='javac'))), walk[0][0])
# Confirm that we produced classpaths for the managed jars.
managed_jars = [self.managed_guava, self.managed_hadoop]
self.assert_select_for_subjects(walk, Classpath,
[self.consumes_managed_thirdparty])
self.assert_select_for_subjects(walk,
Classpath,
managed_jars,
variants={'resolve': 'latest-hadoop'})
# Confirm that the produced jars had the appropriate versions.
self.assertEquals(
{
Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'),
Jar('com.google.guava', 'guava', '18.0')
}, {
ret.value
for (node, ret), _ in walk
if node.product == Jar and isinstance(node, SelectNode)
})
def test_no_configured_thrift_planner(self):
"""Even though the BuildPropertiesPlanner is able to produce a Classpath,
we still fail when a target with thrift sources doesn't have a thrift config.
"""
build_request = BuildRequest(goals=['compile'],
addressable_roots=[self.unconfigured_thrift.address])
walk = self.build_and_walk(build_request)
def test_no_configured_thrift_planner(self):
"""Tests that even though the BuildPropertiesPlanner is able to produce a Classpath,
we still fail when a target with thrift sources doesn't have a thrift config.
"""
build_request = BuildRequest(goals=['compile'],
addressable_roots=[self.unconfigured_thrift.address])
with self.assertRaises(PartiallyConsumedInputsError):
execution_graph = self.scheduler.execution_graph(build_request)
def main_filespecs():
build_root, goals, args = pop_build_root_and_goals(
'[build root path] [filespecs]*', sys.argv[1:])
# Create a PathGlobs object relative to the buildroot.
path_globs = PathGlobs.create('', globs=args)
visualize_build_request(build_root,
BuildRequest(goals=goals, subjects=[path_globs]))
def main_addresses():
build_root, goals, args = pop_build_root_and_goals(
'[build root path] [goal]+ [address spec]*', sys.argv[1:])
cmd_line_spec_parser = CmdLineSpecParser(build_root)
spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in args]
visualize_build_request(build_root,
BuildRequest(goals=goals, subjects=spec_roots))
def assert_resolve_only(self, goals, root_specs, jars):
build_request = BuildRequest(goals=goals,
addressable_roots=[Address.parse(spec) for spec in root_specs])
walk = self.build_and_walk(build_request)
# Expect a SelectNode for each of the Jar and Classpath, and a TaskNode and NativeNode.
self.assertEqual(4 * len(jars), len(walk))
self.assert_product_for_subjects(walk, Jar, jars)
self.assert_product_for_subjects(walk, Classpath, jars)
def assert_resolve_only(self, goals, root_specs, jars):
build_request = BuildRequest(
goals=goals,
addressable_roots=[Address.parse(spec) for spec in root_specs])
walk = self.build_and_walk(build_request)
# Expect a SelectNode for each of the Jar/Classpath.
self.assert_select_for_subjects(walk, Jar, jars)
self.assert_select_for_subjects(walk, Classpath, jars)
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = BuildRequest(goals=['gen'],
addressable_roots=[self.java])
walk = self.build_and_walk(build_request)
self.assert_select_for_subjects(walk, JavaSources, [self.java])
def assert_resolve_only(self, goals, root_specs, jars):
build_request = BuildRequest(goals=goals,
addressable_roots=[Address.parse(spec) for spec in root_specs])
execution_graph = self.scheduler.execution_graph(build_request)
plans = list(execution_graph.walk())
self.assertEqual(1, len(plans))
self.assertEqual((Classpath,
Plan(func_or_task_type=IvyResolve, subjects=jars, jars=list(jars))),
self.extract_product_type_and_plan(plans[0]))
def assert_engine(self, engine):
build_request = BuildRequest(goals=['compile'],
addressable_roots=[self.java.address])
result = engine.execute(build_request)
self.assertEqual(
{
SelectNode(self.java, Classpath, None):
Return(Classpath(creator='javac'))
}, result.root_products)
self.assertIsNone(result.error)
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = BuildRequest(goals=['gen'],
addressable_roots=[self.java.address])
execution_graph = self.scheduler.execution_graph(build_request)
plans = list(execution_graph.walk())
self.assertEqual(0, len(plans))
def resolve(self, spec):
request = BuildRequest(goals=[self._goal], subjects=[spec])
result = LocalSerialEngine(self.scheduler).execute(request)
if result.error:
raise result.error
# Expect a single root.
state, = result.root_products.values()
if type(state) is Throw:
raise state.exc
return state.value
def test_consumes_resources(self):
build_request = BuildRequest(
goals=['compile'], addressable_roots=[self.consumes_resources])
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual(
(SelectNode(self.consumes_resources, Classpath, None,
None), Return(Classpath(creator='javac'))), walk[0][0])
# Confirm a classpath for the resources target and other subjects. We know that they are
# reachable from the root (since it was involved in this walk).
subjects = [self.resources, self.consumes_resources, self.guava]
self.assert_select_for_subjects(walk, Classpath, subjects)
def test_gen(self):
build_request = BuildRequest(goals=['gen'], addressable_roots=[self.thrift.address])
walk = self.build_and_walk(build_request)
# Root: expect JavaSources.
root_entry = walk[0][0]
self.assertEqual(SelectNode(self.thrift, JavaSources, None), root_entry[0])
self.assertIsInstance(root_entry[1], Return)
# Expect an ApacheThriftJavaConfiguration to have been used.
product = ApacheThriftJavaConfiguration
self.assertEqual({NativeNode(self.thrift, product, None)},
{node for (node, _), _ in walk
if node.product == product and isinstance(node, NativeNode)})
def test_dependency_inference(self):
"""Scala dependency inference introduces dependencies that do not exist in BUILD files."""
build_request = BuildRequest(goals=['compile'],
addressable_roots=[self.inferred_deps])
walk = self.build_and_walk(build_request)
# Validate the root.
self.assertEqual(
(SelectNode(self.inferred_deps, Classpath, None, None),
Return(Classpath(creator='scalac'))), walk[0][0])
# Confirm that we requested a classpath for the root and inferred targets.
self.assert_select_for_subjects(walk, Classpath,
[self.inferred_deps, self.java_simple])
def test_gen(self):
build_request = BuildRequest(goals=['gen'], addressable_roots=[self.thrift.address])
execution_graph = self.scheduler.execution_graph(build_request)
plans = list(execution_graph.walk())
self.assertEqual(1, len(plans))
self.assertEqual((Sources.of('.java'),
Plan(func_or_task_type=gen_apache_thrift,
subjects=[self.thrift],
strict=True,
rev='0.9.2',
gen='java',
sources=['src/thrift/codegen/simple/simple.thrift'])),
self.extract_product_type_and_plan(plans[0]))
def test_codegen_simple(self):
build_request = BuildRequest(goals=['compile'],
addressable_roots=[self.java.address])
execution_graph = self.scheduler.execution_graph(build_request)
plans = list(execution_graph.walk())
self.assertEqual(4, len(plans))
thrift_jars = [
Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2'),
Jar(org='commons-lang', name='commons-lang', rev='2.5'),
self.graph.resolve(Address.parse('src/thrift:slf4j-api'))
]
jars = [self.guava] + thrift_jars
# Independent leaves 1st
self.assertEqual(
{(Sources.of('.java'),
Plan(func_or_task_type=gen_apache_thrift,
subjects=[self.thrift],
strict=True,
rev='0.9.2',
gen='java',
sources=['src/thrift/codegen/simple/simple.thrift'])),
(Classpath,
Plan(func_or_task_type=IvyResolve, subjects=jars, jars=jars))},
set(self.extract_product_type_and_plan(p) for p in plans[0:2]))
# The rest is linked.
self.assertEqual(
(Classpath,
Plan(func_or_task_type=Javac,
subjects=[self.thrift],
sources=Promise(Sources.of('.java'), self.thrift),
classpath=[Promise(Classpath, jar) for jar in thrift_jars])),
self.extract_product_type_and_plan(plans[2]))
self.assertEqual((Classpath,
Plan(func_or_task_type=Javac,
subjects=[self.java],
sources=['src/java/codegen/simple/Simple.java'],
classpath=[
Promise(Classpath, self.guava),
Promise(Classpath, self.thrift)
])),
self.extract_product_type_and_plan(plans[3]))
def test_gen_noop(self):
# TODO(John Sirois): Ask around - is this OK?
# This is different than today. There is a gen'able target reachable from the java target, but
# the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's
# handed is not thrift.
build_request = BuildRequest(goals=['gen'], addressable_roots=[self.java.address])
execution_graph = self.scheduler.execution_graph(build_request)
plans = list(execution_graph.walk())
self.assertEqual(1, len(plans))
self.assertEqual((Sources.of('.java'),
Plan(func_or_task_type=lift_native_product,
subjects=[self.java],
subject=self.java,
product_type=Sources.of('.java'))),
self.extract_product_type_and_plan(plans[0]))
def test_codegen_simple(self):
build_request = BuildRequest(goals=['compile'], addressable_roots=[self.java.address])
walk = self.build_and_walk(build_request)
subjects = [self.guava,
Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2'),
Jar(org='commons-lang', name='commons-lang', rev='2.5'),
self.graph.resolve(Address.parse('src/thrift:slf4j-api')),
self.java,
self.thrift]
# Root: expect compilation via javac.
self.assertEqual((SelectNode(self.java, Classpath, None), Return(Classpath(creator='javac'))),
walk[0][0])
# Confirm that exactly the expected subjects got Classpaths.
self.assert_product_for_subjects(walk, Classpath, subjects)
def test_gen(self):
build_request = BuildRequest(goals=['gen'],
addressable_roots=[self.thrift])
walk = self.build_and_walk(build_request)
# Root: expect JavaSources.
root_entry = walk[0][0]
self.assertEqual(SelectNode(self.thrift, JavaSources, None, None),
root_entry[0])
self.assertIsInstance(root_entry[1], Return)
# Expect an ApacheThriftJavaConfiguration to have been used via the default Variants.
self.assert_select_for_subjects(walk,
ApacheThriftJavaConfiguration,
[self.thrift],
variants={'thrift': 'apache_java'},
variant_key='thrift')
def test_codegen_simple(self):
build_request = BuildRequest(goals=['compile'],
addressable_roots=[self.java])
walk = self.build_and_walk(build_request)
# The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants.
subjects = [self.guava, self.java, self.thrift]
variant_subjects = [
Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2'),
Jar(org='commons-lang', name='commons-lang', rev='2.5'),
Address.parse('src/thrift:slf4j-api')
]
# Root: expect compilation via javac.
self.assertEqual(
(SelectNode(self.java, Classpath, None,
None), Return(Classpath(creator='javac'))), walk[0][0])
# Confirm that exactly the expected subjects got Classpaths.
self.assert_select_for_subjects(walk, Classpath, subjects)
self.assert_select_for_subjects(walk,
Classpath,
variant_subjects,
variants={'thrift': 'apache_java'})
def request_specs(self, goals, *specs):
return BuildRequest(goals=goals, subjects=specs)
def request(self, goals, *addresses): specs = [self.spec_parser.parse_spec(str(a)) for a in addresses] return BuildRequest(goals=goals, subjects=specs)
def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = BuildRequest(goals=['compile'], addressable_roots=[self.java_multi.address]) walk = self.build_and_walk(build_request)
def test_multiple_classpath_entries(self):
"""Multiple Classpath products for a single subject currently cause a failure."""
build_request = BuildRequest(
goals=['compile'], addressable_roots=[self.java_multi.address])
execution_graph = self.scheduler.execution_graph(build_request)
def _populate(self, scheduler, address):
"""Make a BuildRequest to parse the given Address into a Struct."""
spec = self._cmd_line_spec_parser.parse_spec(str(address))
request = BuildRequest(goals=[self._goal], subjects=[spec])
LocalSerialEngine(scheduler).reduce(request)
return self._select(address)