def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root, inline_nodes=False) self.pg = self.scheduler.product_graph self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse( 'src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse( 'src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse( 'src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse( '3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse( '3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps')
def execute_request(self, scheduler, product, *subjects): """Creates, runs, and returns an ExecutionRequest for the given product and subjects.""" request = scheduler.execution_request([product], subjects) engine = LocalSerialEngine(scheduler) res = engine.execute(request) if res.error: raise res.error return request
def visualize_build_request(build_root, goals, subjects): with subsystem_instance(Native.Factory) as native_factory: scheduler = setup_json_scheduler(build_root, native_factory.create()) execution_request = scheduler.build_request(goals, subjects) # NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it. engine = LocalSerialEngine(scheduler, Storage.create()) engine.reduce(execution_request) visualize_execution_graph(scheduler)
def visualize_build_request(build_root, goals, subjects): scheduler = setup_json_scheduler(build_root) execution_request = scheduler.build_request(goals, subjects) # NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it. engine = LocalSerialEngine(scheduler, Storage.create()) try: engine.reduce(execution_request) visualize_execution_graph(scheduler, execution_request) finally: engine.close()
def _populate(self, scheduler, address): """Perform an ExecutionRequest to parse the given Address into a Struct.""" request = scheduler.execution_request([self._product], [address]) LocalSerialEngine(scheduler).reduce(request) root_entries = scheduler.root_entries(request).items() self.assertEquals(1, len(root_entries)) return root_entries[0]
def setup_legacy_graph(path_ignore_patterns): """Construct and return the components necessary for LegacyBuildGraph construction. :param list path_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the `--pants-ignore` global option. :returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls). """ build_root = get_buildroot() project_tree = FileSystemProjectTree(build_root, path_ignore_patterns) symbol_table_cls = LegacySymbolTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks() + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls) ) scheduler = LocalScheduler(dict(), tasks, project_tree) engine = LocalSerialEngine(scheduler, Storage.create(debug=False)) return LegacyGraphHelper(scheduler, engine, symbol_table_cls, LegacyBuildGraph)
def setup_legacy_graph(pants_ignore_patterns, workdir, build_root=None, native=None, symbol_table_cls=None, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None): """Construct and return the components necessary for LegacyBuildGraph construction. :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the '--pants-ignore' global option. :param str workdir: The pants workdir. :param str build_root: A path to be used as the build root. If None, then default is used. :param Native native: An instance of the native-engine subsystem. :param SymbolTable symbol_table_cls: A SymbolTable class to use for build file parsing, or None to use the default. :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :param list subproject_roots: Paths that correspond with embedded build roots under the current build root. :returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls). """ build_root = build_root or get_buildroot() scm = get_scm() symbol_table_cls = symbol_table_cls or LegacySymbolTable project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns) # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper( symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_roots) # Load the native backend. native = native or Native.Factory.global_instance().create() # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. tasks = (create_legacy_graph_tasks(symbol_table_cls) + create_fs_rules() + create_graph_rules(address_mapper, symbol_table_cls)) # TODO: Do not use the cache yet, as it incurs a high overhead. scheduler = LocalScheduler(workdir, dict(), tasks, project_tree, native) engine = LocalSerialEngine(scheduler, use_cache=False) change_calculator = EngineChangeCalculator( scheduler, engine, symbol_table_cls, scm) if scm else None return LegacyGraphHelper(scheduler, engine, symbol_table_cls, change_calculator)
def _populate(self, scheduler, address): """Perform an ExecutionRequest to parse the given Address into a Struct.""" # NB: requesting any of the possible types that come from parsing a BUILD file request = scheduler.execution_request([TestTable.constraint()], [address]) LocalSerialEngine(scheduler).reduce(request) root_entries = scheduler.root_entries(request).items() self.assertEquals(1, len(root_entries)) return root_entries[0]
def visualize_build_request(build_root, goals, subjects): scheduler, storage = setup_json_scheduler(build_root) execution_request = scheduler.build_request(goals, subjects) # NB: Calls `reduce` independently of `execute`, in order to render a graph before validating it. engine = LocalSerialEngine(scheduler, storage) engine.start() try: engine.reduce(execution_request) visualize_execution_graph(scheduler, storage, execution_request) finally: engine.close()
def resolve(self, spec): request = self.scheduler.execution_request([UnhydratedStruct], [spec]) result = LocalSerialEngine(self.scheduler, self.storage).execute(request) if result.error: raise result.error # Expect a single root. state, = result.root_products.values() if type(state) is Throw: raise state.exc return state.value
def resolve(self, spec): select = SelectDependencies(UnhydratedStruct, Addresses, field_types=(Address,)) request = self.scheduler.selection_request([(select, spec)]) result = LocalSerialEngine(self.scheduler).execute(request) if result.error: raise result.error # Expect a single root. state, = result.root_products.values() if type(state) is Throw: raise Exception(state.exc) return state.value
def test_gather_snapshot_of_pathglobs(self): project_tree = self.mk_example_fs_tree() scheduler = self.mk_scheduler(project_tree=project_tree, tasks=create_snapshot_tasks(project_tree)) snapshot_archive_root = os.path.join(project_tree.build_root, '.snapshots') request = scheduler.execution_request([Snapshot], [PathGlobs.create('', globs=['fs_test/a/b/*'])]) LocalSerialEngine(scheduler).reduce(request) root_entries = scheduler.root_entries(request).items() self.assertEquals(1, len(root_entries)) state = self.assertFirstEntryIsReturn(root_entries, scheduler) snapshot = state.value self.assert_archive_files(['fs_test/a/b/1.txt', 'fs_test/a/b/2'], snapshot, snapshot_archive_root)
def test_gather_snapshot_of_pathglobs(self): project_tree = self.mk_example_fs_tree() scheduler = self.mk_scheduler(project_tree=project_tree) empty_step_context = StepContext(node_builder=None, project_tree=project_tree, node_states=[], inline_nodes=False) request = scheduler.execution_request([Snapshot], [PathGlobs.create('', globs=['fs_test/a/b/*'])]) LocalSerialEngine(scheduler).reduce(request) root_entries = scheduler.root_entries(request).items() self.assertEquals(1, len(root_entries)) state = self.assertFirstEntryIsReturn(root_entries, scheduler) snapshot = state.value self.assert_archive_files(['fs_test/a/b/1.txt', 'fs_test/a/b/2'], snapshot, empty_step_context)
def setup_legacy_graph(pants_ignore_patterns, symbol_table_cls=None, build_ignore_patterns=None, exclude_target_regexps=None): """Construct and return the components necessary for LegacyBuildGraph construction. :param list pants_ignore_patterns: A list of path ignore patterns for FileSystemProjectTree, usually taken from the '--pants-ignore' global option. :param SymbolTable symbol_table_cls: A SymbolTable class to use for build file parsing, or None to use the default. :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :returns: A tuple of (scheduler, engine, symbol_table_cls, build_graph_cls). """ build_root = get_buildroot() scm = get_scm() project_tree = FileSystemProjectTree(build_root, pants_ignore_patterns) symbol_table_cls = symbol_table_cls or LegacySymbolTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper( symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. tasks = (create_legacy_graph_tasks(symbol_table_cls) + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls)) scheduler = LocalScheduler(dict(), tasks, project_tree) # TODO: Do not use the cache yet, as it incurs a high overhead. engine = LocalSerialEngine(scheduler, Storage.create(), use_cache=False) change_calculator = EngineChangeCalculator(engine, scm) if scm else None return LegacyGraphHelper(scheduler, engine, symbol_table_cls, change_calculator)
def test_failed_command_propagates_throw(self): scheduler = self.mk_scheduler_in_example_fs([ # subject to files / product of subject to files for snapshot. SnapshottedProcess.create(product_type=Concatted, binary_type=ShellFailCommand, input_selectors=tuple(), input_conversion=empty_process_request, output_conversion=fail_process_result), [ShellFailCommand, [], ShellFailCommand] ]) request = scheduler.execution_request([Concatted], [PathGlobs.create('', include=['fs_test/a/b/*'])]) LocalSerialEngine(scheduler).reduce(request) root_entries = scheduler.root_entries(request).items() self.assertEquals(1, len(root_entries)) self.assertFirstEntryIsThrow(root_entries, in_msg='Running ShellFailCommand failed with non-zero exit code: 1')
def test_failed_output_conversion_propagates_throw(self): scheduler = self.mk_scheduler_in_example_fs([ # subject to files / product of subject to files for snapshot. SnapshottedProcess.create(product_type=Concatted, binary_type=ShellCatToOutFile, input_selectors=(Select(Snapshot),), input_conversion=file_list_to_args_for_cat_with_snapshot_subjects_and_output_file, output_conversion=fail_process_result), [ShellCatToOutFile, [], ShellCatToOutFile] ]) request = scheduler.execution_request([Concatted], [PathGlobs.create('', include=['fs_test/a/b/*'])]) LocalSerialEngine(scheduler).reduce(request) root_entries = scheduler.root_entries(request).items() self.assertEquals(1, len(root_entries)) self.assertFirstEntryIsThrow(root_entries, in_msg='Failed in output conversion!')
def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root, self._native) self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps')
def test_integration_concat_with_snapshot_subjects_test(self): scheduler = self.mk_scheduler_in_example_fs([ # subject to files / product of subject to files for snapshot. SnapshottedProcess.create(product_type=Concatted, binary_type=ShellCatToOutFile, input_selectors=(Select(Snapshot),), input_conversion=file_list_to_args_for_cat_with_snapshot_subjects_and_output_file, output_conversion=process_result_to_concatted_from_outfile), [ShellCatToOutFile, [], ShellCatToOutFile], ]) request = scheduler.execution_request([Concatted], [PathGlobs.create('', include=['fs_test/a/b/*'])]) LocalSerialEngine(scheduler).reduce(request) root_entries = scheduler.root_entries(request).items() self.assertEquals(1, len(root_entries)) state = self.assertFirstEntryIsReturn(root_entries, scheduler) concatted = state.value self.assertEqual(Concatted('one\ntwo\n'), concatted)
def setUp(self): build_root = os.path.join(os.path.dirname(__file__), "examples", "scheduler_inputs") self.spec_parser = CmdLineSpecParser(build_root) self.scheduler, self.storage = setup_json_scheduler(build_root) self.pg = self.scheduler.product_graph self.engine = LocalSerialEngine(self.scheduler, self.storage) self.guava = Address.parse("3rdparty/jvm:guava") self.thrift = Address.parse("src/thrift/codegen/simple") self.java = Address.parse("src/java/codegen/simple") self.java_simple = Address.parse("src/java/simple") self.java_multi = Address.parse("src/java/multiple_classpath_entries") self.no_variant_thrift = Address.parse("src/java/codegen/selector:conflict") self.unconfigured_thrift = Address.parse("src/thrift/codegen/unconfigured") self.resources = Address.parse("src/resources/simple") self.consumes_resources = Address.parse("src/java/consumes_resources") self.consumes_managed_thirdparty = Address.parse("src/java/managed_thirdparty") self.managed_guava = Address.parse("3rdparty/jvm/managed:guava") self.managed_hadoop = Address.parse("3rdparty/jvm/managed:hadoop-common") self.managed_resolve_latest = Address.parse("3rdparty/jvm/managed:latest-hadoop") self.inferred_deps = Address.parse("src/scala/inferred_deps")
def test_javac_compilation_example(self): sources = PathGlobs.create('', include=['scheduler_inputs/src/java/simple/Simple.java']) scheduler = self.mk_scheduler_in_example_fs([ SnapshottedProcess.create(ClasspathEntry, Javac, (Select(Snapshot), SelectLiteral(JavaOutputDir('build'), JavaOutputDir)), java_sources_to_javac_args, process_result_to_classpath_entry), [Javac, [], Javac] ]) request = scheduler.execution_request( [ClasspathEntry], [sources]) LocalSerialEngine(scheduler).reduce(request) root_entries = scheduler.root_entries(request).items() self.assertEquals(1, len(root_entries)) state = self.assertFirstEntryIsReturn(root_entries, scheduler) classpath_entry = state.value self.assertIsInstance(classpath_entry, ClasspathEntry) self.assertTrue(os.path.exists(os.path.join(classpath_entry.path, 'simple', 'Simple.class')))
def create_engine(self, root_subject_types, rules, include_trace_on_error): engine = LocalSerialEngine( self.scheduler(root_subject_types, rules), include_trace_on_error=include_trace_on_error) return engine
class SchedulerTest(unittest.TestCase): _native = init_native() def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root, self._native) self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps') def assert_select_for_subjects(self, walk, selector, subjects, variants=None): raise ValueError(walk) def build(self, build_request): """Execute the given request and return roots as a list of ((subject, product), value) tuples.""" result = self.engine.execute(build_request) self.assertIsNone(result.error) return self.scheduler.root_entries(build_request).items() def request(self, goals, *subjects): return self.scheduler.build_request(goals=goals, subjects=subjects) def assert_root(self, root, subject, return_value): """Asserts that the given root has the given result.""" self.assertEquals(subject, root[0][0]) self.assertEquals(Return(return_value), root[1]) def assert_root_failed(self, root, subject, msg_str): """Asserts that the root was a Throw result containing the given msg string.""" self.assertEquals(subject, root[0][0]) self.assertEquals(Throw, type(root[1])) self.assertIn(msg_str, str(root[1].exc)) def test_compile_only_3rdparty(self): build_request = self.request(['compile'], self.guava) root, = self.build(build_request) self.assert_root(root, self.guava, Classpath(creator='ivy_resolve')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_compile_only_3rdparty_internal(self): build_request = self.request(['compile'], '3rdparty/jvm:guava') root, = self.build(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Select(Jar), [self.guava]) self.assert_select_for_subjects(walk, Select(Classpath), [self.guava]) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_gen(self): build_request = self.request(['gen'], self.thrift) root, = self.build(build_request) # Root: expect the synthetic GenGoal product. self.assert_root(root, self.thrift, GenGoal("non-empty input to satisfy the Goal constructor")) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, SelectVariant(ApacheThriftJavaConfiguration, variant_key='thrift'), [self.thrift], variants=variants) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_codegen_simple(self): build_request = self.request(['compile'], self.java) root, = self.build(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'), Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'), Address.parse('src/thrift:slf4j-api')] # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. self.assert_root(root, self.java, Classpath(creator='javac')) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Select(Classpath), subjects) self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = self.request(['compile'], self.consumes_resources) root, = self.build(build_request) self.assert_root(root, self.consumes_resources, Classpath(creator='javac')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_consumes_resources_internal(self): build_request = self.request(['compile'], self.consumes_resources) root, = self.build(build_request) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Select(Classpath), subjects) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request(['compile'], self.consumes_managed_thirdparty) root, = self.build(build_request) # Validate the root. self.assert_root(root, self.consumes_managed_thirdparty, Classpath(creator='javac')) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Select(Classpath), managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0')}, {ret.value for node, ret in walk if node.product == Jar}) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) root, = self.build(build_request) self.assert_root(root, self.inferred_deps, Classpath(creator='scalac')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_dependency_inference_internal(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) root, = self.build(build_request) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request(['compile'], self.java_multi) root, = self.build(build_request) # Validate that the root failed. self.assert_root_failed(root, self.java_multi, "Conflicting values produced for") def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm::') selector = Select(BuildFileAddresses) build_request = self.scheduler.selection_request([(selector, spec)]) ((subject, _), root), = self.build(build_request) # Validate the root. self.assertEqual(spec, subject) self.assertEqual(BuildFileAddresses, type(root.value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root.value.dependencies) self.assertIn(self.managed_guava, root.value.dependencies) self.assertIn(self.managed_resolve_latest, root.value.dependencies) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm:') selector = Select(BuildFileAddresses) build_request = self.scheduler.selection_request([(selector,spec)]) ((subject, _), root), = self.build(build_request) # Validate the root. self.assertEqual(spec, subject) self.assertEqual(BuildFileAddresses, type(root.value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root.value.dependencies) # And that a subdirectory address is not. self.assertNotIn(self.managed_guava, root.value.dependencies) def test_scheduler_visualize(self): spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request(['list'], spec) self.build(build_request) with temporary_dir() as td: output_path = os.path.join(td, 'output.dot') self.scheduler.visualize_graph_to_file(output_path) with open(output_path, 'rb') as fh: graphviz_output = fh.read().strip() self.assertIn('digraph', graphviz_output) self.assertIn(' -> ', graphviz_output)
def serial_engine(self): with closing(LocalSerialEngine(self.scheduler)) as e: yield e
def test_serial_engine_simple(self): with closing(LocalSerialEngine(self.scheduler)) as engine: self.assert_engine(engine)
def test_serial_engine_simple(self): engine = LocalSerialEngine(self.scheduler, self.storage, self.cache) self.assert_engine(engine)
def serial_engine(self): yield LocalSerialEngine(self.scheduler)
class SchedulerTest(unittest.TestCase): _native = init_native() def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root, self._native) self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse( 'src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse( 'src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse( 'src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse( '3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse( '3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps') def assert_select_for_subjects(self, walk, selector, subjects, variants=None): raise ValueError(walk) def build(self, build_request): """Execute the given request and return roots as a list of ((subject, product), value) tuples.""" result = self.engine.execute(build_request) self.assertIsNone(result.error) return self.scheduler.root_entries(build_request).items() def request(self, goals, *subjects): return self.scheduler.build_request(goals=goals, subjects=subjects) def assert_root(self, root, subject, return_value): """Asserts that the given root has the given result.""" self.assertEquals(subject, root[0][0]) self.assertEquals(Return(return_value), root[1]) def assert_root_failed(self, root, subject, msg_str): """Asserts that the root was a Throw result containing the given msg string.""" self.assertEquals(subject, root[0][0]) self.assertEquals(Throw, type(root[1])) self.assertIn(msg_str, str(root[1].exc)) def test_compile_only_3rdparty(self): build_request = self.request(['compile'], self.guava) root, = self.build(build_request) self.assert_root(root, self.guava, Classpath(creator='ivy_resolve')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_compile_only_3rdparty_internal(self): build_request = self.request(['compile'], '3rdparty/jvm:guava') root, = self.build(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Select(Jar), [self.guava]) self.assert_select_for_subjects(walk, Select(Classpath), [self.guava]) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_gen(self): build_request = self.request(['gen'], self.thrift) root, = self.build(build_request) # Root: expect the synthetic GenGoal product. self.assert_root( root, self.thrift, GenGoal("non-empty input to satisfy the Goal constructor")) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, SelectVariant( ApacheThriftJavaConfiguration, variant_key='thrift'), [self.thrift], variants=variants) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_codegen_simple(self): build_request = self.request(['compile'], self.java) root, = self.build(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'), Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'), Address.parse('src/thrift:slf4j-api') ] # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. self.assert_root(root, self.java, Classpath(creator='javac')) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Select(Classpath), subjects) self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = self.request(['compile'], self.consumes_resources) root, = self.build(build_request) self.assert_root(root, self.consumes_resources, Classpath(creator='javac')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_consumes_resources_internal(self): build_request = self.request(['compile'], self.consumes_resources) root, = self.build(build_request) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Select(Classpath), subjects) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request(['compile'], self.consumes_managed_thirdparty) root, = self.build(build_request) # Validate the root. self.assert_root(root, self.consumes_managed_thirdparty, Classpath(creator='javac')) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Select(Classpath), managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals( { Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0') }, {ret.value for node, ret in walk if node.product == Jar}) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) root, = self.build(build_request) self.assert_root(root, self.inferred_deps, Classpath(creator='scalac')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_dependency_inference_internal(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) root, = self.build(build_request) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request(['compile'], self.java_multi) root, = self.build(build_request) # Validate that the root failed. self.assert_root_failed(root, self.java_multi, "Conflicting values produced for") def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm::') selector = SelectDependencies(Address, Addresses, field_types=(Address, )) build_request = self.scheduler.selection_request([(selector, spec)]) ((subject, _), root), = self.build(build_request) # Validate the root. self.assertEqual(spec, subject) self.assertEqual(tuple, type(root.value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root.value) self.assertIn(self.managed_guava, root.value) self.assertIn(self.managed_resolve_latest, root.value) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm:') selector = SelectDependencies(Address, Addresses, field_types=(Address, )) build_request = self.scheduler.selection_request([(selector, spec)]) ((subject, _), root), = self.build(build_request) # Validate the root. self.assertEqual(spec, subject) self.assertEqual(tuple, type(root.value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root.value) # And that a subdirectory address is not. self.assertNotIn(self.managed_guava, root.value) def test_scheduler_visualize(self): spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request(['list'], spec) self.build(build_request) with temporary_dir() as td: output_path = os.path.join(td, 'output.dot') self.scheduler.visualize_graph_to_file(output_path) with open(output_path, 'rb') as fh: graphviz_output = fh.read().strip() self.assertIn('digraph', graphviz_output) self.assertIn(' -> ', graphviz_output)
class SchedulerTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root, inline_nodes=False) self.pg = self.scheduler.product_graph self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse( 'src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse( 'src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse( 'src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse( '3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse( '3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps') def assert_select_for_subjects(self, walk, selector, subjects, variants=None): node_type = SelectNode variants = tuple(variants.items()) if variants else None self.assertEqual( {node_type(subject, variants, selector) for subject in subjects}, { node for node, _ in walk if node.product == selector.product and isinstance(node, node_type) and node.variants == variants }) def build_and_walk(self, build_request): """Build and then walk the given build_request, returning the walked graph as a list.""" result = self.engine.execute(build_request) self.assertIsNone(result.error) return list(self.scheduler.product_graph.walk(build_request.roots)) def request(self, goals, *addresses): return self.request_specs( goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses]) def request_specs(self, goals, *specs): return self.scheduler.build_request(goals=goals, subjects=specs) def assert_resolve_only(self, goals, root_specs, jars): build_request = self.request(goals, *root_specs) walk = self.build_and_walk(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Select(Jar), jars) self.assert_select_for_subjects(walk, Select(Classpath), jars) def assert_root(self, walk, node, return_value): """Asserts that the first Node in a walk was a DependenciesNode with the single given result.""" root, root_state = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Return([return_value]), root_state) self.assertIn( (node, Return(return_value)), [(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)]) def assert_root_failed(self, walk, node, thrown_type): """Asserts that the first Node in a walk was a DependenciesNode with a Throw result.""" root, root_state = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Throw, type(root_state)) dependencies = [(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)] self.assertIn( (node, thrown_type), [(k, type(v.exc)) for k, v in dependencies if type(v) is Throw]) def test_type_error_on_unexpected_subject_type(self): with self.assertRaises(TypeError) as cm: self.scheduler.build_request(goals={}, subjects=['string']) self.assertEquals( "Unsupported root subject type: <type 'unicode'> for u'string'", str(cm.exception)) def test_resolve(self): self.assert_resolve_only(goals=['resolve'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_compile_only_3rdparty(self): self.assert_resolve_only(goals=['compile'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_gen_noop(self): # TODO(John Sirois): Ask around - is this OK? # This is different than today. There is a gen'able target reachable from the java target, but # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's # handed is not thrift. build_request = self.request(['gen'], self.java) walk = self.build_and_walk(build_request) self.assert_select_for_subjects(walk, Select(JavaSources, optional=True), [self.java]) def test_gen(self): build_request = self.request(['gen'], self.thrift) walk = self.build_and_walk(build_request) # Root: expect the synthetic GenGoal product. self.assert_root( walk, SelectNode(self.thrift, None, Select(GenGoal)), GenGoal("non-empty input to satisfy the Goal constructor")) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, SelectVariant( ApacheThriftJavaConfiguration, variant_key='thrift'), [self.thrift], variants=variants) def test_codegen_simple(self): build_request = self.request(['compile'], self.java) walk = self.build_and_walk(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'), Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'), Address.parse('src/thrift:slf4j-api') ] # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. self.assert_root(walk, SelectNode(self.java, None, Select(Classpath)), Classpath(creator='javac')) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Select(Classpath), subjects) self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = self.request(['compile'], self.consumes_resources) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root( walk, SelectNode(self.consumes_resources, None, Select(Classpath)), Classpath(creator='javac')) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Select(Classpath), subjects) def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request(['compile'], self.consumes_managed_thirdparty) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root( walk, SelectNode(self.consumes_managed_thirdparty, None, Select(Classpath)), Classpath(creator='javac')) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Select(Classpath), managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals( { Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0') }, { ret.value for node, ret in walk if node.product == Jar and isinstance(node, SelectNode) }) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root( walk, SelectNode(self.inferred_deps, None, Select(Classpath)), Classpath(creator='scalac')) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request(['compile'], self.java_multi) walk = self.build_and_walk(build_request) # Validate that the root failed. self.assert_root_failed( walk, SelectNode(self.java_multi, None, Select(Classpath)), ConflictingProducersError) def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm::') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0] root_value = root_state.value self.assertEqual( DependenciesNode( spec, None, SelectDependencies(Address, Addresses, field_types=(Address, ))), root) self.assertEqual(list, type(root_value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root_value) self.assertIn(self.managed_guava, root_value) self.assertIn(self.managed_resolve_latest, root_value) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0] root_value = root_state.value self.assertEqual( DependenciesNode( spec, None, SelectDependencies(Address, Addresses, field_types=(Address, ))), root) self.assertEqual(list, type(root_value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root_value) # And that an subdirectory address is not. self.assertNotIn(self.managed_guava, root_value) def test_scheduler_visualize(self): spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request_specs(['list'], spec) self.build_and_walk(build_request) graphviz_output = '\n'.join( self.scheduler.product_graph.visualize(build_request.roots)) with temporary_dir() as td: output_path = os.path.join(td, 'output.dot') self.scheduler.visualize_graph_to_file(build_request.roots, output_path) with open(output_path, 'rb') as fh: graphviz_disk_output = fh.read().strip() self.assertEqual(graphviz_output, graphviz_disk_output) self.assertIn('digraph', graphviz_output) self.assertIn(' -> ', graphviz_output)
class SchedulerTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root, inline_nodes=False) self.pg = self.scheduler.product_graph self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps') def assert_select_for_subjects(self, walk, selector, subjects, variants=None): node_type = SelectNode variants = tuple(variants.items()) if variants else None self.assertEqual({node_type(subject, variants, selector) for subject in subjects}, {node for node, _ in walk if node.product == selector.product and isinstance(node, node_type) and node.variants == variants}) def build_and_walk(self, build_request): """Build and then walk the given build_request, returning the walked graph as a list.""" result = self.engine.execute(build_request) self.assertIsNone(result.error) return list(self.scheduler.product_graph.walk(build_request.roots)) def request(self, goals, *addresses): return self.request_specs(goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses]) def request_specs(self, goals, *specs): return self.scheduler.build_request(goals=goals, subjects=specs) def assert_resolve_only(self, goals, root_specs, jars): build_request = self.request(goals, *root_specs) walk = self.build_and_walk(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Select(Jar), jars) self.assert_select_for_subjects(walk, Select(Classpath), jars) def assert_root(self, walk, node, return_value): """Asserts that the first Node in a walk was a DependenciesNode with the single given result.""" root, root_state = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Return([return_value]), root_state) self.assertIn((node, Return(return_value)), [(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)]) def assert_root_failed(self, walk, node, thrown_type): """Asserts that the first Node in a walk was a DependenciesNode with a Throw result.""" root, root_state = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Throw, type(root_state)) dependencies = [(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)] self.assertIn((node, thrown_type), [(k, type(v.exc)) for k, v in dependencies if type(v) is Throw]) def test_resolve(self): self.assert_resolve_only(goals=['resolve'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_compile_only_3rdparty(self): self.assert_resolve_only(goals=['compile'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_gen_noop(self): # TODO(John Sirois): Ask around - is this OK? # This is different than today. There is a gen'able target reachable from the java target, but # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's # handed is not thrift. build_request = self.request(['gen'], self.java) walk = self.build_and_walk(build_request) self.assert_select_for_subjects(walk, Select(JavaSources, optional=True), [self.java]) def test_gen(self): build_request = self.request(['gen'], self.thrift) walk = self.build_and_walk(build_request) # Root: expect the synthetic GenGoal product. self.assert_root(walk, SelectNode(self.thrift, None, Select(GenGoal)), GenGoal("non-empty input to satisfy the Goal constructor")) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, SelectVariant(ApacheThriftJavaConfiguration, variant_key='thrift'), [self.thrift], variants=variants) def test_codegen_simple(self): build_request = self.request(['compile'], self.java) walk = self.build_and_walk(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [ self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'), Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'), Address.parse('src/thrift:slf4j-api')] # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. self.assert_root(walk, SelectNode(self.java, None, Select(Classpath)), Classpath(creator='javac')) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Select(Classpath), subjects) self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = self.request(['compile'], self.consumes_resources) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.consumes_resources, None, Select(Classpath)), Classpath(creator='javac')) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Select(Classpath), subjects) def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request(['compile'], self.consumes_managed_thirdparty) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.consumes_managed_thirdparty, None, Select(Classpath)), Classpath(creator='javac')) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Select(Classpath), managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0')}, {ret.value for node, ret in walk if node.product == Jar and isinstance(node, SelectNode)}) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.inferred_deps, None, Select(Classpath)), Classpath(creator='scalac')) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request(['compile'], self.java_multi) walk = self.build_and_walk(build_request) # Validate that the root failed. self.assert_root_failed(walk, SelectNode(self.java_multi, None, Select(Classpath)), ConflictingProducersError) def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm::') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0] root_value = root_state.value self.assertEqual(DependenciesNode(spec, None, SelectDependencies(Address, Addresses)), root) self.assertEqual(list, type(root_value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root_value) self.assertIn(self.managed_guava, root_value) self.assertIn(self.managed_resolve_latest, root_value) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0] root_value = root_state.value self.assertEqual(DependenciesNode(spec, None, SelectDependencies(Address, Addresses)), root) self.assertEqual(list, type(root_value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root_value) # And that an subdirectory address is not. self.assertNotIn(self.managed_guava, root_value) def test_scheduler_visualize(self): spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request_specs(['list'], spec) self.build_and_walk(build_request) graphviz_output = '\n'.join(self.scheduler.product_graph.visualize(build_request.roots)) with temporary_dir() as td: output_path = os.path.join(td, 'output.dot') self.scheduler.visualize_graph_to_file(build_request.roots, output_path) with open(output_path, 'rb') as fh: graphviz_disk_output = fh.read().strip() self.assertEqual(graphviz_output, graphviz_disk_output) self.assertIn('digraph', graphviz_output) self.assertIn(' -> ', graphviz_output)