def visualize_build_request(build_root, goals, subjects): native = init_native() scheduler = setup_json_scheduler(build_root, native) execution_request = scheduler.build_request(goals, subjects) # NB: Calls `schedule` independently of `execute`, in order to render a graph before validating it. scheduler.schedule(execution_request) visualize_execution_graph(scheduler)
def create_scheduler(self, rule_index): native = init_native() scheduler = WrappedNativeScheduler( native=native, build_root='/tmp', work_dir='/tmp/.pants.d', ignore_patterns=tuple(), rule_index=rule_index) return scheduler
def _init_engine(cls): if cls._scheduler is not None: return # NB: This uses the long form of initialization because it needs to directly specify # `cls.alias_groups` rather than having them be provided by bootstrap options. graph_session = EngineInitializer.setup_legacy_graph_extended( pants_ignore_patterns=None, workdir=cls._pants_workdir(), build_file_imports_behavior='allow', native=init_native(), build_configuration=cls.build_config(), build_ignore_patterns=None, ).new_session() cls._scheduler = graph_session.scheduler_session cls._build_graph, cls._address_mapper = graph_session.create_build_graph( TargetRoots([]), cls._build_root() )
def _init_engine(cls): if cls._scheduler is not None: return cls._local_store_dir = os.path.realpath(safe_mkdtemp()) safe_mkdir(cls._local_store_dir) # NB: This uses the long form of initialization because it needs to directly specify # `cls.alias_groups` rather than having them be provided by bootstrap options. graph_session = EngineInitializer.setup_legacy_graph_extended( pants_ignore_patterns=None, workdir=cls._pants_workdir(), local_store_dir=cls._local_store_dir, build_file_imports_behavior='allow', native=init_native(), options_bootstrapper=OptionsBootstrapper.create(args=['--pants-config-files=[]']), build_configuration=cls.build_config(), build_ignore_patterns=None, ).new_session(zipkin_trace_v2=False) cls._scheduler = graph_session.scheduler_session cls._build_graph, cls._address_mapper = graph_session.create_build_graph( TargetRoots([]), cls._build_root() )
class SchedulerTest(unittest.TestCase): _native = init_native() def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root, self._native) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse( 'src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse( 'src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse( 'src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse( '3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse( '3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps') def tearDown(self): super(SchedulerTest, self).tearDown() # Without eagerly dropping this reference, each instance created for a test method # will live until all tests in this class have completed: can confirm by editing # the `scheduler_destroy` call in `src/python/pants/engine/native.py`. self.scheduler = None def parse_specs(self, *specs): return Specs(tuple( self.spec_parser.parse_spec(spec) for spec in specs)) def assert_select_for_subjects(self, walk, selector, subjects, variants=None): raise ValueError(walk) def build(self, execution_request): """Execute the given request and return roots as a list of ((subject, product), value) tuples.""" result = self.scheduler.execute(execution_request) self.assertIsNone(result.error) return result.root_products def request(self, products, *subjects): return self.scheduler.execution_request(products, subjects) def assert_root(self, root, subject, return_value): """Asserts that the given root has the given result.""" self.assertEquals(subject, root[0][0]) self.assertEquals(Return(return_value), root[1]) def assert_root_failed(self, root, subject, msg_str): """Asserts that the root was a Throw result containing the given msg string.""" self.assertEquals(subject, root[0][0]) self.assertEquals(Throw, type(root[1])) self.assertIn(msg_str, str(root[1].exc)) def test_compile_only_3rdparty(self): build_request = self.request([Classpath], self.guava) root, = self.build(build_request) self.assert_root(root, self.guava, Classpath(creator='ivy_resolve')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_compile_only_3rdparty_internal(self): build_request = self.request([Classpath], '3rdparty/jvm:guava') root, = self.build(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Select(Jar), [self.guava]) self.assert_select_for_subjects(walk, Select(Classpath), [self.guava]) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_gen(self): build_request = self.request([GenGoal], self.thrift) root, = self.build(build_request) # Root: expect the synthetic GenGoal product. self.assert_root( root, self.thrift, GenGoal("non-empty input to satisfy the Goal constructor")) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, SelectVariant( ApacheThriftJavaConfiguration, variant_key='thrift'), [self.thrift], variants=variants) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_codegen_simple(self): build_request = self.request([Classpath], self.java) root, = self.build(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'), Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'), Address.parse('src/thrift:slf4j-api') ] # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. self.assert_root(root, self.java, Classpath(creator='javac')) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Select(Classpath), subjects) self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = self.request([Classpath], self.consumes_resources) root, = self.build(build_request) self.assert_root(root, self.consumes_resources, Classpath(creator='javac')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_consumes_resources_internal(self): build_request = self.request([Classpath], self.consumes_resources) root, = self.build(build_request) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Select(Classpath), subjects) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request([Classpath], self.consumes_managed_thirdparty) root, = self.build(build_request) # Validate the root. self.assert_root(root, self.consumes_managed_thirdparty, Classpath(creator='javac')) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Select(Classpath), managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals( { Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0') }, {ret.value for node, ret in walk if node.product == Jar}) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request([Classpath], self.inferred_deps) root, = self.build(build_request) self.assert_root(root, self.inferred_deps, Classpath(creator='scalac')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_dependency_inference_internal(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request([Classpath], self.inferred_deps) root, = self.build(build_request) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request([Classpath], self.java_multi) root, = self.build(build_request) # Validate that the root failed. self.assert_root_failed(root, self.java_multi, "Conflicting values produced for") def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" specs = self.parse_specs('3rdparty/jvm::') build_request = self.scheduler.execution_request([BuildFileAddresses], [specs]) ((subject, _), root), = self.build(build_request) # Validate the root. self.assertEqual(specs, subject) self.assertEqual(BuildFileAddresses, type(root.value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root.value.dependencies) self.assertIn(self.managed_guava, root.value.dependencies) self.assertIn(self.managed_resolve_latest, root.value.dependencies) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" specs = self.parse_specs('3rdparty/jvm:') build_request = self.scheduler.execution_request([BuildFileAddresses], [specs]) ((subject, _), root), = self.build(build_request) # Validate the root. self.assertEqual(specs, subject) self.assertEqual(BuildFileAddresses, type(root.value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root.value.dependencies) # And that a subdirectory address is not. self.assertNotIn(self.managed_guava, root.value.dependencies) def test_scheduler_visualize(self): specs = self.parse_specs('3rdparty/jvm::') build_request = self.request([BuildFileAddresses], specs) self.build(build_request) with temporary_dir() as td: output_path = os.path.join(td, 'output.dot') self.scheduler.visualize_graph_to_file(output_path) with open(output_path, 'rb') as fh: graphviz_output = fh.read().strip() self.assertIn('digraph', graphviz_output) self.assertIn(' -> ', graphviz_output)
class GraphTestBase(unittest.TestCase): _native = init_native() def _make_setup_args(self, specs): options = mock.Mock(target_specs=specs) options.for_scope.return_value = mock.Mock(diffspec=None, changes_since=None) options.for_global_scope.return_value = mock.Mock(owner_of=None) return options def _default_build_config(self, options_bootstrapper, build_file_aliases=None): # TODO: Get default BuildFileAliases by extending BaseTest post # https://github.com/pantsbuild/pants/issues/4401 build_config = BuildConfigInitializer.get(options_bootstrapper) if build_file_aliases: build_config.register_aliases(build_file_aliases) return build_config @contextmanager def graph_helper(self, build_configuration=None, build_file_imports_behavior='allow', include_trace_on_error=True, path_ignore_patterns=None): with temporary_dir() as work_dir: with temporary_dir() as local_store_dir: path_ignore_patterns = path_ignore_patterns or [] options_bootstrapper = OptionsBootstrapper.create() build_config = build_configuration or self._default_build_config( options_bootstrapper) # TODO: This test should be swapped to using TestBase. graph_helper = EngineInitializer.setup_legacy_graph_extended( path_ignore_patterns, work_dir, local_store_dir, build_file_imports_behavior, options_bootstrapper=options_bootstrapper, build_configuration=build_config, native=self._native, include_trace_on_error=include_trace_on_error) yield graph_helper @contextmanager def open_scheduler(self, specs, build_configuration=None): with self.graph_helper( build_configuration=build_configuration) as graph_helper: graph, target_roots = self.create_graph_from_specs( graph_helper, specs) addresses = tuple(graph.inject_roots_closure(target_roots)) yield graph, addresses, graph_helper.scheduler.new_session() def create_graph_from_specs(self, graph_helper, specs): Subsystem.reset() session = graph_helper.new_session() target_roots = self.create_target_roots(specs, session, session.symbol_table) graph = session.create_build_graph(target_roots)[0] return graph, target_roots def create_target_roots(self, specs, session, symbol_table): return TargetRootsCalculator.create(self._make_setup_args(specs), session, symbol_table)
class SchedulerTestBase(object): """A mixin for classes (tests, presumably) which need to create temporary schedulers. TODO: In the medium term, this should be part of pants_test.base_test.BaseTest. """ _native = init_native() def _create_work_dir(self): work_dir = safe_mkdtemp() self.addCleanup(safe_rmtree, work_dir) return work_dir def mk_fs_tree(self, build_root_src=None, ignore_patterns=None, work_dir=None): """Create a temporary FilesystemProjectTree. :param build_root_src: Optional directory to pre-populate from; otherwise, empty. :returns: A FilesystemProjectTree. """ work_dir = work_dir or self._create_work_dir() build_root = os.path.join(work_dir, 'build_root') if build_root_src is not None: shutil.copytree(build_root_src, build_root, symlinks=True) else: os.makedirs(build_root) return FileSystemProjectTree(build_root, ignore_patterns=ignore_patterns) def mk_scheduler(self, rules=None, project_tree=None, work_dir=None, include_trace_on_error=True): """Creates a Scheduler with the given Rules installed.""" rules = rules or [] goals = {} work_dir = work_dir or self._create_work_dir() project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir) return LocalScheduler(work_dir, goals, rules, project_tree, self._native, include_trace_on_error=include_trace_on_error) def execute_request(self, scheduler, product, *subjects): """Creates, runs, and returns an ExecutionRequest for the given product and subjects.""" request = scheduler.execution_request([product], subjects) res = scheduler.execute(request) if res.error: raise res.error return request def execute(self, scheduler, product, *subjects): """Runs an ExecutionRequest for the given product and subjects, and returns the result value.""" request = self.execute_request(scheduler, product, *subjects) states = [state for _, state in scheduler.root_entries(request)] if any(type(state) is not Return for state in states): with temporary_file_path(cleanup=False, suffix='.dot') as dot_file: scheduler.visualize_graph_to_file(dot_file) raise ValueError( 'At least one request failed: {}. Visualized as {}'.format( states, dot_file)) return list(state.value for state in states)
def create_native_scheduler(self, root_subject_types, rules): rule_index = RuleIndex.create(rules) native = init_native() scheduler = WrappedNativeScheduler(native, '.', './.pants.d', [], rule_index, root_subject_types) return scheduler
class LegacyAddressMapperTest(unittest.TestCase): _native = init_native() def create_build_files(self, build_root): # Create BUILD files # build_root: # BUILD # BUILD.other # dir_a: # BUILD # BUILD.other # subdir: # BUILD # dir_b: # BUILD dir_a = os.path.join(build_root, 'dir_a') dir_b = os.path.join(build_root, 'dir_b') dir_a_subdir = os.path.join(dir_a, 'subdir') safe_mkdir(dir_a) safe_mkdir(dir_b) safe_mkdir(dir_a_subdir) safe_file_dump(os.path.join(build_root, 'BUILD'), 'target(name="a")\ntarget(name="b")') safe_file_dump(os.path.join(build_root, 'BUILD.other'), 'target(name="c")') safe_file_dump(os.path.join(dir_a, 'BUILD'), 'target(name="a")\ntarget(name="b")') safe_file_dump(os.path.join(dir_a, 'BUILD.other'), 'target(name="c")') safe_file_dump(os.path.join(dir_b, 'BUILD'), 'target(name="a")') safe_file_dump(os.path.join(dir_a_subdir, 'BUILD'), 'target(name="a")') def create_address_mapper(self, build_root): work_dir = os.path.join(build_root, '.pants.d') scheduler = EngineInitializer.setup_legacy_graph( [], work_dir, build_file_imports_behavior='allow', build_root=build_root, native=self._native).scheduler return LegacyAddressMapper(scheduler.new_session(), build_root) def test_is_valid_single_address(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) self.assertFalse( mapper.is_valid_single_address(SingleAddress('dir_a', 'foo'))) self.assertTrue( mapper.is_valid_single_address(SingleAddress('dir_a', 'a'))) with self.assertRaises(TypeError): mapper.is_valid_single_address('foo') def test_scan_build_files(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) build_files = mapper.scan_build_files('') self.assertEqual( build_files, { 'BUILD', 'BUILD.other', 'dir_a/BUILD', 'dir_a/BUILD.other', 'dir_b/BUILD', 'dir_a/subdir/BUILD' }) build_files = mapper.scan_build_files('dir_a/subdir') self.assertEqual(build_files, {'dir_a/subdir/BUILD'}) def test_scan_build_files_edge_cases(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) # A non-existent dir. build_files = mapper.scan_build_files('foo') self.assertEqual(build_files, set()) # A dir with no BUILD files. safe_mkdir(os.path.join(build_root, 'empty')) build_files = mapper.scan_build_files('empty') self.assertEqual(build_files, set()) def test_is_declaring_file(self): scheduler = mock.Mock() mapper = LegacyAddressMapper(scheduler, '') self.assertTrue( mapper.is_declaring_file(Address('path', 'name'), 'path/BUILD')) self.assertTrue( mapper.is_declaring_file(Address('path', 'name'), 'path/BUILD.suffix')) self.assertFalse( mapper.is_declaring_file(Address('path', 'name'), 'path/not_a_build_file')) self.assertFalse( mapper.is_declaring_file(Address('path', 'name'), 'differing-path/BUILD')) self.assertFalse( mapper.is_declaring_file( BuildFileAddress(target_name='name', rel_path='path/BUILD.new'), 'path/BUILD')) self.assertTrue( mapper.is_declaring_file( BuildFileAddress(target_name='name', rel_path='path/BUILD'), 'path/BUILD')) def test_addresses_in_spec_path(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) addresses = mapper.addresses_in_spec_path('dir_a') self.assertEqual( addresses, { Address('dir_a', 'a'), Address('dir_a', 'b'), Address('dir_a', 'c') }) def test_addresses_in_spec_path_no_dir(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) with self.assertRaises(AddressMapper.BuildFileScanError) as cm: mapper.addresses_in_spec_path('foo') self.assertIn('does not match any targets.', str(cm.exception)) def test_addresses_in_spec_path_no_build_files(self): with temporary_dir() as build_root: self.create_build_files(build_root) safe_mkdir(os.path.join(build_root, 'foo')) mapper = self.create_address_mapper(build_root) with self.assertRaises(AddressMapper.BuildFileScanError) as cm: mapper.addresses_in_spec_path('foo') self.assertIn('does not match any targets.', str(cm.exception)) def test_scan_specs(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) addresses = mapper.scan_specs( [SingleAddress('dir_a', 'a'), SiblingAddresses('')]) self.assertEqual( addresses, { Address('', 'a'), Address('', 'b'), Address('', 'c'), Address('dir_a', 'a') }) def test_scan_specs_bad_spec(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) with self.assertRaises(AddressMapper.BuildFileScanError) as cm: mapper.scan_specs([SingleAddress('dir_a', 'd')]) self.assertIn('does not match any targets.', str(cm.exception)) def test_scan_addresses(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) addresses = mapper.scan_addresses() self.assertEqual( addresses, { Address('', 'a'), Address('', 'b'), Address('', 'c'), Address('dir_a', 'a'), Address('dir_a', 'b'), Address('dir_a', 'c'), Address('dir_b', 'a'), Address('dir_a/subdir', 'a') }) def test_scan_addresses_with_root_specified(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) addresses = mapper.scan_addresses(os.path.join( build_root, 'dir_a')) self.assertEqual( addresses, { Address('dir_a', 'a'), Address('dir_a', 'b'), Address('dir_a', 'c'), Address('dir_a/subdir', 'a') }) def test_scan_addresses_bad_dir(self): # scan_addresses() should not raise an error. with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) addresses = mapper.scan_addresses(os.path.join(build_root, 'foo')) self.assertEqual(addresses, set()) def test_other_throw_is_fail(self): # scan_addresses() should raise an error if the scheduler returns an error it can't ignore. class ThrowReturningScheduler(object): def execution_request(self, *args): pass def execute(self, *args): return ExecutionResult( None, [(('some-thing', None), Throw(Exception('just an exception')))]) with temporary_dir() as build_root: mapper = LegacyAddressMapper(ThrowReturningScheduler(), build_root) with self.assertRaises( LegacyAddressMapper.BuildFileScanError) as cm: mapper.scan_addresses(os.path.join(build_root, 'foo')) self.assertIn('just an exception', str(cm.exception))
class LegacyAddressMapperTest(unittest.TestCase): _native = init_native() def create_build_files(self, build_root): # Create BUILD files # build_root: # BUILD # BUILD.other # dir_a: # BUILD # BUILD.other # subdir: # BUILD # dir_b: # BUILD dir_a = os.path.join(build_root, 'dir_a') dir_b = os.path.join(build_root, 'dir_b') dir_a_subdir = os.path.join(dir_a, 'subdir') safe_mkdir(dir_a) safe_mkdir(dir_b) safe_mkdir(dir_a_subdir) safe_file_dump(os.path.join(build_root, 'BUILD'), 'target(name="a")\ntarget(name="b")') safe_file_dump(os.path.join(build_root, 'BUILD.other'), 'target(name="c")') safe_file_dump(os.path.join(dir_a, 'BUILD'), 'target(name="a")\ntarget(name="b")') safe_file_dump(os.path.join(dir_a, 'BUILD.other'), 'target(name="c")') safe_file_dump(os.path.join(dir_b, 'BUILD'), 'target(name="a")') safe_file_dump(os.path.join(dir_a_subdir, 'BUILD'), 'target(name="a")') def create_address_mapper(self, build_root): scheduler, engine, _, _ = EngineInitializer.setup_legacy_graph( [], build_root=build_root, native=self._native) return LegacyAddressMapper(scheduler, engine, build_root) def test_is_valid_single_address(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) self.assertFalse( mapper.is_valid_single_address(SingleAddress('dir_a', 'foo'))) self.assertTrue( mapper.is_valid_single_address(SingleAddress('dir_a', 'a'))) with self.assertRaises(TypeError): mapper.is_valid_single_address('foo') def test_scan_build_files(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) build_files = mapper.scan_build_files('') self.assertEqual( build_files, { 'BUILD', 'BUILD.other', 'dir_a/BUILD', 'dir_a/BUILD.other', 'dir_b/BUILD', 'dir_a/subdir/BUILD' }) build_files = mapper.scan_build_files('dir_a/subdir') self.assertEqual(build_files, {'dir_a/subdir/BUILD'}) def test_scan_build_files_edge_cases(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) # A non-existent dir. build_files = mapper.scan_build_files('foo') self.assertEqual(build_files, set()) # A dir with no BUILD files. safe_mkdir(os.path.join(build_root, 'empty')) build_files = mapper.scan_build_files('empty') self.assertEqual(build_files, set()) def test_is_declaring_file(self): scheduler = mock.Mock() mapper = LegacyAddressMapper(scheduler, None, '') self.assertTrue( mapper.is_declaring_file(Address('path', 'name'), 'path/BUILD')) self.assertTrue( mapper.is_declaring_file(Address('path', 'name'), 'path/BUILD.suffix')) self.assertFalse( mapper.is_declaring_file(Address('path', 'name'), 'path/not_a_build_file')) self.assertFalse( mapper.is_declaring_file(Address('path', 'name'), 'differing-path/BUILD')) def test_addresses_in_spec_path(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) addresses = mapper.addresses_in_spec_path('dir_a') self.assertEqual( addresses, { Address('dir_a', 'a'), Address('dir_a', 'b'), Address('dir_a', 'c') }) def test_addresses_in_spec_path_no_dir(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) with self.assertRaises(AddressMapper.BuildFileScanError): mapper.addresses_in_spec_path('foo') # TODO: https://github.com/pantsbuild/pants/issues/4025 # self.assertIn('Directory "foo" does not exist.', str(cm.exception)) def test_addresses_in_spec_path_no_build_files(self): with temporary_dir() as build_root: self.create_build_files(build_root) safe_mkdir(os.path.join(build_root, 'foo')) mapper = self.create_address_mapper(build_root) with self.assertRaises(AddressMapper.BuildFileScanError): mapper.addresses_in_spec_path('foo') # TODO: https://github.com/pantsbuild/pants/issues/4025 # self.assertIn('does not contain build files.', str(cm.exception)) def test_scan_specs(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) addresses = mapper.scan_specs( [SingleAddress('dir_a', 'a'), SiblingAddresses('')]) self.assertEqual( addresses, { Address('', 'a'), Address('', 'b'), Address('', 'c'), Address('dir_a', 'a') }) def test_scan_specs_bad_spec(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) with self.assertRaises(AddressMapper.BuildFileScanError): mapper.scan_specs([SingleAddress('dir_a', 'd')]) # TODO: https://github.com/pantsbuild/pants/issues/4025 # self.assertIn('not found in namespace dir_a for name "d".', str(cm.exception)) def test_scan_addresses(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) addresses = mapper.scan_addresses() self.assertEqual( addresses, { Address('', 'a'), Address('', 'b'), Address('', 'c'), Address('dir_a', 'a'), Address('dir_a', 'b'), Address('dir_a', 'c'), Address('dir_b', 'a'), Address('dir_a/subdir', 'a') }) def test_scan_addresses_with_root_specified(self): with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) addresses = mapper.scan_addresses(os.path.join( build_root, 'dir_a')) self.assertEqual( addresses, { Address('dir_a', 'a'), Address('dir_a', 'b'), Address('dir_a', 'c'), Address('dir_a/subdir', 'a') }) def test_scan_addresses_bad_dir(self): # scan_addresses() should not raise an error. with temporary_dir() as build_root: self.create_build_files(build_root) mapper = self.create_address_mapper(build_root) addresses = mapper.scan_addresses(os.path.join(build_root, 'foo')) self.assertEqual(addresses, set())
class SchedulerTestBase: """A mixin for classes (tests, presumably) which need to create temporary schedulers. TODO: In the medium term, this should be part of pants_test.test_base.TestBase. """ _native = init_native() def _create_work_dir(self): work_dir = safe_mkdtemp() self.addCleanup(safe_rmtree, work_dir) return work_dir def mk_fs_tree(self, build_root_src=None, ignore_patterns=None, work_dir=None): """Create a temporary FilesystemProjectTree. :param build_root_src: Optional directory to pre-populate from; otherwise, empty. :returns: A FilesystemProjectTree. """ work_dir = work_dir or self._create_work_dir() build_root = os.path.join(work_dir, 'build_root') if build_root_src is not None: shutil.copytree(build_root_src, build_root, symlinks=True) else: os.makedirs(build_root) return FileSystemProjectTree(build_root, ignore_patterns=ignore_patterns) def mk_scheduler(self, rules=None, union_rules=None, project_tree=None, work_dir=None, include_trace_on_error=True): """Creates a SchedulerSession for a Scheduler with the given Rules installed.""" rules = rules or [] work_dir = work_dir or self._create_work_dir() project_tree = project_tree or self.mk_fs_tree(work_dir=work_dir) local_store_dir = os.path.realpath(safe_mkdtemp()) scheduler = Scheduler(self._native, project_tree, local_store_dir, rules, union_rules, DEFAULT_EXECUTION_OPTIONS, include_trace_on_error=include_trace_on_error) return scheduler.new_session(zipkin_trace_v2=False, build_id="buildid_for_test") def context_with_scheduler(self, scheduler, *args, **kwargs): return self.context(*args, scheduler=scheduler, **kwargs) def execute(self, scheduler, product, *subjects): """Runs an ExecutionRequest for the given product and subjects, and returns the result value.""" request = scheduler.execution_request([product], subjects) return self.execute_literal(scheduler, request) def execute_literal(self, scheduler, execution_request): returns, throws = scheduler.execute(execution_request) if throws: with temporary_file_path(cleanup=False, suffix='.dot') as dot_file: scheduler.visualize_graph_to_file(dot_file) raise ValueError( 'At least one root failed: {}. Visualized as {}'.format( throws, dot_file)) return list(state.value for _, state in returns) def execute_expecting_one_result(self, scheduler, product, subject): request = scheduler.execution_request([product], [subject]) returns, throws = scheduler.execute(request) if throws: _, state = throws[0] raise state.exc self.assertEqual(len(returns), 1) _, state = returns[0] return state def execute_raising_throw(self, scheduler, product, subject): resulting_value = self.execute_expecting_one_result( scheduler, product, subject) self.assertTrue(type(resulting_value) is Throw) raise resulting_value.exc
class GraphInvalidationTest(unittest.TestCase): _native = init_native() def _make_setup_args(self, specs): options = mock.Mock() options.target_specs = specs return options @contextmanager def open_scheduler(self, specs, symbol_table_cls=None): path_ignore_patterns = ['.*'] target_roots = TargetRoots.create(options=self._make_setup_args(specs)) graph_helper = EngineInitializer.setup_legacy_graph( path_ignore_patterns, symbol_table_cls=symbol_table_cls, native=self._native) graph = graph_helper.create_build_graph(target_roots)[0] addresses = tuple(graph.inject_specs_closure(target_roots.as_specs())) yield graph, addresses, graph_helper.scheduler def test_invalidate_fsnode(self): with self.open_scheduler(['3rdparty/python::']) as (_, _, scheduler): initial_node_count = scheduler.node_count() self.assertGreater(initial_node_count, 0) invalidated_count = scheduler.invalidate_files( ['3rdparty/python/BUILD']) self.assertGreater(invalidated_count, 0) self.assertLess(scheduler.node_count(), initial_node_count) def test_invalidate_fsnode_incremental(self): with self.open_scheduler(['//:', '3rdparty/::']) as (_, _, scheduler): node_count = scheduler.node_count() self.assertGreater(node_count, 0) # Invalidate the '3rdparty/python' DirectoryListing, the `3rdparty` DirectoryListing, # and then the root DirectoryListing by "touching" files/dirs. # NB: Invalidation of entries in the root directory is special: because Watchman will # never trigger an event for the root itself, we treat changes to files in the root # directory as events for the root. for filename in ('3rdparty/python/BUILD', '3rdparty/python', 'non_existing_file'): invalidated_count = scheduler.invalidate_files([filename]) self.assertGreater( invalidated_count, 0, 'File {} did not invalidate any Nodes.'.format(filename)) node_count, last_node_count = scheduler.node_count( ), node_count self.assertLess(node_count, last_node_count) def test_sources_ordering(self): spec = 'testprojects/src/resources/org/pantsbuild/testproject/ordering' with self.open_scheduler([spec]) as (graph, _, _): target = graph.get_target(Address.parse(spec)) sources = [ os.path.basename(s) for s in target.sources_relative_to_buildroot() ] self.assertEquals( ['p', 'a', 'n', 't', 's', 'b', 'u', 'i', 'l', 'd'], sources) def test_implicit_sources(self): expected_sources = { 'testprojects/tests/python/pants/file_sets:implicit_sources': ['a.py', 'aa.py', 'aaa.py', 'aabb.py', 'ab.py'], 'testprojects/tests/python/pants/file_sets:test_with_implicit_sources': ['test_a.py'] } for spec, exp_sources in expected_sources.items(): with self.open_scheduler([spec]) as (graph, _, _): target = graph.get_target(Address.parse(spec)) sources = sorted([ os.path.basename(s) for s in target.sources_relative_to_buildroot() ]) self.assertEquals(exp_sources, sources) def test_target_macro_override(self): """Tests that we can "wrap" an existing target type with additional functionality. Installs an additional TargetMacro that wraps `target` aliases to add a tag to all definitions. """ spec = 'testprojects/tests/python/pants/build_parsing:' # Confirm that python_tests in a small directory are marked. with self.open_scheduler( [spec], symbol_table_cls=TaggingSymbolTable) as (graph, addresses, _): self.assertTrue( len(addresses) > 0, 'No targets matched by {}'.format(addresses)) for address in addresses: self.assertIn(TaggingSymbolTable.tag, graph.get_target(address).tags)