def setup(options=None): if not options: options, _ = OptionsInitializer(OptionsBootstrapper()).setup() build_root = get_buildroot() cmd_line_spec_parser = CmdLineSpecParser(build_root) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs] storage = Storage.create(debug=False) # Ignore any dotfile below build_root except . itself project_tree = FileSystemProjectTree(build_root, ['.*', 'build-support/*.venv/']) symbol_table_cls = LegacyTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph # will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks() + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls) ) return ( LocalScheduler(dict(), tasks, storage, project_tree), storage, options, spec_roots, symbol_table_cls )
def setup(options=None): if not options: options, _ = OptionsInitializer(OptionsBootstrapper()).setup() build_root = get_buildroot() cmd_line_spec_parser = CmdLineSpecParser(build_root) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs] storage = Storage.create(debug=False) project_tree = FileSystemProjectTree(build_root) symbol_table_cls = LegacyTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph # will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks() + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls) ) return ( LocalScheduler(dict(), tasks, storage, project_tree), storage, options, spec_roots, symbol_table_cls )
def _run_and_check(self, target_specs, incremental_import=None): """ Invoke idea-plugin goal and check for target specs and project in the generated project and workspace file. :param target_specs: list of target specs :return: n/a """ self.assertTrue("targets are empty", target_specs) spec_parser = CmdLineSpecParser(get_buildroot()) # project_path is always the directory of the first target, # which is where intellij is going to zoom in under project view. project_path = spec_parser.parse_spec(target_specs[0]).directory with self.temporary_workdir() as workdir: with temporary_file(root_dir=workdir, cleanup=True) as output_file: args = [ 'idea-plugin', '--output-file={}'.format(output_file.name), '--no-open', ] if incremental_import is not None: args.append( '--incremental-import={}'.format(incremental_import)) pants_run = self.run_pants_with_workdir( args + target_specs, workdir) self.assert_success(pants_run) project_dir = self._get_project_dir(output_file.name) self.assertTrue(os.path.exists(project_dir), "{} does not exist".format(project_dir)) self._do_check(project_dir, project_path, target_specs, incremental_import=incremental_import)
def main_addresses(): build_root, goals, args = pop_build_root_and_goals( '[build root path] [goal]+ [address spec]*', sys.argv[1:]) cmd_line_spec_parser = CmdLineSpecParser(build_root) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in args] visualize_build_request(build_root, goals, spec_roots)
def _run_and_check(self, target_specs, incremental_import=None): """ Invoke idea-plugin goal and check for target specs and project in the generated project and workspace file. :param target_specs: list of target specs :return: n/a """ self.assertTrue("targets are empty", target_specs) spec_parser = CmdLineSpecParser(get_buildroot()) # project_path is always the directory of the first target, # which is where intellij is going to zoom in under project view. project_path = spec_parser.parse_spec(target_specs[0]).directory with self.temporary_workdir() as workdir: with temporary_file(root_dir=workdir, cleanup=True) as output_file: args = [ 'idea-plugin', '--output-file={}'.format(output_file.name), '--no-open', ] if incremental_import is not None: args.append('--incremental-import={}'.format(incremental_import)) pants_run = self.run_pants_with_workdir(args + target_specs, workdir) self.assert_success(pants_run) project_dir = self._get_project_dir(output_file.name) self.assertTrue(os.path.exists(project_dir), "{} does not exist".format(project_dir)) self._do_check(project_dir, project_path, target_specs, incremental_import=incremental_import)
def set_start_time(self, start_time: Optional[float]) -> None: # Launch RunTracker as early as possible (before .run() is called). self._run_tracker = RunTracker.global_instance() # Propagates parent_build_id to pants runs that may be called from this pants run. os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id self._reporting = Reporting.global_instance() self._reporting.initialize(self._run_tracker, self.options, start_time=start_time) spec_parser = CmdLineSpecParser(get_buildroot()) specs = [ spec_parser.parse_spec(spec).to_spec_string() for spec in self.options.specs ] # Note: This will not include values from `--changed-*` flags. self._run_tracker.run_info.add_info("specs_from_command_line", specs, stringify=False) # Capture a repro of the 'before' state for this build, if needed. self._repro = Reproducer.global_instance().create_repro() if self._repro: self._repro.capture(self._run_tracker.run_info.get_as_dict())
def parse_specs( cls, raw_specs: Iterable[str], build_root: Optional[str] = None, exclude_patterns: Optional[Iterable[str]] = None, tags: Optional[Iterable[str]] = None, ) -> Specs: """Parse raw string specs into a Specs object.""" build_root = build_root or get_buildroot() spec_parser = CmdLineSpecParser(build_root) address_specs: OrderedSet[AddressSpec] = OrderedSet() filesystem_specs: OrderedSet[FilesystemSpec] = OrderedSet() for spec_str in raw_specs: parsed_spec = spec_parser.parse_spec(spec_str) if isinstance(parsed_spec, AddressSpec): address_specs.add(parsed_spec) else: filesystem_specs.add(parsed_spec) address_specs_collection = AddressSpecs( dependencies=address_specs, exclude_patterns=exclude_patterns if exclude_patterns else tuple(), tags=tags, ) filesystem_specs_collection = FilesystemSpecs(filesystem_specs) return Specs( address_specs=address_specs_collection, filesystem_specs=filesystem_specs_collection, )
def main_addresses(): build_root, goals, args = pop_build_root_and_goals( '[build root path] [goal]+ [address spec]*', sys.argv[1:]) cmd_line_spec_parser = CmdLineSpecParser(build_root) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in args] visualize_build_request(build_root, goals, spec_roots)
def setup(): build_root = get_buildroot() cmd_line_spec_parser = CmdLineSpecParser(build_root) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in sys.argv[1:]] storage = Storage.create(debug=False) project_tree = FileSystemProjectTree(build_root) symbol_table_cls = LegacyTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph # will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks() + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls) ) return ( LocalScheduler(dict(), tasks, symbol_table_cls, project_tree), storage, spec_roots, symbol_table_cls )
def execute(self): # NOTE(pl): We now rely on the fact that we've scheduled Buildgen (the dummy task in the # buildgen goal) to run before the real buildgen tasks, e.g. buildgen-scala, buildgen-thrift, # etc. Since we are being run before the real tasks but after everything else upstream, # we can fix the target roots back up to be whatever the buildgen tasks are supposed to # operate on (instead of the entire build graph, which the upstream operated on). build_graph = self.context.build_graph bg_target_roots = set() # NOTE(mateo): Using all source roots adds a scan of 3rdparty - may have a minor perf penalty. # I would like to switch to the configured source roots if I can find a way to surface it everywhere. buildgen_dirs = self.buildgen_subsystem.source_dirs + self.buildgen_subsystem.test_dirs all_buildgen_specs = ['{}::'.format(d) for d in buildgen_dirs] spec_parser = CmdLineSpecParser(get_buildroot()) target_specs = self.context.options._target_specs or all_buildgen_specs parsed_specs = [ spec_parser.parse_spec(target_spec) for target_spec in target_specs ] for address in self.context.address_mapper.scan_specs(parsed_specs): build_graph.inject_address_closure(address) bg_target_roots.add(build_graph.get_target(address)) # Disable building from SCM for now, and instead always build everything unless the user # specifies less. # bg_target_roots.update( # target for target in # build_graph.transitive_dependees_of_addresses(t.address for t in self.targets_from_scm()) # if target.is_original # ) self.context._replace_targets(list(bg_target_roots))
def _run_and_check(self, address_specs, incremental_import=None): """Invoke idea-plugin goal and check for target specs and project in the generated project and workspace file. :param address_specs: list of address specs :return: n/a """ self.assertTrue(address_specs, "targets are empty") spec_parser = CmdLineSpecParser(get_buildroot()) # project_path is always the directory of the first target, # which is where intellij is going to zoom in under project view. spec = spec_parser.parse_spec(address_specs[0]) assert isinstance(spec, (SingleAddress, SiblingAddresses, DescendantAddresses)) project_path = spec.directory with self.temporary_workdir() as workdir: with temporary_file(root_dir=workdir, cleanup=True) as output_file: args = [ "idea-plugin", f"--output-file={output_file.name}", "--no-open", ] if incremental_import is not None: args.append(f"--incremental-import={incremental_import}") pants_run = self.run_pants_with_workdir(args + address_specs, workdir) self.assert_success(pants_run) project_dir = self._get_project_dir(output_file.name) self.assertTrue(os.path.exists(project_dir), f"{project_dir} does not exist") self._do_check( project_dir, project_path, address_specs, incremental_import=incremental_import )
def parse_specs(cls, target_specs, build_root=None, exclude_patterns=None, tags=None): """Parse string specs into unique `Spec` objects. :param iterable target_specs: An iterable of string specs. :param string build_root: The path to the build root. :returns: An `OrderedSet` of `Spec` objects. """ build_root = build_root or get_buildroot() spec_parser = CmdLineSpecParser(build_root) dependencies = tuple( OrderedSet( spec_parser.parse_spec(spec_str) for spec_str in target_specs)) if not dependencies: return None return [ Specs(dependencies=dependencies, exclude_patterns=exclude_patterns if exclude_patterns else tuple(), tags=tags) ]
def parse_specs(cls, target_specs, build_root=None): """Parse string specs into unique `Spec` objects. :param iterable target_specs: An iterable of string specs. :param string build_root: The path to the build root. :returns: An `OrderedSet` of `Spec` objects. """ build_root = build_root or get_buildroot() spec_parser = CmdLineSpecParser(build_root) return OrderedSet(spec_parser.parse_spec(spec_str) for spec_str in target_specs)
def parse_specs(cls, target_specs, build_root=None): """Parse string specs into unique `Spec` objects. :param iterable target_specs: An iterable of string specs. :param string build_root: The path to the build root. :returns: An `OrderedSet` of `Spec` objects. """ build_root = build_root or get_buildroot() spec_parser = CmdLineSpecParser(build_root) return OrderedSet(spec_parser.parse_spec(spec_str) for spec_str in target_specs)
def _determine_goals(self, address_mapper, requested_goals): """Check and populate the requested goals for a given run.""" spec_parser = CmdLineSpecParser(self._root_dir) for goal in requested_goals: if address_mapper.is_valid_single_address(spec_parser.parse_spec(goal)): logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) return [Goal.by_name(goal) for goal in requested_goals]
def _determine_goals(self, requested_goals): """Check and populate the requested goals for a given run.""" spec_parser = CmdLineSpecParser(self._root_dir) for goal in requested_goals: if self._address_mapper.is_valid_single_address(spec_parser.parse_spec(goal)): logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) goals = [Goal.by_name(goal) for goal in requested_goals] return goals
def parse_commandline_to_spec_roots(options=None, args=None, build_root=None): if not options: options, _ = OptionsInitializer(OptionsBootstrapper(args=args), init_logging=False).setup() cmd_line_spec_parser = CmdLineSpecParser(build_root or get_buildroot()) spec_roots = [ cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs ] return spec_roots
def _determine_goals(self, requested_goals): """Check and populate the requested goals for a given run.""" def is_quiet(goals): return any(goal.has_task_of_type(QuietTaskMixin) for goal in goals) or self._explain spec_parser = CmdLineSpecParser(self._root_dir) for goal in requested_goals: if self._address_mapper.is_valid_single_address(spec_parser.parse_spec(goal)): logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) goals = [Goal.by_name(goal) for goal in requested_goals] return goals, is_quiet(goals)
def _set_start_time(self, start_time: float) -> None: # Propagates parent_build_id to pants runs that may be called from this pants run. os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id self._run_tracker.start(self.options, run_start_time=start_time) spec_parser = CmdLineSpecParser(get_buildroot()) specs = [ str(spec_parser.parse_spec(spec)) for spec in self.options.specs ] # Note: This will not include values from `--changed-*` flags. self._run_tracker.run_info.add_info("specs_from_command_line", specs, stringify=False)
def _get_targets(spec_str): spec_parser = CmdLineSpecParser(get_buildroot()) try: spec = spec_parser.parse_spec(spec_str) addresses = self.context.address_mapper.scan_specs([spec]) except AddressLookupError as e: raise TaskError('Failed to parse address selector: {spec_str}\n {message}'.format(spec_str=spec_str, message=e)) # filter specs may not have been parsed as part of the context: force parsing matches = set() for address in addresses: self.context.build_graph.inject_address_closure(address) matches.add(self.context.build_graph.get_target(address)) if not matches: raise TaskError('No matches for address selector: {spec_str}'.format(spec_str=spec_str)) return matches
def _get_targets(spec_str): spec_parser = CmdLineSpecParser(get_buildroot()) try: spec = spec_parser.parse_spec(spec_str) addresses = self.context.address_mapper.scan_specs([spec]) except AddressLookupError as e: raise TaskError('Failed to parse address selector: {spec_str}\n {message}'.format(spec_str=spec_str, message=e)) # filter specs may not have been parsed as part of the context: force parsing matches = set() for address in addresses: self.context.build_graph.inject_address_closure(address) matches.add(self.context.build_graph.get_target(address)) if not matches: raise TaskError('No matches for address selector: {spec_str}'.format(spec_str=spec_str)) return matches
def parse_specs(cls, target_specs, build_root=None, exclude_patterns=None, tags=None): """Parse string specs into unique `Spec` objects. :param iterable target_specs: An iterable of string specs. :param string build_root: The path to the build root. :returns: A `Specs` object. """ build_root = build_root or get_buildroot() spec_parser = CmdLineSpecParser(build_root) dependencies = tuple(OrderedSet(spec_parser.parse_spec(spec_str) for spec_str in target_specs)) return Specs( dependencies=dependencies, exclude_patterns=exclude_patterns if exclude_patterns else tuple(), tags=tags)
def _expand_specs(self, spec_strs, fail_fast): """Populate the BuildGraph and target list from a set of input specs.""" with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag)) # Parse all specs into unique Spec objects. spec_parser = CmdLineSpecParser(self._root_dir) specs = OrderedSet() for spec_str in spec_strs: specs.add(spec_parser.parse_spec(spec_str)) # Then scan them to generate unique Addresses. for address in self._build_graph.inject_specs_closure(specs, fail_fast): target = self._build_graph.get_target(address) if tag_filter(target): self._targets.append(target)
def _expand_specs(self, spec_strs, fail_fast): """Populate the BuildGraph and target list from a set of input specs.""" with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag)) # Parse all specs into unique Spec objects. spec_parser = CmdLineSpecParser(self._root_dir) specs = OrderedSet() for spec_str in spec_strs: specs.add(spec_parser.parse_spec(spec_str)) # Then scan them to generate unique Addresses. for address in self._address_mapper.scan_specs(specs, fail_fast, self._spec_excludes): self._build_graph.inject_address_closure(address) target = self._build_graph.get_target(address) if tag_filter(target): self._targets.append(target)
def set_start_time(self, start_time): # Launch RunTracker as early as possible (before .run() is called). self._run_tracker = RunTracker.global_instance() self._reporting = Reporting.global_instance() self._run_start_time = start_time self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time) spec_parser = CmdLineSpecParser(get_buildroot()) target_specs = [ spec_parser.parse_spec(spec).to_spec_string() for spec in self._options.positional_args ] # Note: This will not include values from `--owner-of` or `--changed-*` flags. self._run_tracker.run_info.add_info("specs_from_command_line", target_specs, stringify=False) # Capture a repro of the 'before' state for this build, if needed. self._repro = Reproducer.global_instance().create_repro() if self._repro: self._repro.capture(self._run_tracker.run_info.get_as_dict())
class EngineTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.scheduler = setup_json_scheduler(build_root) self.spec_parser = CmdLineSpecParser(build_root) self.java = Address.parse('src/java/codegen/simple') def request(self, goals, *addresses): specs = [self.spec_parser.parse_spec(str(a)) for a in addresses] return BuildRequest(goals=goals, subjects=specs) def assert_engine(self, engine): result = engine.execute(self.request(['compile'], self.java)) self.assertEqual({SelectNode(self.java, Classpath, None, None): Return(Classpath(creator='javac'))}, result.root_products) self.assertIsNone(result.error) @contextmanager def multiprocessing_engine(self, pool_size=None): with closing(LocalMultiprocessEngine(self.scheduler, pool_size=pool_size, debug=True)) as e: yield e def test_serial_engine_simple(self): engine = LocalSerialEngine(self.scheduler) self.assert_engine(engine) def test_multiprocess_engine_multi(self): with self.multiprocessing_engine() as engine: self.assert_engine(engine) def test_multiprocess_engine_single(self): with self.multiprocessing_engine(pool_size=1) as engine: self.assert_engine(engine) def test_multiprocess_unpickleable(self): build_request = self.request(['unpickleable'], self.java) with self.multiprocessing_engine() as engine: with self.assertRaises(SerializationError): engine.execute(build_request)
def parse_specs(cls, raw_specs: Iterable[str], *, build_root: Optional[str] = None) -> Specs: """Parse raw string specs into a Specs object.""" build_root = build_root or get_buildroot() spec_parser = CmdLineSpecParser(build_root) address_specs: OrderedSet[AddressSpec] = OrderedSet() filesystem_specs: OrderedSet[FilesystemSpec] = OrderedSet() for spec_str in raw_specs: parsed_spec = spec_parser.parse_spec(spec_str) if isinstance(parsed_spec, AddressSpec): address_specs.add(parsed_spec) else: filesystem_specs.add(parsed_spec) return Specs( AddressSpecs(address_specs, filter_by_global_options=True), FilesystemSpecs(filesystem_specs), )
def create(cls, options=None, args=None, build_root=None, change_calculator=None): """ :param Options options: An `Options` instance to use, if available. :param string args: Raw cli args to use for parsing if an `Options` instance isn't available. :param string build_root: The build root. :param ChangeCalculator change_calculator: A `ChangeCalculator` for calculating changes. """ if not options: assert args is not None, 'must pass `args` if not passing `options`' options, _ = OptionsInitializer(OptionsBootstrapper(args=args)).setup(init_logging=False) # Determine the literal target roots. cmd_line_spec_parser = CmdLineSpecParser(build_root or get_buildroot()) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs] # Determine `Changed` arguments directly from options to support pre-`Subsystem` initialization paths. changed_options = options.for_scope('changed') changed_request = ChangedRequest.from_options(changed_options) logger.debug('args are: %s', args) logger.debug('spec_roots are: %s', spec_roots) logger.debug('changed_request is: %s', changed_request) if change_calculator and changed_request.is_actionable(): if spec_roots: # We've been provided spec roots (e.g. `./pants list ::`) AND a changed request. Error out. raise InvalidSpecConstraint('cannot provide changed parameters and target specs!') # We've been provided no spec roots (e.g. `./pants list`) AND a changed request. Compute # alternate target roots. changed_addresses = change_calculator.changed_target_addresses(changed_request) logger.debug('changed addresses: %s', changed_addresses) return ChangedTargetRoots(changed_addresses) # If no spec roots are passed, assume `::` as the intended target. return LiteralTargetRoots(spec_roots if spec_roots else [DescendantAddresses('')])
def parse_commandline_to_spec_roots(options=None, args=None, build_root=None): if not options: options, _ = OptionsInitializer(OptionsBootstrapper(args=args)).setup(init_logging=False) cmd_line_spec_parser = CmdLineSpecParser(build_root or get_buildroot()) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs] return spec_roots
class CmdLineSpecParserTest(BaseTest): def setUp(self): super(CmdLineSpecParserTest, self).setUp() self._spec_parser = CmdLineSpecParser(self.build_root) def test_normal(self): self.assert_parsed(':root', single('', 'root')) self.assert_parsed('//:root', single('', 'root')) self.assert_parsed('a', single('a')) self.assert_parsed('a:a', single('a', 'a')) self.assert_parsed('a/b', single('a/b')) self.assert_parsed('a/b:b', single('a/b', 'b')) self.assert_parsed('a/b:c', single('a/b', 'c')) def test_sibling(self): self.assert_parsed(':', sib('')) self.assert_parsed('//:', sib('')) self.assert_parsed('a:', sib('a')) self.assert_parsed('//a:', sib('a')) self.assert_parsed('a/b:', sib('a/b')) self.assert_parsed('//a/b:', sib('a/b')) def test_sibling_or_descendents(self): self.assert_parsed('::', desc('')) self.assert_parsed('//::', desc('')) self.assert_parsed('a::', desc('a')) self.assert_parsed('//a::', desc('a')) self.assert_parsed('a/b::', desc('a/b')) self.assert_parsed('//a/b::', desc('a/b')) def test_absolute(self): self.assert_parsed(os.path.join(self.build_root, 'a'), single('a')) self.assert_parsed(os.path.join(self.build_root, 'a:a'), single('a', 'a')) self.assert_parsed(os.path.join(self.build_root, 'a:'), sib('a')) self.assert_parsed(os.path.join(self.build_root, 'a::'), desc('a')) with self.assertRaises(CmdLineSpecParser.BadSpecError): self.assert_parsed('/not/the/buildroot/a', sib('a')) def test_absolute_double_slashed(self): # By adding a double slash, we are insisting that this absolute path is actually # relative to the buildroot. Thus, it should parse correctly. double_absolute = '/' + os.path.join(self.build_root, 'a') self.assertEquals('//', double_absolute[:2], 'A sanity check we have a leading-// absolute spec') self.assert_parsed(double_absolute, single(double_absolute[2:])) def test_cmd_line_affordances(self): self.assert_parsed('./:root', single('', 'root')) self.assert_parsed('//./:root', single('', 'root')) self.assert_parsed('//./a/../:root', single('', 'root')) self.assert_parsed(os.path.join(self.build_root, './a/../:root'), single('', 'root')) self.assert_parsed('a/', single('a')) self.assert_parsed('./a/', single('a')) self.assert_parsed(os.path.join(self.build_root, './a/'), single('a')) self.assert_parsed('a/b/:b', single('a/b', 'b')) self.assert_parsed('./a/b/:b', single('a/b', 'b')) self.assert_parsed(os.path.join(self.build_root, './a/b/:b'), single('a/b', 'b')) def assert_parsed(self, spec_str, expected_spec): self.assertEqual(self._spec_parser.parse_spec(spec_str), expected_spec)
class SchedulerTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), "examples", "scheduler_inputs") self.spec_parser = CmdLineSpecParser(build_root) self.scheduler, storage = setup_json_scheduler(build_root) self.storage = storage self.engine = LocalSerialEngine(self.scheduler, storage) self.guava = Address.parse("3rdparty/jvm:guava") self.thrift = Address.parse("src/thrift/codegen/simple") self.java = Address.parse("src/java/codegen/simple") self.java_simple = Address.parse("src/java/simple") self.java_multi = Address.parse("src/java/multiple_classpath_entries") self.no_variant_thrift = Address.parse("src/java/codegen/selector:conflict") self.unconfigured_thrift = Address.parse("src/thrift/codegen/unconfigured") self.resources = Address.parse("src/resources/simple") self.consumes_resources = Address.parse("src/java/consumes_resources") self.consumes_managed_thirdparty = Address.parse("src/java/managed_thirdparty") self.managed_guava = Address.parse("3rdparty/jvm/managed:guava") self.managed_hadoop = Address.parse("3rdparty/jvm/managed:hadoop-common") self.managed_resolve_latest = Address.parse("3rdparty/jvm/managed:latest-hadoop") self.inferred_deps = Address.parse("src/scala/inferred_deps") def assert_select_for_subjects(self, walk, product, subjects, variants=None, variant_key=None): node_type = SelectNode variants = tuple(variants.items()) if variants else None self.assertEqual( {node_type(subject, product, variants, variant_key) for subject in subjects}, { node for (node, _), _ in walk if node.product == product and isinstance(node, node_type) and node.variants == variants }, ) def build_and_walk(self, build_request, failures=False): """Build and then walk the given build_request, returning the walked graph as a list.""" predicate = (lambda _: True) if failures else None result = self.engine.execute(build_request) self.assertIsNone(result.error) return list(self.scheduler.product_graph.walk(build_request.roots, predicate=predicate)) def request(self, goals, *addresses): return self.request_specs(goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses]) def request_specs(self, goals, *specs): return self.scheduler.build_request(goals=goals, subjects=specs) def assert_resolve_only(self, goals, root_specs, jars): build_request = self.request(goals, *root_specs) walk = self.build_and_walk(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Jar, jars) self.assert_select_for_subjects(walk, Classpath, jars) def assert_root(self, walk, node, return_value): """Asserts that the first Node in a walk was a DependenciesNode with the single given result.""" ((root, root_state), dependencies) = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Return([return_value]), root_state) self.assertIn((node, Return(return_value)), dependencies) def assert_root_failed(self, walk, node, thrown_type): """Asserts that the first Node in a walk was a DependenciesNode with a Throw result.""" ((root, root_state), dependencies) = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Throw, type(root_state)) self.assertIn((node, thrown_type), [(k, type(v.exc)) for k, v in dependencies if type(v) is Throw]) def test_resolve(self): self.assert_resolve_only(goals=["resolve"], root_specs=["3rdparty/jvm:guava"], jars=[self.guava]) def test_compile_only_3rdparty(self): self.assert_resolve_only(goals=["compile"], root_specs=["3rdparty/jvm:guava"], jars=[self.guava]) def test_gen_noop(self): # TODO(John Sirois): Ask around - is this OK? # This is different than today. There is a gen'able target reachable from the java target, but # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's # handed is not thrift. build_request = self.request(["gen"], self.java) walk = self.build_and_walk(build_request) self.assert_select_for_subjects(walk, JavaSources, [self.java]) def test_gen(self): build_request = self.request(["gen"], self.thrift) walk = self.build_and_walk(build_request) # Root: expect the synthetic GenGoal product. self.assert_root( walk, SelectNode(self.thrift, GenGoal, None, None), GenGoal("non-empty input to satisfy the Goal constructor"), ) variants = {"thrift": "apache_java"} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, ThriftSources, [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects( walk, ApacheThriftJavaConfiguration, [self.thrift], variants=variants, variant_key="thrift" ) def test_codegen_simple(self): build_request = self.request(["compile"], self.java) walk = self.build_and_walk(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [self.guava, self.java, self.thrift] variant_subjects = [ Jar(org="org.apache.thrift", name="libthrift", rev="0.9.2", type_alias="jar"), Jar(org="commons-lang", name="commons-lang", rev="2.5", type_alias="jar"), Address.parse("src/thrift:slf4j-api"), ] # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. self.assert_root(walk, SelectNode(self.java, Classpath, None, None), Classpath(creator="javac")) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Classpath, subjects) self.assert_select_for_subjects(walk, Classpath, variant_subjects, variants={"thrift": "apache_java"}) def test_consumes_resources(self): build_request = self.request(["compile"], self.consumes_resources) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.consumes_resources, Classpath, None, None), Classpath(creator="javac")) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Classpath, subjects) def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request(["compile"], self.consumes_managed_thirdparty) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root( walk, SelectNode(self.consumes_managed_thirdparty, Classpath, None, None), Classpath(creator="javac") ) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Classpath, managed_jars, variants={"resolve": "latest-hadoop"}) # Confirm that the produced jars had the appropriate versions. self.assertEquals( {Jar("org.apache.hadoop", "hadoop-common", "2.7.0"), Jar("com.google.guava", "guava", "18.0")}, {ret.value for (node, ret), _ in walk if node.product == Jar and isinstance(node, SelectNode)}, ) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(["compile"], self.inferred_deps) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.inferred_deps, Classpath, None, None), Classpath(creator="scalac")) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Classpath, [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request(["compile"], self.java_multi) walk = self.build_and_walk(build_request, failures=True) # Validate that the root failed. self.assert_root_failed(walk, SelectNode(self.java_multi, Classpath, None, None), ConflictingProducersError) def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec("3rdparty/jvm::") build_request = self.request_specs(["list"], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0][0] root_value = root_state.value self.assertEqual(DependenciesNode(spec, Address, None, Addresses, None), root) self.assertEqual(list, type(root_value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root_value) self.assertIn(self.managed_guava, root_value) self.assertIn(self.managed_resolve_latest, root_value) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec("3rdparty/jvm:") build_request = self.request_specs(["list"], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0][0] root_value = root_state.value self.assertEqual(DependenciesNode(spec, Address, None, Addresses, None), root) self.assertEqual(list, type(root_value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root_value) # And that an subdirectory address is not. self.assertNotIn(self.managed_guava, root_value) def test_scheduler_visualize(self): spec = self.spec_parser.parse_spec("3rdparty/jvm:") build_request = self.request_specs(["list"], spec) self.build_and_walk(build_request) graphviz_output = "\n".join(self.scheduler.product_graph.visualize(build_request.roots)) with temporary_dir() as td: output_path = os.path.join(td, "output.dot") self.scheduler.visualize_graph_to_file(build_request.roots, output_path) with open(output_path, "rb") as fh: graphviz_disk_output = fh.read().strip() self.assertEqual(graphviz_output, graphviz_disk_output) self.assertIn("digraph", graphviz_output) self.assertIn(" -> ", graphviz_output)
class SchedulerTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root, inline_nodes=False) self.pg = self.scheduler.product_graph self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse( 'src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse( 'src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse( 'src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse( '3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse( '3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps') def assert_select_for_subjects(self, walk, selector, subjects, variants=None): node_type = SelectNode variants = tuple(variants.items()) if variants else None self.assertEqual( {node_type(subject, variants, selector) for subject in subjects}, { node for node, _ in walk if node.product == selector.product and isinstance(node, node_type) and node.variants == variants }) def build_and_walk(self, build_request): """Build and then walk the given build_request, returning the walked graph as a list.""" result = self.engine.execute(build_request) self.assertIsNone(result.error) return list(self.scheduler.product_graph.walk(build_request.roots)) def request(self, goals, *addresses): return self.request_specs( goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses]) def request_specs(self, goals, *specs): return self.scheduler.build_request(goals=goals, subjects=specs) def assert_resolve_only(self, goals, root_specs, jars): build_request = self.request(goals, *root_specs) walk = self.build_and_walk(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Select(Jar), jars) self.assert_select_for_subjects(walk, Select(Classpath), jars) def assert_root(self, walk, node, return_value): """Asserts that the first Node in a walk was a DependenciesNode with the single given result.""" root, root_state = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Return([return_value]), root_state) self.assertIn( (node, Return(return_value)), [(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)]) def assert_root_failed(self, walk, node, thrown_type): """Asserts that the first Node in a walk was a DependenciesNode with a Throw result.""" root, root_state = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Throw, type(root_state)) dependencies = [(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)] self.assertIn( (node, thrown_type), [(k, type(v.exc)) for k, v in dependencies if type(v) is Throw]) def test_type_error_on_unexpected_subject_type(self): with self.assertRaises(TypeError) as cm: self.scheduler.build_request(goals={}, subjects=['string']) self.assertEquals( "Unsupported root subject type: <type 'unicode'> for u'string'", str(cm.exception)) def test_resolve(self): self.assert_resolve_only(goals=['resolve'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_compile_only_3rdparty(self): self.assert_resolve_only(goals=['compile'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_gen_noop(self): # TODO(John Sirois): Ask around - is this OK? # This is different than today. There is a gen'able target reachable from the java target, but # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's # handed is not thrift. build_request = self.request(['gen'], self.java) walk = self.build_and_walk(build_request) self.assert_select_for_subjects(walk, Select(JavaSources, optional=True), [self.java]) def test_gen(self): build_request = self.request(['gen'], self.thrift) walk = self.build_and_walk(build_request) # Root: expect the synthetic GenGoal product. self.assert_root( walk, SelectNode(self.thrift, None, Select(GenGoal)), GenGoal("non-empty input to satisfy the Goal constructor")) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, SelectVariant( ApacheThriftJavaConfiguration, variant_key='thrift'), [self.thrift], variants=variants) def test_codegen_simple(self): build_request = self.request(['compile'], self.java) walk = self.build_and_walk(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'), Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'), Address.parse('src/thrift:slf4j-api') ] # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. self.assert_root(walk, SelectNode(self.java, None, Select(Classpath)), Classpath(creator='javac')) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Select(Classpath), subjects) self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = self.request(['compile'], self.consumes_resources) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root( walk, SelectNode(self.consumes_resources, None, Select(Classpath)), Classpath(creator='javac')) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Select(Classpath), subjects) def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request(['compile'], self.consumes_managed_thirdparty) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root( walk, SelectNode(self.consumes_managed_thirdparty, None, Select(Classpath)), Classpath(creator='javac')) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Select(Classpath), managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals( { Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0') }, { ret.value for node, ret in walk if node.product == Jar and isinstance(node, SelectNode) }) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root( walk, SelectNode(self.inferred_deps, None, Select(Classpath)), Classpath(creator='scalac')) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request(['compile'], self.java_multi) walk = self.build_and_walk(build_request) # Validate that the root failed. self.assert_root_failed( walk, SelectNode(self.java_multi, None, Select(Classpath)), ConflictingProducersError) def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm::') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0] root_value = root_state.value self.assertEqual( DependenciesNode( spec, None, SelectDependencies(Address, Addresses, field_types=(Address, ))), root) self.assertEqual(list, type(root_value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root_value) self.assertIn(self.managed_guava, root_value) self.assertIn(self.managed_resolve_latest, root_value) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0] root_value = root_state.value self.assertEqual( DependenciesNode( spec, None, SelectDependencies(Address, Addresses, field_types=(Address, ))), root) self.assertEqual(list, type(root_value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root_value) # And that an subdirectory address is not. self.assertNotIn(self.managed_guava, root_value) def test_scheduler_visualize(self): spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request_specs(['list'], spec) self.build_and_walk(build_request) graphviz_output = '\n'.join( self.scheduler.product_graph.visualize(build_request.roots)) with temporary_dir() as td: output_path = os.path.join(td, 'output.dot') self.scheduler.visualize_graph_to_file(build_request.roots, output_path) with open(output_path, 'rb') as fh: graphviz_disk_output = fh.read().strip() self.assertEqual(graphviz_output, graphviz_disk_output) self.assertIn('digraph', graphviz_output) self.assertIn(' -> ', graphviz_output)
class CmdLineSpecParserTest(TestBase): def setUp(self) -> None: super().setUp() self._spec_parser = CmdLineSpecParser(self.build_root) def test_normal_address_specs(self) -> None: self.assert_address_spec_parsed(":root", single("", "root")) self.assert_address_spec_parsed("//:root", single("", "root")) self.assert_address_spec_parsed("a", single("a")) self.assert_address_spec_parsed("a:a", single("a", "a")) self.assert_address_spec_parsed("a/b", single("a/b")) self.assert_address_spec_parsed("a/b:b", single("a/b", "b")) self.assert_address_spec_parsed("a/b:c", single("a/b", "c")) def test_sibling(self) -> None: self.assert_address_spec_parsed(":", sib("")) self.assert_address_spec_parsed("//:", sib("")) self.assert_address_spec_parsed("a:", sib("a")) self.assert_address_spec_parsed("//a:", sib("a")) self.assert_address_spec_parsed("a/b:", sib("a/b")) self.assert_address_spec_parsed("//a/b:", sib("a/b")) def test_descendant(self) -> None: self.assert_address_spec_parsed("::", desc("")) self.assert_address_spec_parsed("//::", desc("")) self.assert_address_spec_parsed("a::", desc("a")) self.assert_address_spec_parsed("//a::", desc("a")) self.assert_address_spec_parsed("a/b::", desc("a/b")) self.assert_address_spec_parsed("//a/b::", desc("a/b")) def test_files(self) -> None: # We assume that specs with an extension are meant to be interpreted as filesystem specs. for f in ["a.txt", "a.tmp.cache.txt.bak", "a/b/c.txt", ".a.txt"]: self.assert_filesystem_spec_parsed(f, literal(f)) self.assert_filesystem_spec_parsed("./a.txt", literal("a.txt")) self.assert_filesystem_spec_parsed("//./a.txt", literal("a.txt")) def test_globs(self) -> None: for glob_str in [ "*", "**/*", "a/b/*", "a/b/test_*.py", "a/b/**/test_*" ]: self.assert_filesystem_spec_parsed(glob_str, glob(glob_str)) def test_excludes(self) -> None: for glob_str in ["!", "!a/b/", "!/a/b/*"]: self.assert_filesystem_spec_parsed(glob_str, ignore(glob_str[1:])) def test_ambiguous_files(self) -> None: # These could either be files or the shorthand for single addresses. We check if they exist # on the file system to disambiguate. for spec in ["a", "b/c"]: self.assert_address_spec_parsed(spec, single(spec)) self.create_file(spec) self.assert_filesystem_spec_parsed(spec, literal(spec)) def test_absolute(self) -> None: self.assert_address_spec_parsed(os.path.join(self.build_root, "a"), single("a")) self.assert_address_spec_parsed(os.path.join(self.build_root, "a:a"), single("a", "a")) self.assert_address_spec_parsed(os.path.join(self.build_root, "a:"), sib("a")) self.assert_address_spec_parsed(os.path.join(self.build_root, "a::"), desc("a")) self.assert_filesystem_spec_parsed( os.path.join(self.build_root, "a.txt"), literal("a.txt")) with self.assertRaises(CmdLineSpecParser.BadSpecError): self.assert_address_spec_parsed("/not/the/buildroot/a", sib("a")) self.assert_filesystem_spec_parsed("/not/the/buildroot/a.txt", literal("a.txt")) def test_absolute_double_slashed(self) -> None: # By adding a double slash, we are insisting that this absolute path is actually # relative to the buildroot. Thus, it should parse correctly. double_absolute_address = "/" + os.path.join(self.build_root, "a") double_absolute_file = "/" + os.path.join(self.build_root, "a.txt") for spec in [double_absolute_address, double_absolute_file]: assert "//" == spec[:2] self.assert_address_spec_parsed(double_absolute_address, single(double_absolute_address[2:])) self.assert_filesystem_spec_parsed(double_absolute_file, literal(double_absolute_file[2:])) def test_cmd_line_affordances(self) -> None: self.assert_address_spec_parsed("./:root", single("", "root")) self.assert_address_spec_parsed("//./:root", single("", "root")) self.assert_address_spec_parsed("//./a/../:root", single("", "root")) self.assert_address_spec_parsed( os.path.join(self.build_root, "./a/../:root"), single("", "root")) self.assert_address_spec_parsed("a/", single("a")) self.assert_address_spec_parsed("./a/", single("a")) self.assert_address_spec_parsed(os.path.join(self.build_root, "./a/"), single("a")) self.assert_address_spec_parsed("a/b/:b", single("a/b", "b")) self.assert_address_spec_parsed("./a/b/:b", single("a/b", "b")) self.assert_address_spec_parsed( os.path.join(self.build_root, "./a/b/:b"), single("a/b", "b")) def assert_address_spec_parsed(self, spec_str: str, expected_spec: AddressSpec) -> None: spec = self._spec_parser.parse_spec(spec_str) assert isinstance(spec, AddressSpec) assert spec == expected_spec def assert_filesystem_spec_parsed(self, spec_str: str, expected_spec: FilesystemSpec) -> None: spec = self._spec_parser.parse_spec(spec_str) assert isinstance(spec, FilesystemSpec) assert spec == expected_spec
class SchedulerTest(unittest.TestCase): _native = init_native() def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root, self._native) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse( 'src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse( 'src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse( 'src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse( '3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse( '3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps') def tearDown(self): super(SchedulerTest, self).tearDown() # Without eagerly dropping this reference, each instance created for a test method # will live until all tests in this class have completed: can confirm by editing # the `scheduler_destroy` call in `src/python/pants/engine/native.py`. self.scheduler = None def parse_specs(self, *specs): return Specs(tuple( self.spec_parser.parse_spec(spec) for spec in specs)) def assert_select_for_subjects(self, walk, selector, subjects, variants=None): raise ValueError(walk) def build(self, execution_request): """Execute the given request and return roots as a list of ((subject, product), value) tuples.""" result = self.scheduler.execute(execution_request) self.assertIsNone(result.error) return result.root_products def request(self, products, *subjects): return self.scheduler.execution_request(products, subjects) def assert_root(self, root, subject, return_value): """Asserts that the given root has the given result.""" self.assertEquals(subject, root[0][0]) self.assertEquals(Return(return_value), root[1]) def assert_root_failed(self, root, subject, msg_str): """Asserts that the root was a Throw result containing the given msg string.""" self.assertEquals(subject, root[0][0]) self.assertEquals(Throw, type(root[1])) self.assertIn(msg_str, str(root[1].exc)) def test_compile_only_3rdparty(self): build_request = self.request([Classpath], self.guava) root, = self.build(build_request) self.assert_root(root, self.guava, Classpath(creator='ivy_resolve')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_compile_only_3rdparty_internal(self): build_request = self.request([Classpath], '3rdparty/jvm:guava') root, = self.build(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Select(Jar), [self.guava]) self.assert_select_for_subjects(walk, Select(Classpath), [self.guava]) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_gen(self): build_request = self.request([GenGoal], self.thrift) root, = self.build(build_request) # Root: expect the synthetic GenGoal product. self.assert_root( root, self.thrift, GenGoal("non-empty input to satisfy the Goal constructor")) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, SelectVariant( ApacheThriftJavaConfiguration, variant_key='thrift'), [self.thrift], variants=variants) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_codegen_simple(self): build_request = self.request([Classpath], self.java) root, = self.build(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'), Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'), Address.parse('src/thrift:slf4j-api') ] # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. self.assert_root(root, self.java, Classpath(creator='javac')) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Select(Classpath), subjects) self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = self.request([Classpath], self.consumes_resources) root, = self.build(build_request) self.assert_root(root, self.consumes_resources, Classpath(creator='javac')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_consumes_resources_internal(self): build_request = self.request([Classpath], self.consumes_resources) root, = self.build(build_request) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Select(Classpath), subjects) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request([Classpath], self.consumes_managed_thirdparty) root, = self.build(build_request) # Validate the root. self.assert_root(root, self.consumes_managed_thirdparty, Classpath(creator='javac')) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Select(Classpath), managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals( { Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0') }, {ret.value for node, ret in walk if node.product == Jar}) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request([Classpath], self.inferred_deps) root, = self.build(build_request) self.assert_root(root, self.inferred_deps, Classpath(creator='scalac')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_dependency_inference_internal(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request([Classpath], self.inferred_deps) root, = self.build(build_request) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request([Classpath], self.java_multi) root, = self.build(build_request) # Validate that the root failed. self.assert_root_failed(root, self.java_multi, "Conflicting values produced for") def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" specs = self.parse_specs('3rdparty/jvm::') build_request = self.scheduler.execution_request([BuildFileAddresses], [specs]) ((subject, _), root), = self.build(build_request) # Validate the root. self.assertEqual(specs, subject) self.assertEqual(BuildFileAddresses, type(root.value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root.value.dependencies) self.assertIn(self.managed_guava, root.value.dependencies) self.assertIn(self.managed_resolve_latest, root.value.dependencies) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" specs = self.parse_specs('3rdparty/jvm:') build_request = self.scheduler.execution_request([BuildFileAddresses], [specs]) ((subject, _), root), = self.build(build_request) # Validate the root. self.assertEqual(specs, subject) self.assertEqual(BuildFileAddresses, type(root.value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root.value.dependencies) # And that a subdirectory address is not. self.assertNotIn(self.managed_guava, root.value.dependencies) def test_scheduler_visualize(self): specs = self.parse_specs('3rdparty/jvm::') build_request = self.request([BuildFileAddresses], specs) self.build(build_request) with temporary_dir() as td: output_path = os.path.join(td, 'output.dot') self.scheduler.visualize_graph_to_file(build_request, output_path) with open(output_path, 'rb') as fh: graphviz_output = fh.read().strip() self.assertIn('digraph', graphviz_output) self.assertIn(' -> ', graphviz_output)
class SchedulerTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root) self.subjects = self.scheduler._subjects self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps') def key(self, subject): return self.subjects.put(subject) def assert_select_for_subjects(self, walk, product, subjects, variants=None, variant_key=None): node_type = SelectNode variants = tuple(variants.items()) if variants else None self.assertEqual({node_type(self.key(subject), product, variants, variant_key) for subject in subjects}, {node for (node, _), _ in walk if node.product == product and isinstance(node, node_type) and node.variants == variants}) def build_and_walk(self, build_request, failures=False): """Build and then walk the given build_request, returning the walked graph as a list.""" predicate = (lambda _: True) if failures else None result = self.engine.execute(build_request) self.assertIsNone(result.error) return list(self.scheduler.product_graph.walk(build_request.roots, predicate=predicate)) def request(self, goals, *addresses): return self.request_specs(goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses]) def request_specs(self, goals, *specs): return self.scheduler.build_request(goals=goals, subjects=specs) def assert_resolve_only(self, goals, root_specs, jars): build_request = self.request(goals, *root_specs) walk = self.build_and_walk(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Jar, jars) self.assert_select_for_subjects(walk, Classpath, jars) def assert_root(self, walk, node, return_value): """Asserts that the first Node in a walk was a DependenciesNode with the single given result.""" ((root, root_state), dependencies) = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Return([return_value]), root_state) self.assertIn((node, Return(return_value)), dependencies) def assert_root_failed(self, walk, node, thrown_type): """Asserts that the first Node in a walk was a DependenciesNode with a Throw result.""" ((root, root_state), dependencies) = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Throw, type(root_state)) self.assertIn((node, thrown_type), [(k, type(v.exc)) for k, v in dependencies if type(v) is Throw]) def test_resolve(self): self.assert_resolve_only(goals=['resolve'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_compile_only_3rdparty(self): self.assert_resolve_only(goals=['compile'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_gen_noop(self): # TODO(John Sirois): Ask around - is this OK? # This is different than today. There is a gen'able target reachable from the java target, but # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's # handed is not thrift. build_request = self.request(['gen'], self.java) walk = self.build_and_walk(build_request) self.assert_select_for_subjects(walk, JavaSources, [self.java]) def test_gen(self): build_request = self.request(['gen'], self.thrift) walk = self.build_and_walk(build_request) # Root: expect the synthetic GenGoal product. self.assert_root(walk, SelectNode(self.key(self.thrift), GenGoal, None, None), GenGoal("non-empty input to satisfy the Goal constructor")) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, ThriftSources, [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, ApacheThriftJavaConfiguration, [self.thrift], variants=variants, variant_key='thrift') def test_codegen_simple(self): build_request = self.request(['compile'], self.java) walk = self.build_and_walk(build_request) # TODO: Utter insanity. Pickle only encodes a unique object (ie, `if A is B`) a single # time on the wire. Because the copy of this object that we're comparing to is coming # from a file, the string will be encoded twice. Thus, to match (with pickle) we need # to ensure that `(cl1 is not cl2)` here. See: # https://github.com/pantsbuild/pants/issues/2969 cl1 = 'commons-lang' cl2 = 'commons' + '-lang' # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [ self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'), Jar(org=cl1, name=cl2, rev='2.5', type_alias='jar'), Address.parse('src/thrift:slf4j-api')] # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. self.assert_root(walk, SelectNode(self.key(self.java), Classpath, None, None), Classpath(creator='javac')) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Classpath, subjects) self.assert_select_for_subjects(walk, Classpath, variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = self.request(['compile'], self.consumes_resources) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.key(self.consumes_resources), Classpath, None, None), Classpath(creator='javac')) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Classpath, subjects) def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request(['compile'], self.consumes_managed_thirdparty) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.key(self.consumes_managed_thirdparty), Classpath, None, None), Classpath(creator='javac')) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Classpath, managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0')}, {ret.value for (node, ret), _ in walk if node.product == Jar and isinstance(node, SelectNode)}) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.key(self.inferred_deps), Classpath, None, None), Classpath(creator='scalac')) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Classpath, [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request(['compile'], self.java_multi) walk = self.build_and_walk(build_request, failures=True) # Validate that the root failed. self.assert_root_failed(walk, SelectNode(self.key(self.java_multi), Classpath, None, None), ConflictingProducersError) def test_no_variant_thrift(self): """No `thrift` variant is configured, and so no configuration is selected.""" build_request = self.request(['compile'], self.no_variant_thrift) with self.assertRaises(PartiallyConsumedInputsError): self.build_and_walk(build_request) def test_unconfigured_thrift(self): """The BuildPropertiesPlanner is able to produce a Classpath, but we should still fail. A target with ThriftSources doesn't have a thrift config: that input is partially consumed. """ build_request = self.request(['compile'], self.unconfigured_thrift) with self.assertRaises(PartiallyConsumedInputsError): self.build_and_walk(build_request) def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm::') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0][0] root_value = root_state.value self.assertEqual(DependenciesNode(self.key(spec), Address, None, Addresses, None), root) self.assertEqual(list, type(root_value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root_value) self.assertIn(self.managed_guava, root_value) self.assertIn(self.managed_resolve_latest, root_value) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0][0] root_value = root_state.value self.assertEqual(DependenciesNode(self.key(spec), Address, None, Addresses, None), root) self.assertEqual(list, type(root_value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root_value) # And that an subdirectory address is not. self.assertNotIn(self.managed_guava, root_value)
class SchedulerTest(unittest.TestCase): _native = init_native() def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root, self._native) self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps') def assert_select_for_subjects(self, walk, selector, subjects, variants=None): raise ValueError(walk) def build(self, build_request): """Execute the given request and return roots as a list of ((subject, product), value) tuples.""" result = self.engine.execute(build_request) self.assertIsNone(result.error) return self.scheduler.root_entries(build_request).items() def request(self, goals, *subjects): return self.scheduler.build_request(goals=goals, subjects=subjects) def assert_root(self, root, subject, return_value): """Asserts that the given root has the given result.""" self.assertEquals(subject, root[0][0]) self.assertEquals(Return(return_value), root[1]) def assert_root_failed(self, root, subject, msg_str): """Asserts that the root was a Throw result containing the given msg string.""" self.assertEquals(subject, root[0][0]) self.assertEquals(Throw, type(root[1])) self.assertIn(msg_str, str(root[1].exc)) def test_compile_only_3rdparty(self): build_request = self.request(['compile'], self.guava) root, = self.build(build_request) self.assert_root(root, self.guava, Classpath(creator='ivy_resolve')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_compile_only_3rdparty_internal(self): build_request = self.request(['compile'], '3rdparty/jvm:guava') root, = self.build(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Select(Jar), [self.guava]) self.assert_select_for_subjects(walk, Select(Classpath), [self.guava]) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_gen(self): build_request = self.request(['gen'], self.thrift) root, = self.build(build_request) # Root: expect the synthetic GenGoal product. self.assert_root(root, self.thrift, GenGoal("non-empty input to satisfy the Goal constructor")) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, SelectVariant(ApacheThriftJavaConfiguration, variant_key='thrift'), [self.thrift], variants=variants) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_codegen_simple(self): build_request = self.request(['compile'], self.java) root, = self.build(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'), Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'), Address.parse('src/thrift:slf4j-api')] # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. self.assert_root(root, self.java, Classpath(creator='javac')) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Select(Classpath), subjects) self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = self.request(['compile'], self.consumes_resources) root, = self.build(build_request) self.assert_root(root, self.consumes_resources, Classpath(creator='javac')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_consumes_resources_internal(self): build_request = self.request(['compile'], self.consumes_resources) root, = self.build(build_request) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Select(Classpath), subjects) @unittest.skip('Skipped to expedite landing #3821; see: #4020.') def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request(['compile'], self.consumes_managed_thirdparty) root, = self.build(build_request) # Validate the root. self.assert_root(root, self.consumes_managed_thirdparty, Classpath(creator='javac')) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Select(Classpath), managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0')}, {ret.value for node, ret in walk if node.product == Jar}) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) root, = self.build(build_request) self.assert_root(root, self.inferred_deps, Classpath(creator='scalac')) @unittest.skip('Skipped to expedite landing #3821; see: #4027.') def test_dependency_inference_internal(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) root, = self.build(build_request) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request(['compile'], self.java_multi) root, = self.build(build_request) # Validate that the root failed. self.assert_root_failed(root, self.java_multi, "Conflicting values produced for") def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm::') selector = Select(BuildFileAddresses) build_request = self.scheduler.selection_request([(selector, spec)]) ((subject, _), root), = self.build(build_request) # Validate the root. self.assertEqual(spec, subject) self.assertEqual(BuildFileAddresses, type(root.value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root.value.dependencies) self.assertIn(self.managed_guava, root.value.dependencies) self.assertIn(self.managed_resolve_latest, root.value.dependencies) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm:') selector = Select(BuildFileAddresses) build_request = self.scheduler.selection_request([(selector,spec)]) ((subject, _), root), = self.build(build_request) # Validate the root. self.assertEqual(spec, subject) self.assertEqual(BuildFileAddresses, type(root.value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root.value.dependencies) # And that a subdirectory address is not. self.assertNotIn(self.managed_guava, root.value.dependencies) def test_scheduler_visualize(self): spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request(['list'], spec) self.build(build_request) with temporary_dir() as td: output_path = os.path.join(td, 'output.dot') self.scheduler.visualize_graph_to_file(output_path) with open(output_path, 'rb') as fh: graphviz_output = fh.read().strip() self.assertIn('digraph', graphviz_output) self.assertIn(' -> ', graphviz_output)
class BuildFileAddressMapperScanTest(BaseTest): NO_FAIL_FAST_RE = re.compile(r"""^-------------------- .* Exception message: name 'a_is_bad' is not defined while executing BUILD file BuildFile\(bad/a/BUILD, FileSystemProjectTree\(.*\)\) Loading addresses from 'bad/a' failed\. .* Exception message: name 'b_is_bad' is not defined while executing BUILD file BuildFile\(bad/b/BUILD, FileSystemProjectTree\(.*\)\) Loading addresses from 'bad/b' failed\. Invalid BUILD files for \[::\]$""", re.DOTALL) FAIL_FAST_RE = """^name 'a_is_bad' is not defined while executing BUILD file BuildFile\(bad/a/BUILD\, FileSystemProjectTree\(.*\)\) Loading addresses from 'bad/a' failed.$""" def setUp(self): super(BuildFileAddressMapperScanTest, self).setUp() def add_target(path, name): self.add_to_build_file(path, 'target(name="{name}")\n'.format(name=name)) add_target('BUILD', 'root') add_target('a', 'a') add_target('a', 'b') add_target('a/b', 'b') add_target('a/b', 'c') self._spec_parser = CmdLineSpecParser(self.build_root) def test_bad_build_files(self): self.add_to_build_file('bad/a', 'a_is_bad') self.add_to_build_file('bad/b', 'b_is_bad') with self.assertRaisesRegexp(AddressLookupError, self.NO_FAIL_FAST_RE): list(self.address_mapper.scan_specs([DescendantAddresses('')], fail_fast=False)) def test_bad_build_files_fail_fast(self): self.add_to_build_file('bad/a', 'a_is_bad') self.add_to_build_file('bad/b', 'b_is_bad') with self.assertRaisesRegexp(AddressLookupError, self.FAIL_FAST_RE): list(self.address_mapper.scan_specs([DescendantAddresses('')], fail_fast=True)) def test_normal(self): self.assert_scanned([':root'], expected=[':root']) self.assert_scanned(['//:root'], expected=[':root']) self.assert_scanned(['a'], expected=['a']) self.assert_scanned(['a:a'], expected=['a']) self.assert_scanned(['a/b'], expected=['a/b']) self.assert_scanned(['a/b:b'], expected=['a/b']) self.assert_scanned(['a/b:c'], expected=['a/b:c']) def test_sibling(self): self.assert_scanned([':'], expected=[':root']) self.assert_scanned(['//:'], expected=[':root']) self.assert_scanned(['a:'], expected=['a', 'a:b']) self.assert_scanned(['//a:'], expected=['a', 'a:b']) self.assert_scanned(['a/b:'], expected=['a/b', 'a/b:c']) self.assert_scanned(['//a/b:'], expected=['a/b', 'a/b:c']) def test_sibling_or_descendents(self): self.assert_scanned(['::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c']) self.assert_scanned(['//::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c']) self.assert_scanned(['a::'], expected=['a', 'a:b', 'a/b', 'a/b:c']) self.assert_scanned(['//a::'], expected=['a', 'a:b', 'a/b', 'a/b:c']) self.assert_scanned(['a/b::'], expected=['a/b', 'a/b:c']) self.assert_scanned(['//a/b::'], expected=['a/b', 'a/b:c']) def test_cmd_line_affordances(self): self.assert_scanned(['./:root'], expected=[':root']) self.assert_scanned(['//./:root'], expected=[':root']) self.assert_scanned(['//./a/../:root'], expected=[':root']) self.assert_scanned([os.path.join(self.build_root, './a/../:root')], expected=[':root']) self.assert_scanned(['a/'], expected=['a']) self.assert_scanned(['./a/'], expected=['a']) self.assert_scanned([os.path.join(self.build_root, './a/')], expected=['a']) self.assert_scanned(['a/b/:b'], expected=['a/b']) self.assert_scanned(['./a/b/:b'], expected=['a/b']) self.assert_scanned([os.path.join(self.build_root, './a/b/:b')], expected=['a/b']) def test_cmd_line_spec_list(self): self.assert_scanned(['a', 'a/b'], expected=['a', 'a/b']) self.assert_scanned(['::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c']) def test_does_not_exist(self): with self.assertRaises(AddressLookupError): self.assert_scanned(['c'], expected=[]) with self.assertRaises(AddressLookupError): self.assert_scanned(['c:'], expected=[]) with self.assertRaises(AddressLookupError): self.assert_scanned(['c::'], expected=[]) def test_build_ignore_patterns(self): expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c'] # This bogus BUILD file gets in the way of parsing. self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n') with self.assertRaises(AddressLookupError): self.assert_scanned(['::'], expected=expected_specs) address_mapper_with_ignore = BuildFileAddressMapper(self.build_file_parser, self.project_tree, build_ignore_patterns=['some']) self.assert_scanned(['::'], expected=expected_specs, address_mapper=address_mapper_with_ignore) def test_exclude_target_regexps(self): address_mapper_with_exclude = BuildFileAddressMapper(self.build_file_parser, self.project_tree, exclude_target_regexps=[r'.*:b.*']) self.assert_scanned(['::'], expected=[':root', 'a', 'a/b:c'], address_mapper=address_mapper_with_exclude) def assert_scanned(self, specs_strings, expected, address_mapper=None): """Parse and scan the given specs.""" address_mapper = address_mapper or self.address_mapper def sort(addresses): return sorted(addresses, key=lambda address: address.spec) specs = [self._spec_parser.parse_spec(s) for s in specs_strings] self.assertEqual(sort(Address.parse(addr) for addr in expected), sort(address_mapper.scan_specs(specs)))
class CmdLineSpecParserTest(BaseTest): def setUp(self): super(CmdLineSpecParserTest, self).setUp() self._spec_parser = CmdLineSpecParser(self.build_root) def test_normal(self): self.assert_parsed(':root', single('', 'root')) self.assert_parsed('//:root', single('', 'root')) self.assert_parsed('a', single('a')) self.assert_parsed('a:a', single('a', 'a')) self.assert_parsed('a/b', single('a/b')) self.assert_parsed('a/b:b', single('a/b', 'b')) self.assert_parsed('a/b:c', single('a/b', 'c')) def test_sibling(self): self.assert_parsed(':', sib('')) self.assert_parsed('//:', sib('')) self.assert_parsed('a:', sib('a')) self.assert_parsed('//a:', sib('a')) self.assert_parsed('a/b:', sib('a/b')) self.assert_parsed('//a/b:', sib('a/b')) def test_sibling_or_descendents(self): self.assert_parsed('::', desc('')) self.assert_parsed('//::', desc('')) self.assert_parsed('a::', desc('a')) self.assert_parsed('//a::', desc('a')) self.assert_parsed('a/b::', desc('a/b')) self.assert_parsed('//a/b::', desc('a/b')) def test_absolute(self): self.assert_parsed(os.path.join(self.build_root, 'a'), single('a')) self.assert_parsed(os.path.join(self.build_root, 'a:a'), single('a', 'a')) self.assert_parsed(os.path.join(self.build_root, 'a:'), sib('a')) self.assert_parsed(os.path.join(self.build_root, 'a::'), desc('a')) with self.assertRaises(CmdLineSpecParser.BadSpecError): self.assert_parsed('/not/the/buildroot/a', sib('a')) def test_absolute_double_slashed(self): # By adding a double slash, we are insisting that this absolute path is actually # relative to the buildroot. Thus, it should parse correctly. double_absolute = '/' + os.path.join(self.build_root, 'a') self.assertEquals('//', double_absolute[:2], 'A sanity check we have a leading-// absolute spec') self.assert_parsed(double_absolute, single(double_absolute[2:])) def test_cmd_line_affordances(self): self.assert_parsed('./:root', single('', 'root')) self.assert_parsed('//./:root', single('', 'root')) self.assert_parsed('//./a/../:root', single('', 'root')) self.assert_parsed(os.path.join(self.build_root, './a/../:root'), single('', 'root')) self.assert_parsed('a/', single('a')) self.assert_parsed('./a/', single('a')) self.assert_parsed(os.path.join(self.build_root, './a/'), single('a')) self.assert_parsed('a/b/:b', single('a/b', 'b')) self.assert_parsed('./a/b/:b', single('a/b', 'b')) self.assert_parsed(os.path.join(self.build_root, './a/b/:b'), single('a/b', 'b')) def assert_parsed(self, spec_str, expected_spec): self.assertEqual(self._spec_parser.parse_spec(spec_str), expected_spec)
class SchedulerTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root) self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse( 'src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse( 'src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse( 'src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse( '3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse( '3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps') def assert_select_for_subjects(self, walk, product, subjects, variants=None, variant_key=None): node_type = SelectNode variants = tuple(variants.items()) if variants else None self.assertEqual( { node_type(subject, product, variants, variant_key) for subject in subjects }, { node for (node, _), _ in walk if node.product == product and isinstance(node, node_type) and node.variants == variants }) def build_and_walk(self, build_request, failures=False): """Build and then walk the given build_request, returning the walked graph as a list.""" predicate = (lambda _: True) if failures else None result = self.engine.execute(build_request) self.assertIsNone(result.error) return list( self.scheduler.walk_product_graph(build_request, predicate=predicate)) def request(self, goals, *addresses): return self.request_specs( goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses]) def request_specs(self, goals, *specs): return BuildRequest(goals=goals, subjects=specs) def assert_resolve_only(self, goals, root_specs, jars): build_request = self.request(goals, *root_specs) walk = self.build_and_walk(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Jar, jars) self.assert_select_for_subjects(walk, Classpath, jars) def test_resolve(self): self.assert_resolve_only(goals=['resolve'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_compile_only_3rdparty(self): self.assert_resolve_only(goals=['compile'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_gen_noop(self): # TODO(John Sirois): Ask around - is this OK? # This is different than today. There is a gen'able target reachable from the java target, but # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's # handed is not thrift. build_request = self.request(['gen'], self.java) walk = self.build_and_walk(build_request) self.assert_select_for_subjects(walk, JavaSources, [self.java]) def test_gen(self): build_request = self.request(['gen'], self.thrift) walk = self.build_and_walk(build_request) # Root: expect the synthetic GenGoal product. root_entry = walk[0][0] self.assertEqual(SelectNode(self.thrift, GenGoal, None, None), root_entry[0]) self.assertIsInstance(root_entry[1], Return) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, ThriftSources, [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, ApacheThriftJavaConfiguration, [self.thrift], variants=variants, variant_key='thrift') def test_codegen_simple(self): build_request = self.request(['compile'], self.java) walk = self.build_and_walk(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2'), Jar(org='commons-lang', name='commons-lang', rev='2.5'), Address.parse('src/thrift:slf4j-api') ] # Root: expect compilation via javac. self.assertEqual( (SelectNode(self.java, Classpath, None, None), Return(Classpath(creator='javac'))), walk[0][0]) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Classpath, subjects) self.assert_select_for_subjects(walk, Classpath, variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = self.request(['compile'], self.consumes_resources) walk = self.build_and_walk(build_request) # Validate the root. self.assertEqual( (SelectNode(self.consumes_resources, Classpath, None, None), Return(Classpath(creator='javac'))), walk[0][0]) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Classpath, subjects) def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request(['compile'], self.consumes_managed_thirdparty) walk = self.build_and_walk(build_request) # Validate the root. self.assertEqual( (SelectNode(self.consumes_managed_thirdparty, Classpath, None, None), Return(Classpath(creator='javac'))), walk[0][0]) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Classpath, [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Classpath, managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals( { Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0') }, { ret.value for (node, ret), _ in walk if node.product == Jar and isinstance(node, SelectNode) }) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) walk = self.build_and_walk(build_request) # Validate the root. self.assertEqual( (SelectNode(self.inferred_deps, Classpath, None, None), Return(Classpath(creator='scalac'))), walk[0][0]) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Classpath, [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request(['compile'], self.java_multi) walk = self.build_and_walk(build_request, failures=True) # Validate that the root failed. root_node, root_state = walk[0][0] self.assertEqual(SelectNode(self.java_multi, Classpath, None, None), root_node) self.assertEqual(Throw, type(root_state)) def test_no_variant_thrift(self): """No `thrift` variant is configured, and so no configuration is selected.""" build_request = self.request(['compile'], self.no_variant_thrift) with self.assertRaises(PartiallyConsumedInputsError): self.build_and_walk(build_request) def test_unconfigured_thrift(self): """The BuildPropertiesPlanner is able to produce a Classpath, but we should still fail. A target with ThriftSources doesn't have a thrift config: that input is partially consumed. """ build_request = self.request(['compile'], self.unconfigured_thrift) with self.assertRaises(PartiallyConsumedInputsError): self.build_and_walk(build_request) def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm::') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0][0] root_value = root_state.value self.assertEqual( DependenciesNode(spec, Address, None, Addresses, None), root) self.assertEqual(list, type(root_value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root_value) self.assertIn(self.managed_guava, root_value) self.assertIn(self.managed_resolve_latest, root_value) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0][0] root_value = root_state.value self.assertEqual( DependenciesNode(spec, Address, None, Addresses, None), root) self.assertEqual(list, type(root_value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root_value) # And that an subdirectory address is not. self.assertNotIn(self.managed_guava, root_value)
class BuildFileAddressMapperScanTest(BaseTest): NO_FAIL_FAST_RE = re.compile(r"""^-------------------- .* Exception message: name 'a_is_bad' is not defined while executing BUILD file BuildFile\(bad/a/BUILD, FileSystemProjectTree\(.*\)\) Loading addresses from 'bad/a' failed\. .* Exception message: name 'b_is_bad' is not defined while executing BUILD file BuildFile\(bad/b/BUILD, FileSystemProjectTree\(.*\)\) Loading addresses from 'bad/b' failed\. Invalid BUILD files for \[::\]$""", re.DOTALL) FAIL_FAST_RE = """^name 'a_is_bad' is not defined while executing BUILD file BuildFile\(bad/a/BUILD\, FileSystemProjectTree\(.*\)\) Loading addresses from 'bad/a' failed.$""" def setUp(self): super(BuildFileAddressMapperScanTest, self).setUp() def add_target(path, name): self.add_to_build_file(path, 'target(name="{name}")\n'.format(name=name)) add_target('BUILD', 'root') add_target('a', 'a') add_target('a', 'b') add_target('a/b', 'b') add_target('a/b', 'c') self._spec_parser = CmdLineSpecParser(self.build_root) def test_bad_build_files(self): self.add_to_build_file('bad/a', 'a_is_bad') self.add_to_build_file('bad/b', 'b_is_bad') with self.assertRaisesRegexp(AddressLookupError, self.NO_FAIL_FAST_RE): list(self.address_mapper.scan_specs([DescendantAddresses('')], fail_fast=False)) def test_bad_build_files_fail_fast(self): self.add_to_build_file('bad/a', 'a_is_bad') self.add_to_build_file('bad/b', 'b_is_bad') with self.assertRaisesRegexp(AddressLookupError, self.FAIL_FAST_RE): list(self.address_mapper.scan_specs([DescendantAddresses('')], fail_fast=True)) def test_normal(self): self.assert_scanned([':root'], expected=[':root']) self.assert_scanned(['//:root'], expected=[':root']) self.assert_scanned(['a'], expected=['a']) self.assert_scanned(['a:a'], expected=['a']) self.assert_scanned(['a/b'], expected=['a/b']) self.assert_scanned(['a/b:b'], expected=['a/b']) self.assert_scanned(['a/b:c'], expected=['a/b:c']) def test_sibling(self): self.assert_scanned([':'], expected=[':root']) self.assert_scanned(['//:'], expected=[':root']) self.assert_scanned(['a:'], expected=['a', 'a:b']) self.assert_scanned(['//a:'], expected=['a', 'a:b']) self.assert_scanned(['a/b:'], expected=['a/b', 'a/b:c']) self.assert_scanned(['//a/b:'], expected=['a/b', 'a/b:c']) def test_sibling_or_descendents(self): self.assert_scanned(['::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c']) self.assert_scanned(['//::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c']) self.assert_scanned(['a::'], expected=['a', 'a:b', 'a/b', 'a/b:c']) self.assert_scanned(['//a::'], expected=['a', 'a:b', 'a/b', 'a/b:c']) self.assert_scanned(['a/b::'], expected=['a/b', 'a/b:c']) self.assert_scanned(['//a/b::'], expected=['a/b', 'a/b:c']) def test_cmd_line_affordances(self): self.assert_scanned(['./:root'], expected=[':root']) self.assert_scanned(['//./:root'], expected=[':root']) self.assert_scanned(['//./a/../:root'], expected=[':root']) self.assert_scanned([os.path.join(self.build_root, './a/../:root')], expected=[':root']) self.assert_scanned(['a/'], expected=['a']) self.assert_scanned(['./a/'], expected=['a']) self.assert_scanned([os.path.join(self.build_root, './a/')], expected=['a']) self.assert_scanned(['a/b/:b'], expected=['a/b']) self.assert_scanned(['./a/b/:b'], expected=['a/b']) self.assert_scanned([os.path.join(self.build_root, './a/b/:b')], expected=['a/b']) def test_cmd_line_spec_list(self): self.assert_scanned(['a', 'a/b'], expected=['a', 'a/b']) self.assert_scanned(['::'], expected=[':root', 'a', 'a:b', 'a/b', 'a/b:c']) def test_does_not_exist(self): with self.assertRaises(AddressLookupError): self.assert_scanned(['c'], expected=[]) with self.assertRaises(AddressLookupError): self.assert_scanned(['c:'], expected=[]) with self.assertRaises(AddressLookupError): self.assert_scanned(['c::'], expected=[]) def test_build_ignore_patterns(self): expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c'] # This bogus BUILD file gets in the way of parsing. self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n') with self.assertRaises(AddressLookupError): self.assert_scanned(['::'], expected=expected_specs) address_mapper_with_ignore = BuildFileAddressMapper(self.build_file_parser, self.project_tree, build_ignore_patterns=['some']) self.assert_scanned(['::'], expected=expected_specs, address_mapper=address_mapper_with_ignore) def test_exclude_target_regexps(self): address_mapper_with_exclude = BuildFileAddressMapper(self.build_file_parser, self.project_tree, exclude_target_regexps=[r'.*:b.*']) self.assert_scanned(['::'], expected=[':root', 'a', 'a/b:c'], address_mapper=address_mapper_with_exclude) def assert_scanned(self, specs_strings, expected, address_mapper=None): """Parse and scan the given specs.""" address_mapper = address_mapper or self.address_mapper def sort(addresses): return sorted(addresses, key=lambda address: address.spec) specs = [self._spec_parser.parse_spec(s) for s in specs_strings] self.assertEqual(sort(Address.parse(addr) for addr in expected), sort(address_mapper.scan_specs(specs)))
class SchedulerTest(unittest.TestCase): def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root, inline_nodes=False) self.pg = self.scheduler.product_graph self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps') def assert_select_for_subjects(self, walk, selector, subjects, variants=None): node_type = SelectNode variants = tuple(variants.items()) if variants else None self.assertEqual({node_type(subject, variants, selector) for subject in subjects}, {node for node, _ in walk if node.product == selector.product and isinstance(node, node_type) and node.variants == variants}) def build_and_walk(self, build_request): """Build and then walk the given build_request, returning the walked graph as a list.""" result = self.engine.execute(build_request) self.assertIsNone(result.error) return list(self.scheduler.product_graph.walk(build_request.roots)) def request(self, goals, *addresses): return self.request_specs(goals, *[self.spec_parser.parse_spec(str(a)) for a in addresses]) def request_specs(self, goals, *specs): return self.scheduler.build_request(goals=goals, subjects=specs) def assert_resolve_only(self, goals, root_specs, jars): build_request = self.request(goals, *root_specs) walk = self.build_and_walk(build_request) # Expect a SelectNode for each of the Jar/Classpath. self.assert_select_for_subjects(walk, Select(Jar), jars) self.assert_select_for_subjects(walk, Select(Classpath), jars) def assert_root(self, walk, node, return_value): """Asserts that the first Node in a walk was a DependenciesNode with the single given result.""" root, root_state = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Return([return_value]), root_state) self.assertIn((node, Return(return_value)), [(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)]) def assert_root_failed(self, walk, node, thrown_type): """Asserts that the first Node in a walk was a DependenciesNode with a Throw result.""" root, root_state = walk[0] self.assertEquals(type(root), DependenciesNode) self.assertEquals(Throw, type(root_state)) dependencies = [(d, self.pg.state(d)) for d in self.pg.dependencies_of(root)] self.assertIn((node, thrown_type), [(k, type(v.exc)) for k, v in dependencies if type(v) is Throw]) def test_resolve(self): self.assert_resolve_only(goals=['resolve'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_compile_only_3rdparty(self): self.assert_resolve_only(goals=['compile'], root_specs=['3rdparty/jvm:guava'], jars=[self.guava]) def test_gen_noop(self): # TODO(John Sirois): Ask around - is this OK? # This is different than today. There is a gen'able target reachable from the java target, but # the scheduler 'pull-seeding' has ApacheThriftPlanner stopping short since the subject it's # handed is not thrift. build_request = self.request(['gen'], self.java) walk = self.build_and_walk(build_request) self.assert_select_for_subjects(walk, Select(JavaSources, optional=True), [self.java]) def test_gen(self): build_request = self.request(['gen'], self.thrift) walk = self.build_and_walk(build_request) # Root: expect the synthetic GenGoal product. self.assert_root(walk, SelectNode(self.thrift, None, Select(GenGoal)), GenGoal("non-empty input to satisfy the Goal constructor")) variants = {'thrift': 'apache_java'} # Expect ThriftSources to have been selected. self.assert_select_for_subjects(walk, Select(ThriftSources), [self.thrift], variants=variants) # Expect an ApacheThriftJavaConfiguration to have been used via the default Variants. self.assert_select_for_subjects(walk, SelectVariant(ApacheThriftJavaConfiguration, variant_key='thrift'), [self.thrift], variants=variants) def test_codegen_simple(self): build_request = self.request(['compile'], self.java) walk = self.build_and_walk(build_request) # The subgraph below 'src/thrift/codegen/simple' will be affected by its default variants. subjects = [ self.guava, self.java, self.thrift] variant_subjects = [ Jar(org='org.apache.thrift', name='libthrift', rev='0.9.2', type_alias='jar'), Jar(org='commons-lang', name='commons-lang', rev='2.5', type_alias='jar'), Address.parse('src/thrift:slf4j-api')] # Root: expect a DependenciesNode depending on a SelectNode with compilation via javac. self.assert_root(walk, SelectNode(self.java, None, Select(Classpath)), Classpath(creator='javac')) # Confirm that exactly the expected subjects got Classpaths. self.assert_select_for_subjects(walk, Select(Classpath), subjects) self.assert_select_for_subjects(walk, Select(Classpath), variant_subjects, variants={'thrift': 'apache_java'}) def test_consumes_resources(self): build_request = self.request(['compile'], self.consumes_resources) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.consumes_resources, None, Select(Classpath)), Classpath(creator='javac')) # Confirm a classpath for the resources target and other subjects. We know that they are # reachable from the root (since it was involved in this walk). subjects = [self.resources, self.consumes_resources, self.guava] self.assert_select_for_subjects(walk, Select(Classpath), subjects) def test_managed_resolve(self): """A managed resolve should consume a ManagedResolve and ManagedJars to produce Jars.""" build_request = self.request(['compile'], self.consumes_managed_thirdparty) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.consumes_managed_thirdparty, None, Select(Classpath)), Classpath(creator='javac')) # Confirm that we produced classpaths for the managed jars. managed_jars = [self.managed_guava, self.managed_hadoop] self.assert_select_for_subjects(walk, Select(Classpath), [self.consumes_managed_thirdparty]) self.assert_select_for_subjects(walk, Select(Classpath), managed_jars, variants={'resolve': 'latest-hadoop'}) # Confirm that the produced jars had the appropriate versions. self.assertEquals({Jar('org.apache.hadoop', 'hadoop-common', '2.7.0'), Jar('com.google.guava', 'guava', '18.0')}, {ret.value for node, ret in walk if node.product == Jar and isinstance(node, SelectNode)}) def test_dependency_inference(self): """Scala dependency inference introduces dependencies that do not exist in BUILD files.""" build_request = self.request(['compile'], self.inferred_deps) walk = self.build_and_walk(build_request) # Validate the root. self.assert_root(walk, SelectNode(self.inferred_deps, None, Select(Classpath)), Classpath(creator='scalac')) # Confirm that we requested a classpath for the root and inferred targets. self.assert_select_for_subjects(walk, Select(Classpath), [self.inferred_deps, self.java_simple]) def test_multiple_classpath_entries(self): """Multiple Classpath products for a single subject currently cause a failure.""" build_request = self.request(['compile'], self.java_multi) walk = self.build_and_walk(build_request) # Validate that the root failed. self.assert_root_failed(walk, SelectNode(self.java_multi, None, Select(Classpath)), ConflictingProducersError) def test_descendant_specs(self): """Test that Addresses are produced via recursive globs of the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm::') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0] root_value = root_state.value self.assertEqual(DependenciesNode(spec, None, SelectDependencies(Address, Addresses)), root) self.assertEqual(list, type(root_value)) # Confirm that a few expected addresses are in the list. self.assertIn(self.guava, root_value) self.assertIn(self.managed_guava, root_value) self.assertIn(self.managed_resolve_latest, root_value) def test_sibling_specs(self): """Test that sibling Addresses are parsed in the 3rdparty/jvm directory.""" spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request_specs(['list'], spec) walk = self.build_and_walk(build_request) # Validate the root. root, root_state = walk[0] root_value = root_state.value self.assertEqual(DependenciesNode(spec, None, SelectDependencies(Address, Addresses)), root) self.assertEqual(list, type(root_value)) # Confirm that an expected address is in the list. self.assertIn(self.guava, root_value) # And that an subdirectory address is not. self.assertNotIn(self.managed_guava, root_value) def test_scheduler_visualize(self): spec = self.spec_parser.parse_spec('3rdparty/jvm:') build_request = self.request_specs(['list'], spec) self.build_and_walk(build_request) graphviz_output = '\n'.join(self.scheduler.product_graph.visualize(build_request.roots)) with temporary_dir() as td: output_path = os.path.join(td, 'output.dot') self.scheduler.visualize_graph_to_file(build_request.roots, output_path) with open(output_path, 'rb') as fh: graphviz_disk_output = fh.read().strip() self.assertEqual(graphviz_output, graphviz_disk_output) self.assertIn('digraph', graphviz_output) self.assertIn(' -> ', graphviz_output)