def _expand_goals_and_specs(self): goals = self.options.goals specs = self.options.target_specs fail_fast = self.options.for_global_scope().fail_fast for goal in goals: if self.address_mapper.from_cache(get_buildroot(), goal, must_exist=False).file_exists(): logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) if self.options.help_request: help_printer = HelpPrinter(self.options) help_printer.print_help() self._exiter(0) self.requested_goals = goals with self.run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]): spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper, spec_excludes=self.spec_excludes, exclude_target_regexps=self.global_options.exclude_target_regexp) with self.run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self.global_options.tag, filter_for_tag)) for spec in specs: for address in spec_parser.parse_addresses(spec, fail_fast): self.build_graph.inject_address_closure(address) tgt = self.build_graph.get_target(address) if tag_filter(tgt): self.targets.append(tgt) self.goals = [Goal.by_name(goal) for goal in goals]
def main_addresses(): build_root, goals, args = pop_build_root_and_goals( '[build root path] [goal]+ [address spec]*', sys.argv[1:]) cmd_line_spec_parser = CmdLineSpecParser(build_root) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in args] visualize_build_request(build_root, goals, spec_roots)
def setup(): build_root = get_buildroot() cmd_line_spec_parser = CmdLineSpecParser(build_root) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in sys.argv[1:]] storage = Storage.create(debug=False) project_tree = FileSystemProjectTree(build_root) symbol_table_cls = LegacyTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph # will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks() + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls) ) return ( LocalScheduler(dict(), tasks, symbol_table_cls, project_tree), storage, spec_roots, symbol_table_cls )
def _run_and_check(self, target_specs, incremental_import=None): """ Invoke idea-plugin goal and check for target specs and project in the generated project and workspace file. :param target_specs: list of target specs :return: n/a """ self.assertTrue("targets are empty", target_specs) spec_parser = CmdLineSpecParser(get_buildroot()) # project_path is always the directory of the first target, # which is where intellij is going to zoom in under project view. project_path = spec_parser.parse_spec(target_specs[0]).directory with self.temporary_workdir() as workdir: with temporary_file(root_dir=workdir, cleanup=True) as output_file: args = [ 'idea-plugin', '--output-file={}'.format(output_file.name), '--no-open', ] if incremental_import is not None: args.append('--incremental-import={}'.format(incremental_import)) pants_run = self.run_pants_with_workdir(args + target_specs, workdir) self.assert_success(pants_run) project_dir = self._get_project_dir(output_file.name) self.assertTrue(os.path.exists(project_dir), "{} does not exist".format(project_dir)) self._do_check(project_dir, project_path, target_specs, incremental_import=incremental_import)
def set_start_time(self, start_time: Optional[float]) -> None: # Launch RunTracker as early as possible (before .run() is called). self._run_tracker = RunTracker.global_instance() # Propagates parent_build_id to pants runs that may be called from this pants run. os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id self._reporting = Reporting.global_instance() self._reporting.initialize(self._run_tracker, self.options, start_time=start_time) spec_parser = CmdLineSpecParser(get_buildroot()) specs = [ spec_parser.parse_spec(spec).to_spec_string() for spec in self.options.specs ] # Note: This will not include values from `--changed-*` flags. self._run_tracker.run_info.add_info("specs_from_command_line", specs, stringify=False) # Capture a repro of the 'before' state for this build, if needed. self._repro = Reproducer.global_instance().create_repro() if self._repro: self._repro.capture(self._run_tracker.run_info.get_as_dict())
def setup(options=None): if not options: options, _ = OptionsInitializer(OptionsBootstrapper()).setup() build_root = get_buildroot() cmd_line_spec_parser = CmdLineSpecParser(build_root) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs] storage = Storage.create(debug=False) project_tree = FileSystemProjectTree(build_root) symbol_table_cls = LegacyTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph # will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks() + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls) ) return ( LocalScheduler(dict(), tasks, storage, project_tree), storage, options, spec_roots, symbol_table_cls )
def setup(options=None): if not options: options, _ = OptionsInitializer(OptionsBootstrapper()).setup() build_root = get_buildroot() cmd_line_spec_parser = CmdLineSpecParser(build_root) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs] storage = Storage.create(debug=False) # Ignore any dotfile below build_root except . itself project_tree = FileSystemProjectTree(build_root, ['.*', 'build-support/*.venv/']) symbol_table_cls = LegacyTable # Register "literal" subjects required for these tasks. # TODO: Replace with `Subsystems`. address_mapper = AddressMapper(symbol_table_cls=symbol_table_cls, parser_cls=LegacyPythonCallbacksParser) # Create a Scheduler containing graph and filesystem tasks, with no installed goals. The ExpGraph # will explicitly request the products it needs. tasks = ( create_legacy_graph_tasks() + create_fs_tasks() + create_graph_tasks(address_mapper, symbol_table_cls) ) return ( LocalScheduler(dict(), tasks, storage, project_tree), storage, options, spec_roots, symbol_table_cls )
def _run_and_check(self, target_specs, incremental_import=None): """ Invoke idea-plugin goal and check for target specs and project in the generated project and workspace file. :param target_specs: list of target specs :return: n/a """ self.assertTrue("targets are empty", target_specs) spec_parser = CmdLineSpecParser(get_buildroot()) # project_path is always the directory of the first target, # which is where intellij is going to zoom in under project view. project_path = spec_parser.parse_spec(target_specs[0]).directory with self.temporary_workdir() as workdir: with temporary_file(root_dir=workdir, cleanup=True) as output_file: args = [ 'idea-plugin', '--output-file={}'.format(output_file.name), '--no-open', ] if incremental_import is not None: args.append( '--incremental-import={}'.format(incremental_import)) pants_run = self.run_pants_with_workdir( args + target_specs, workdir) self.assert_success(pants_run) project_dir = self._get_project_dir(output_file.name) self.assertTrue(os.path.exists(project_dir), "{} does not exist".format(project_dir)) self._do_check(project_dir, project_path, target_specs, incremental_import=incremental_import)
def _run_and_check(self, address_specs, incremental_import=None): """Invoke idea-plugin goal and check for target specs and project in the generated project and workspace file. :param address_specs: list of address specs :return: n/a """ self.assertTrue(address_specs, "targets are empty") spec_parser = CmdLineSpecParser(get_buildroot()) # project_path is always the directory of the first target, # which is where intellij is going to zoom in under project view. spec = spec_parser.parse_spec(address_specs[0]) assert isinstance(spec, (SingleAddress, SiblingAddresses, DescendantAddresses)) project_path = spec.directory with self.temporary_workdir() as workdir: with temporary_file(root_dir=workdir, cleanup=True) as output_file: args = [ "idea-plugin", f"--output-file={output_file.name}", "--no-open", ] if incremental_import is not None: args.append(f"--incremental-import={incremental_import}") pants_run = self.run_pants_with_workdir(args + address_specs, workdir) self.assert_success(pants_run) project_dir = self._get_project_dir(output_file.name) self.assertTrue(os.path.exists(project_dir), f"{project_dir} does not exist") self._do_check( project_dir, project_path, address_specs, incremental_import=incremental_import )
def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root) self.engine = LocalSerialEngine(self.scheduler) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse( 'src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse( 'src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse( 'src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse( '3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse( '3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps')
class CmdLineSpecParserBadBuildTest(BaseTest): def setUp(self): super(CmdLineSpecParserBadBuildTest, self).setUp() self.add_to_build_file('bad/a', 'a_is_bad') self.add_to_build_file('bad/b', 'b_is_bad') self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) self.NO_FAIL_FAST_RE = re.compile( r"""^-------------------- .* Exception message: name 'a_is_bad' is not defined while executing BUILD file (/[^/]+)*/bad/a/BUILD Loading addresses from 'bad/a' failed\. .* Exception message: name 'b_is_bad' is not defined while executing BUILD file (/[^/]+)*T/bad/b/BUILD Loading addresses from 'bad/b' failed\. Invalid BUILD files for \[::\]$""", re.DOTALL) self.FAIL_FAST_RE = """^name 'a_is_bad' is not defined while executing BUILD file (/[^/]+)*/bad/a/BUILD Loading addresses from 'bad/a' failed.$""" def test_bad_build_files(self): with self.assertRaisesRegexp(self.spec_parser.BadSpecError, self.NO_FAIL_FAST_RE): list(self.spec_parser.parse_addresses('::')) def test_bad_build_files_fail_fast(self): with self.assertRaisesRegexp(self.spec_parser.BadSpecError, self.FAIL_FAST_RE): list(self.spec_parser.parse_addresses('::', True))
class CmdLineSpecParserBadBuildTest(BaseTest): def setUp(self): super(CmdLineSpecParserBadBuildTest, self).setUp() self.add_to_build_file('bad/a', 'a_is_bad') self.add_to_build_file('bad/b', 'b_is_bad') self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) self.NO_FAIL_FAST_RE = re.compile(r"""^-------------------- .* Exception message: name 'a_is_bad' is not defined while executing BUILD file FilesystemBuildFile\((/[^/]+)*/bad/a/BUILD\) Loading addresses from 'bad/a' failed\. .* Exception message: name 'b_is_bad' is not defined while executing BUILD file FilesystemBuildFile\((/[^/]+)*T/bad/b/BUILD\) Loading addresses from 'bad/b' failed\. Invalid BUILD files for \[::\]$""", re.DOTALL) self.FAIL_FAST_RE = """^name 'a_is_bad' is not defined while executing BUILD file FilesystemBuildFile\((/[^/]+)*/bad/a/BUILD\) Loading addresses from 'bad/a' failed.$""" def test_bad_build_files(self): with self.assertRaisesRegexp(self.spec_parser.BadSpecError, self.NO_FAIL_FAST_RE): list(self.spec_parser.parse_addresses('::')) def test_bad_build_files_fail_fast(self): with self.assertRaisesRegexp(self.spec_parser.BadSpecError, self.FAIL_FAST_RE): list(self.spec_parser.parse_addresses('::', True))
def parse_specs( cls, raw_specs: Iterable[str], build_root: Optional[str] = None, exclude_patterns: Optional[Iterable[str]] = None, tags: Optional[Iterable[str]] = None, ) -> Specs: """Parse raw string specs into a Specs object.""" build_root = build_root or get_buildroot() spec_parser = CmdLineSpecParser(build_root) address_specs: OrderedSet[AddressSpec] = OrderedSet() filesystem_specs: OrderedSet[FilesystemSpec] = OrderedSet() for spec_str in raw_specs: parsed_spec = spec_parser.parse_spec(spec_str) if isinstance(parsed_spec, AddressSpec): address_specs.add(parsed_spec) else: filesystem_specs.add(parsed_spec) address_specs_collection = AddressSpecs( dependencies=address_specs, exclude_patterns=exclude_patterns if exclude_patterns else tuple(), tags=tags, ) filesystem_specs_collection = FilesystemSpecs(filesystem_specs) return Specs( address_specs=address_specs_collection, filesystem_specs=filesystem_specs_collection, )
def _expand_goals_and_specs(self): goals = self.options.goals specs = self.options.target_specs fail_fast = self.options.for_global_scope().fail_fast for goal in goals: if BuildFile.from_cache(get_buildroot(), goal, must_exist=False).exists(): logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) if self.options.print_help_if_requested(): sys.exit(0) self.requested_goals = goals with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]): spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper, spec_excludes=self.spec_excludes, exclude_target_regexps=self.global_options.exclude_target_regexp) with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]): for spec in specs: for address in spec_parser.parse_addresses(spec, fail_fast): self.build_graph.inject_address_closure(address) self.targets.append(self.build_graph.get_target(address)) self.goals = [Goal.by_name(goal) for goal in goals]
def execute(self): # NOTE(pl): We now rely on the fact that we've scheduled Buildgen (the dummy task in the # buildgen goal) to run before the real buildgen tasks, e.g. buildgen-scala, buildgen-thrift, # etc. Since we are being run before the real tasks but after everything else upstream, # we can fix the target roots back up to be whatever the buildgen tasks are supposed to # operate on (instead of the entire build graph, which the upstream operated on). build_graph = self.context.build_graph bg_target_roots = set() # NOTE(mateo): Using all source roots adds a scan of 3rdparty - may have a minor perf penalty. # I would like to switch to the configured source roots if I can find a way to surface it everywhere. buildgen_dirs = self.buildgen_subsystem.source_dirs + self.buildgen_subsystem.test_dirs all_buildgen_specs = ['{}::'.format(d) for d in buildgen_dirs] spec_parser = CmdLineSpecParser(get_buildroot()) target_specs = self.context.options._target_specs or all_buildgen_specs parsed_specs = [ spec_parser.parse_spec(target_spec) for target_spec in target_specs ] for address in self.context.address_mapper.scan_specs(parsed_specs): build_graph.inject_address_closure(address) bg_target_roots.add(build_graph.get_target(address)) # Disable building from SCM for now, and instead always build everything unless the user # specifies less. # bg_target_roots.update( # target for target in # build_graph.transitive_dependees_of_addresses(t.address for t in self.targets_from_scm()) # if target.is_original # ) self.context._replace_targets(list(bg_target_roots))
def parse_specs(cls, target_specs, build_root=None, exclude_patterns=None, tags=None): """Parse string specs into unique `Spec` objects. :param iterable target_specs: An iterable of string specs. :param string build_root: The path to the build root. :returns: An `OrderedSet` of `Spec` objects. """ build_root = build_root or get_buildroot() spec_parser = CmdLineSpecParser(build_root) dependencies = tuple( OrderedSet( spec_parser.parse_spec(spec_str) for spec_str in target_specs)) if not dependencies: return None return [ Specs(dependencies=dependencies, exclude_patterns=exclude_patterns if exclude_patterns else tuple(), tags=tags) ]
def _expand_goals_and_specs(self): goals = self.options.goals specs = self.options.target_specs fail_fast = self.options.for_global_scope().fail_fast for goal in goals: if BuildFile.from_cache(get_buildroot(), goal, must_exist=False).exists(): logger.warning( " Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}." .format(goal)) if self.options.print_help_if_requested(): sys.exit(0) self.requested_goals = goals with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]): spec_parser = CmdLineSpecParser( self.root_dir, self.address_mapper, spec_excludes=self.spec_excludes, exclude_target_regexps=self.global_options. exclude_target_regexp) with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]): for spec in specs: for address in spec_parser.parse_addresses( spec, fail_fast): self.build_graph.inject_address_closure(address) self.targets.append( self.build_graph.get_target(address)) self.goals = [Goal.by_name(goal) for goal in goals]
class CmdLineSpecParserBadBuildTest(BaseTest): def setUp(self): super(CmdLineSpecParserBadBuildTest, self).setUp() self.add_to_build_file("bad/a", "a_is_bad") self.add_to_build_file("bad/b", "b_is_bad") self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) self.NO_FAIL_FAST_RE = """^Exception message: name 'a_is_bad' is not defined while executing BUILD file (/[^/]+)*/bad/a/BUILD Loading addresses from 'bad/a' failed\. Exception message: name 'b_is_bad' is not defined while executing BUILD file (/[^/]+)*T/bad/b/BUILD Loading addresses from 'bad/b' failed\. Invalid BUILD files for \[::\]$""" self.FAIL_FAST_RE = """^name 'a_is_bad' is not defined while executing BUILD file (/[^/]+)*/bad/a/BUILD Loading addresses from 'bad/a' failed.$""" def test_bad_build_files(self): with self.assertRaisesRegexp(self.spec_parser.BadSpecError, self.NO_FAIL_FAST_RE): list(self.spec_parser.parse_addresses("::")) def test_bad_build_files_fail_fast(self): with self.assertRaisesRegexp(self.spec_parser.BadSpecError, self.FAIL_FAST_RE): list(self.spec_parser.parse_addresses("::", True))
def setup_parser(self, parser, args): if not args: args.append('help') logger = logging.getLogger(__name__) goals = self.new_options.goals specs = self.new_options.target_specs fail_fast = self.new_options.for_global_scope().fail_fast for goal in goals: if BuildFile.from_cache(get_buildroot(), goal, must_exist=False).exists(): logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) if self.new_options.is_help: self.new_options.print_help(goals=goals) sys.exit(0) self.requested_goals = goals with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]): spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper, spec_excludes=self.get_spec_excludes()) with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]): for spec in specs: for address in spec_parser.parse_addresses(spec, fail_fast): self.build_graph.inject_address_closure(address) self.targets.append(self.build_graph.get_target(address)) self.goals = [Goal.by_name(goal) for goal in goals] rcfiles = self.config.getdefault('rcfiles', type=list, default=['/etc/pantsrc', '~/.pants.rc']) if rcfiles: rcfile = RcFile(rcfiles, default_prepend=False, process_default=True) # Break down the goals specified on the command line to the full set that will be run so we # can apply default flags to inner goal nodes. Also break down goals by Task subclass and # register the task class hierarchy fully qualified names so we can apply defaults to # baseclasses. sections = OrderedSet() for goal in Engine.execution_order(self.goals): for task_name in goal.ordered_task_names(): sections.add(task_name) task_type = goal.task_type_by_name(task_name) for clazz in task_type.mro(): if clazz == Task: break sections.add('%s.%s' % (clazz.__module__, clazz.__name__)) augmented_args = rcfile.apply_defaults(sections, args) if augmented_args != args: # TODO(John Sirois): Cleanup this currently important mutation of the passed in args # once the 2-layer of command -> goal is squashed into one. args[:] = augmented_args sys.stderr.write("(using pantsrc expansion: pants goal %s)\n" % ' '.join(augmented_args))
def _get_targets(spec): try: spec_parser = CmdLineSpecParser(get_buildroot(), self.context.build_file_parser) addresses = spec_parser.parse_addresses(spec) except (IOError, ValueError) as e: raise TaskError('Failed to parse spec: %s: %s' % (spec, e)) matches = set(self.context.build_graph.get_target(address) for address in addresses) if not matches: raise TaskError('No matches for spec: %s' % spec) return matches
def _determine_goals(self, address_mapper, requested_goals): """Check and populate the requested goals for a given run.""" spec_parser = CmdLineSpecParser(self._root_dir) for goal in requested_goals: if address_mapper.is_valid_single_address(spec_parser.parse_spec(goal)): logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) return [Goal.by_name(goal) for goal in requested_goals]
def parse_specs(cls, target_specs, build_root=None): """Parse string specs into unique `Spec` objects. :param iterable target_specs: An iterable of string specs. :param string build_root: The path to the build root. :returns: An `OrderedSet` of `Spec` objects. """ build_root = build_root or get_buildroot() spec_parser = CmdLineSpecParser(build_root) return OrderedSet(spec_parser.parse_spec(spec_str) for spec_str in target_specs)
def __init__(self, *args, **kwargs): super(Build, self).__init__(*args, **kwargs) if not self.args: self.error("A spec argument is required") self.config = Config.load() interpreters = self.options.interpreters or [b''] self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug) self.interpreter_cache.setup(filters=interpreters) interpreters = self.interpreter_cache.select_interpreter( list(self.interpreter_cache.matches(interpreters))) if len(interpreters) != 1: self.error('Unable to detect suitable interpreter.') else: self.debug('Selected %s' % interpreters[0]) self.interpreter = interpreters[0] try: specs_end = self.args.index('--') if len(self.args) > specs_end: self.build_args = self.args[specs_end + 1:len(self.args) + 1] else: self.build_args = [] except ValueError: specs_end = 1 self.build_args = self.args[1:] if len(self.args) > 1 else [] self.targets = OrderedSet() spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper) self.top_level_addresses = set() specs = self.args[0:specs_end] addresses = spec_parser.parse_addresses(specs) for address in addresses: self.top_level_addresses.add(address) try: self.build_graph.inject_address_closure(address) target = self.build_graph.get_target(address) except: self.error("Problem parsing BUILD target %s: %s" % (address, traceback.format_exc())) if not target: self.error("Target %s does not exist" % address) transitive_targets = self.build_graph.transitive_subgraph_of_addresses( [target.address]) for transitive_target in transitive_targets: self.targets.add(transitive_target) self.targets = [target for target in self.targets if target.is_python]
def test_exclude_target_regexps(self): expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c'] # This bogus BUILD file gets in the way of parsing. self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n') with self.assertRaises(CmdLineSpecParser.BadSpecError): self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs) self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper, exclude_target_regexps=[r'.*some/dir.*']) self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs)
def __init__(self, *args, **kwargs): super(Build, self).__init__(*args, **kwargs) if not self.args: self.error("A spec argument is required") self._verbose = self.options.verbose self.config = Config.load() interpreters = self.options.interpreters or [b''] self.interpreter_cache = PythonInterpreterCache(self.config, logger=self.debug) self.interpreter_cache.setup(filters=interpreters) interpreters = self.interpreter_cache.select_interpreter( list(self.interpreter_cache.matches(interpreters))) if len(interpreters) != 1: self.error('Unable to detect suitable interpreter.') else: self.debug('Selected %s' % interpreters[0]) self.interpreter = interpreters[0] try: specs_end = self.args.index('--') if len(self.args) > specs_end: self.build_args = self.args[specs_end+1:len(self.args)+1] else: self.build_args = [] except ValueError: specs_end = 1 self.build_args = self.args[1:] if len(self.args) > 1 else [] self.targets = OrderedSet() spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper) self.top_level_addresses = set() specs = self.args[0:specs_end] addresses = spec_parser.parse_addresses(specs) for address in addresses: self.top_level_addresses.add(address) try: self.build_graph.inject_address_closure(address) target = self.build_graph.get_target(address) except: self.error("Problem parsing BUILD target %s: %s" % (address, traceback.format_exc())) if not target: self.error("Target %s does not exist" % address) transitive_targets = self.build_graph.transitive_subgraph_of_addresses([target.address]) for transitive_target in transitive_targets: self.targets.add(transitive_target) self.targets = [target for target in self.targets if target.is_python]
def _determine_goals(self, requested_goals): """Check and populate the requested goals for a given run.""" spec_parser = CmdLineSpecParser(self._root_dir) for goal in requested_goals: if self._address_mapper.is_valid_single_address(spec_parser.parse_spec(goal)): logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) goals = [Goal.by_name(goal) for goal in requested_goals] return goals
def parse_commandline_to_spec_roots(options=None, args=None, build_root=None): if not options: options, _ = OptionsInitializer(OptionsBootstrapper(args=args), init_logging=False).setup() cmd_line_spec_parser = CmdLineSpecParser(build_root or get_buildroot()) spec_roots = [ cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs ] return spec_roots
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast # Will be provided through context.address_mapper.build_ignore_patterns. self._spec_excludes = None self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns self._project_tree = self._get_project_tree( self._global_options.build_file_rev) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) build_ignore_patterns = self._global_options.ignore_patterns or [] build_ignore_patterns.extend( BuildFile._spec_excludes_to_gitignore_syntax( self._root_dir, self._global_options.spec_excludes)) self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._project_tree, build_ignore_patterns) self._build_graph = BuildGraph(self._address_mapper) self._spec_parser = CmdLineSpecParser( self._root_dir, self._address_mapper, spec_excludes=self._spec_excludes, exclude_target_regexps=self._global_options.exclude_target_regexp)
def _determine_goals(self, requested_goals): """Check and populate the requested goals for a given run.""" def is_quiet(goals): return any(goal.has_task_of_type(QuietTaskMixin) for goal in goals) or self._explain spec_parser = CmdLineSpecParser(self._root_dir) for goal in requested_goals: if self._address_mapper.is_valid_single_address(spec_parser.parse_spec(goal)): logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) goals = [Goal.by_name(goal) for goal in requested_goals] return goals, is_quiet(goals)
def setUp(self): super(CmdLineSpecParserTest, self).setUp() def add_target(path, name): self.add_to_build_file(path, 'generic(name="{name}")\n'.format(name=name)) add_target('BUILD', 'root') add_target('a', 'a') add_target('a', 'b') add_target('a/b', 'b') add_target('a/b', 'c') self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
def setUp(self): super(BuildFileAddressMapperScanTest, self).setUp() def add_target(path, name): self.add_to_build_file(path, 'target(name="{name}")\n'.format(name=name)) add_target('BUILD', 'root') add_target('a', 'a') add_target('a', 'b') add_target('a/b', 'b') add_target('a/b', 'c') self._spec_parser = CmdLineSpecParser(self.build_root)
def targets(self, spec): """Resolves a target spec to one or more Target objects. spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec_parser = CmdLineSpecParser(self.build_root, self.build_file_parser) addresses = list(spec_parser.parse_addresses(spec)) for address in addresses: self.build_file_parser.inject_spec_closure_into_build_graph(address.spec, self.build_graph) targets = [self.build_graph.get_target(address) for address in addresses] return targets
def _get_targets(spec): try: spec_parser = CmdLineSpecParser(get_buildroot(), self.context.address_mapper) addresses = spec_parser.parse_addresses(spec) except AddressLookupError as e: raise TaskError('Failed to parse address selector: {spec}\n {message}'.format(spec=spec, message=e)) # filter specs may not have been parsed as part of the context: force parsing matches = set() for address in addresses: self.context.build_graph.inject_address_closure(address) matches.add(self.context.build_graph.get_target(address)) if not matches: raise TaskError('No matches for address selector: {spec}'.format(spec=spec)) return matches
def _get_targets(spec): try: spec_parser = CmdLineSpecParser(get_buildroot(), self.context.build_file_parser) addresses = spec_parser.parse_addresses(spec) except (IOError, ValueError) as e: raise TaskError('Failed to parse spec: %s: %s' % (spec, e)) # filter specs may not have been parsed as part of the context: force parsing matches = set() for address in addresses: self.context.build_file_parser.inject_address_into_build_graph(address, self.context.build_graph) matches.add(self.context.build_graph.get_target(address)) if not matches: raise TaskError('No matches for spec: %s' % spec) return matches
def _set_start_time(self, start_time: float) -> None: # Propagates parent_build_id to pants runs that may be called from this pants run. os.environ["PANTS_PARENT_BUILD_ID"] = self._run_tracker.run_id self._run_tracker.start(self.options, run_start_time=start_time) spec_parser = CmdLineSpecParser(get_buildroot()) specs = [ str(spec_parser.parse_spec(spec)) for spec in self.options.specs ] # Note: This will not include values from `--changed-*` flags. self._run_tracker.run_info.add_info("specs_from_command_line", specs, stringify=False)
def parse_specs(cls, target_specs, build_root=None, exclude_patterns=None, tags=None): """Parse string specs into unique `Spec` objects. :param iterable target_specs: An iterable of string specs. :param string build_root: The path to the build root. :returns: A `Specs` object. """ build_root = build_root or get_buildroot() spec_parser = CmdLineSpecParser(build_root) dependencies = tuple(OrderedSet(spec_parser.parse_spec(spec_str) for spec_str in target_specs)) return Specs( dependencies=dependencies, exclude_patterns=exclude_patterns if exclude_patterns else tuple(), tags=tags)
def targets(self, spec): """Resolves a target spec to one or more Target objects. spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) addresses = list(spec_parser.parse_addresses(spec)) for address in addresses: self.build_graph.inject_address_closure(address) targets = [self.build_graph.get_target(address) for address in addresses] return targets
def test_pants_dot_d_excluded(self): expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c'] # This bogus BUILD file gets in the way of parsing. self.add_to_build_file('.pants.d/some/dir', 'COMPLETELY BOGUS BUILDFILE)\n') with self.assertRaises(CmdLineSpecParser.BadSpecError): self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs) self.spec_parser = CmdLineSpecParser( self.build_root, self.address_mapper, spec_excludes=[os.path.join(self.build_root, '.pants.d')]) self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs)
def test_build_ignore_patterns(self): expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c'] # This bogus BUILD file gets in the way of parsing. self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n') with self.assertRaises(CmdLineSpecParser.BadSpecError): self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs) address_mapper_with_ignore = BuildFileAddressMapper( self.build_file_parser, self.project_tree, build_ignore_patterns=['some']) self.spec_parser = CmdLineSpecParser(self.build_root, address_mapper_with_ignore) self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs)
def test_spec_excludes(self): expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c'] # This bogus BUILD file gets in the way of parsing. self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n') with self.assertRaises(CmdLineSpecParser.BadSpecError): self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs) # Test absolute path in spec_excludes. self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper, spec_excludes=[os.path.join(self.build_root, 'some')]) self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs) # Test relative path in spec_excludes. self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper, spec_excludes=['some']) self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs)
def parse_address_specs( cls, target_specs: Iterable[str], build_root: Optional[str] = None, exclude_patterns: Optional[Iterable[str]] = None, tags: Optional[Iterable[str]] = None, ) -> AddressSpecs: """Parse string specs into unique `AddressSpec` objects.""" build_root = build_root or get_buildroot() spec_parser = CmdLineSpecParser(build_root) dependencies = tuple( OrderedSet( spec_parser.parse_address_spec(spec_str) for spec_str in target_specs)) return AddressSpecs( dependencies=dependencies, exclude_patterns=exclude_patterns if exclude_patterns else tuple(), tags=tags)
def _expand_specs(self, spec_strs, fail_fast): """Populate the BuildGraph and target list from a set of input specs.""" with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag)) # Parse all specs into unique Spec objects. spec_parser = CmdLineSpecParser(self._root_dir) specs = OrderedSet() for spec_str in spec_strs: specs.add(spec_parser.parse_spec(spec_str)) # Then scan them to generate unique Addresses. for address in self._build_graph.inject_specs_closure(specs, fail_fast): target = self._build_graph.get_target(address) if tag_filter(target): self._targets.append(target)
def setUp(self): super(CmdLineSpecParserBadBuildTest, self).setUp() self.add_to_build_file('bad/a', 'a_is_bad') self.add_to_build_file('bad/b', 'b_is_bad') self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) self.NO_FAIL_FAST_RE = re.compile(r"""^-------------------- .* Exception message: name 'a_is_bad' is not defined while executing BUILD file FilesystemBuildFile\((/[^/]+)*/bad/a/BUILD\) Loading addresses from 'bad/a' failed\. .* Exception message: name 'b_is_bad' is not defined while executing BUILD file FilesystemBuildFile\((/[^/]+)*T/bad/b/BUILD\) Loading addresses from 'bad/b' failed\. Invalid BUILD files for \[::\]$""", re.DOTALL) self.FAIL_FAST_RE = """^name 'a_is_bad' is not defined
def _expand_specs(self, spec_strs, fail_fast): """Populate the BuildGraph and target list from a set of input specs.""" with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag)) # Parse all specs into unique Spec objects. spec_parser = CmdLineSpecParser(self._root_dir) specs = OrderedSet() for spec_str in spec_strs: specs.add(spec_parser.parse_spec(spec_str)) # Then scan them to generate unique Addresses. for address in self._address_mapper.scan_specs(specs, fail_fast, self._spec_excludes): self._build_graph.inject_address_closure(address) target = self._build_graph.get_target(address) if tag_filter(target): self._targets.append(target)
def set_start_time(self, start_time): # Launch RunTracker as early as possible (before .run() is called). self._run_tracker = RunTracker.global_instance() self._reporting = Reporting.global_instance() self._run_start_time = start_time self._reporting.initialize(self._run_tracker, self._options, start_time=self._run_start_time) spec_parser = CmdLineSpecParser(get_buildroot()) target_specs = [ spec_parser.parse_spec(spec).to_spec_string() for spec in self._options.positional_args ] # Note: This will not include values from `--owner-of` or `--changed-*` flags. self._run_tracker.run_info.add_info("specs_from_command_line", target_specs, stringify=False) # Capture a repro of the 'before' state for this build, if needed. self._repro = Reproducer.global_instance().create_repro() if self._repro: self._repro.capture(self._run_tracker.run_info.get_as_dict())
def test_build_ignore_patterns(self): expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c'] # This bogus BUILD file gets in the way of parsing. self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n') with self.assertRaises(CmdLineSpecParser.BadSpecError): self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs) address_mapper_with_ignore = BuildFileAddressMapper(self.build_file_parser, self.project_tree, build_ignore_patterns=['some']) self.spec_parser = CmdLineSpecParser(self.build_root, address_mapper_with_ignore) self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs)
def parse_specs(cls, raw_specs: Iterable[str], *, build_root: Optional[str] = None) -> Specs: """Parse raw string specs into a Specs object.""" build_root = build_root or get_buildroot() spec_parser = CmdLineSpecParser(build_root) address_specs: OrderedSet[AddressSpec] = OrderedSet() filesystem_specs: OrderedSet[FilesystemSpec] = OrderedSet() for spec_str in raw_specs: parsed_spec = spec_parser.parse_spec(spec_str) if isinstance(parsed_spec, AddressSpec): address_specs.add(parsed_spec) else: filesystem_specs.add(parsed_spec) return Specs( AddressSpecs(address_specs, filter_by_global_options=True), FilesystemSpecs(filesystem_specs), )
def setUp(self): super(BuildFileAddressMapperScanTest, self).setUp() def add_target(path, name): self.add_to_build_file(path, 'target(name="{name}")\n'.format(name=name)) add_target("BUILD", "root") add_target("a", "a") add_target("a", "b") add_target("a/b", "b") add_target("a/b", "c") self._spec_parser = CmdLineSpecParser(self.build_root)
def setUp(self): super(CmdLineSpecParserTest, self).setUp() def add_target(path, name): self.add_to_build_file(path, 'generic(name="{name}")\n'.format(name=name)) add_target("BUILD", "root") add_target("a", "a") add_target("a", "b") add_target("a/b", "b") add_target("a/b", "c") self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
class GraphTestBase(unittest.TestCase): _goal = 'parse' _product = Struct _build_root = os.path.dirname(__file__) _cmd_line_spec_parser = CmdLineSpecParser(_build_root) def _select(self, address): return SelectNode(address, self._product, None, None) def create(self, build_pattern=None, parser_cls=None, inline=False): symbol_table_cls = TestTable mapper = AddressMapper(symbol_table_cls=symbol_table_cls, build_pattern=build_pattern, parser_cls=parser_cls) tasks = (create_fs_tasks(self._build_root) + create_graph_tasks(mapper, symbol_table_cls)) return LocalScheduler({self._goal: self._product}, symbol_table_cls, tasks) def create_json(self): return self.create(build_pattern=r'.+\.BUILD.json$', parser_cls=JsonParser) def _populate(self, scheduler, address): """Make a BuildRequest to parse the given Address into a Struct.""" spec = self._cmd_line_spec_parser.parse_spec(str(address)) request = BuildRequest(goals=[self._goal], subjects=[spec]) LocalSerialEngine(scheduler).reduce(request) return self._select(address) def walk(self, scheduler, address): """Return a list of all (Node, State) tuples reachable from the given Address.""" root = self._populate(scheduler, address) return list(e for e, _ in scheduler.product_graph.walk( [root], predicate=lambda _: True)) def resolve_failure(self, scheduler, address): root = self._populate(scheduler, address) state = scheduler.product_graph.state(root) self.assertEquals(type(state), Throw, '{} is not a Throw.'.format(state)) return state.exc def resolve(self, scheduler, address): root = self._populate(scheduler, address) state = scheduler.product_graph.state(root) self.assertEquals(type(state), Return, '{} is not a Return.'.format(state)) return state.value
def create(cls, options=None, args=None, build_root=None, change_calculator=None): """ :param Options options: An `Options` instance to use, if available. :param string args: Raw cli args to use for parsing if an `Options` instance isn't available. :param string build_root: The build root. :param ChangeCalculator change_calculator: A `ChangeCalculator` for calculating changes. """ if not options: assert args is not None, 'must pass `args` if not passing `options`' options, _ = OptionsInitializer(OptionsBootstrapper(args=args)).setup(init_logging=False) # Determine the literal target roots. cmd_line_spec_parser = CmdLineSpecParser(build_root or get_buildroot()) spec_roots = [cmd_line_spec_parser.parse_spec(spec) for spec in options.target_specs] # Determine `Changed` arguments directly from options to support pre-`Subsystem` initialization paths. changed_options = options.for_scope('changed') changed_request = ChangedRequest.from_options(changed_options) logger.debug('args are: %s', args) logger.debug('spec_roots are: %s', spec_roots) logger.debug('changed_request is: %s', changed_request) if change_calculator and changed_request.is_actionable(): if spec_roots: # We've been provided spec roots (e.g. `./pants list ::`) AND a changed request. Error out. raise InvalidSpecConstraint('cannot provide changed parameters and target specs!') # We've been provided no spec roots (e.g. `./pants list`) AND a changed request. Compute # alternate target roots. changed_addresses = change_calculator.changed_target_addresses(changed_request) logger.debug('changed addresses: %s', changed_addresses) return ChangedTargetRoots(changed_addresses) # If no spec roots are passed, assume `::` as the intended target. return LiteralTargetRoots(spec_roots if spec_roots else [DescendantAddresses('')])
def setUp(self): super(CmdLineSpecParserBadBuildTest, self).setUp() self.add_to_build_file('bad/a', 'a_is_bad') self.add_to_build_file('bad/b', 'b_is_bad') self.spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) self.NO_FAIL_FAST_RE = """^Exception message: name 'a_is_bad' is not defined while executing BUILD file (/[^/]+)*/bad/a/BUILD Loading addresses from 'bad/a' failed\. Exception message: name 'b_is_bad' is not defined while executing BUILD file (/[^/]+)*T/bad/b/BUILD Loading addresses from 'bad/b' failed\. Invalid BUILD files for \[::\]$""" self.FAIL_FAST_RE = """^name 'a_is_bad' is not defined
def setUp(self): build_root = os.path.join(os.path.dirname(__file__), 'examples', 'scheduler_inputs') self.spec_parser = CmdLineSpecParser(build_root) self.scheduler = setup_json_scheduler(build_root, self._native) self.guava = Address.parse('3rdparty/jvm:guava') self.thrift = Address.parse('src/thrift/codegen/simple') self.java = Address.parse('src/java/codegen/simple') self.java_simple = Address.parse('src/java/simple') self.java_multi = Address.parse('src/java/multiple_classpath_entries') self.no_variant_thrift = Address.parse('src/java/codegen/selector:conflict') self.unconfigured_thrift = Address.parse('src/thrift/codegen/unconfigured') self.resources = Address.parse('src/resources/simple') self.consumes_resources = Address.parse('src/java/consumes_resources') self.consumes_managed_thirdparty = Address.parse('src/java/managed_thirdparty') self.managed_guava = Address.parse('3rdparty/jvm/managed:guava') self.managed_hadoop = Address.parse('3rdparty/jvm/managed:hadoop-common') self.managed_resolve_latest = Address.parse('3rdparty/jvm/managed:latest-hadoop') self.inferred_deps = Address.parse('src/scala/inferred_deps')