def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper( self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns) self.build_graph = MutableBuildGraph( address_mapper=self.address_mapper)
def test_address_lookup_error_hierarchy(self): self.assertIsInstance(BuildFileAddressMapper.AddressNotInBuildFile(), AddressLookupError) self.assertIsInstance(BuildFileAddressMapper.EmptyBuildFileError(), AddressLookupError) self.assertIsInstance( BuildFileAddressMapper.InvalidBuildFileReference(), AddressLookupError) self.assertIsInstance(BuildFileAddressMapper.InvalidAddressError(), AddressLookupError) self.assertIsInstance(BuildFileAddressMapper.BuildFileScanError(), AddressLookupError)
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast # Will be provided through context.address_mapper.build_ignore_patterns. self._spec_excludes = None self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns self._project_tree = self._get_project_tree( self._global_options.build_file_rev) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) build_ignore_patterns = self._global_options.ignore_patterns or [] build_ignore_patterns.extend( BuildFile._spec_excludes_to_gitignore_syntax( self._root_dir, self._global_options.spec_excludes)) self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._project_tree, build_ignore_patterns) self._build_graph = BuildGraph(self._address_mapper) self._spec_parser = CmdLineSpecParser( self._root_dir, self._address_mapper, spec_excludes=self._spec_excludes, exclude_target_regexps=self._global_options.exclude_target_regexp)
def setUp(self): super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'cache_key_gen_version': '0-test', } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) # We need a pants.ini, even if empty. get_buildroot() uses its presence. self.create_file('pants.ini') self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def test_exclude_target_regexps(self): address_mapper_with_exclude = BuildFileAddressMapper( self.build_file_parser, self.project_tree, exclude_target_regexps=[r'.*:b.*']) self.assert_scanned(['::'], expected=[':root', 'a', 'a/b:c'], address_mapper=address_mapper_with_exclude)
def setUp(self): """ :API: public """ super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.subprocess_dir = os.path.join(self.build_root, '.pids') self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'pants_subprocessdir': self.subprocess_dir, 'cache_key_gen_version': '0-test', } self.options['cache'] = { 'read_from': [], 'write_to': [], } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.project_tree = FileSystemProjectTree(self.build_root) self.address_mapper = BuildFileAddressMapper( self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns) self.build_graph = MutableBuildGraph( address_mapper=self.address_mapper)
def _init_graph(self, use_engine, pants_ignore_patterns, build_ignore_patterns, exclude_target_regexps, target_specs, workdir, graph_helper=None, subproject_build_roots=None): """Determine the BuildGraph, AddressMapper and spec_roots for a given run. :param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph. :param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'. :param list build_ignore_patterns: The build ignore patterns from '--build-ignore', applied during BUILD file searching. :param str workdir: The pants workdir. :param list exclude_target_regexps: Regular expressions for targets to be excluded. :param list target_specs: The original target specs. :param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction, if available. This would usually come from the daemon. :returns: A tuple of (BuildGraph, AddressMapper, opt Scheduler, spec_roots). """ # N.B. Use of the daemon implies use of the v2 engine. if graph_helper or use_engine: # The daemon may provide a `graph_helper`. If that's present, use it for graph construction. if not graph_helper: native = Native.create(self._global_options) native.set_panic_handler() graph_helper = EngineInitializer.setup_legacy_graph( pants_ignore_patterns, workdir, self._global_options.build_file_imports, native=native, build_file_aliases=self._build_config.registered_aliases(), build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_build_roots, include_trace_on_error=self._options.for_global_scope( ).print_exception_stacktrace) target_roots = TargetRoots.create( options=self._options, build_root=self._root_dir, change_calculator=graph_helper.change_calculator) graph, address_mapper = graph_helper.create_build_graph( target_roots, self._root_dir) return graph, address_mapper, graph_helper.scheduler, target_roots.as_specs( ) else: spec_roots = TargetRoots.parse_specs(target_specs, self._root_dir) address_mapper = BuildFileAddressMapper( self._build_file_parser, get_project_tree(self._global_options), build_ignore_patterns, exclude_target_regexps, subproject_build_roots) return MutableBuildGraph( address_mapper), address_mapper, None, spec_roots
def test_build_ignore_patterns(self): expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c'] # This bogus BUILD file gets in the way of parsing. self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n') with self.assertRaises(AddressLookupError): self.assert_scanned(['::'], expected=expected_specs) address_mapper_with_ignore = BuildFileAddressMapper(self.build_file_parser, self.project_tree, build_ignore_patterns=['some']) self.assert_scanned(['::'], expected=expected_specs, address_mapper=address_mapper_with_ignore)
def test_build_ignore_patterns(self): expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c'] # This bogus BUILD file gets in the way of parsing. self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n') with self.assertRaises(CmdLineSpecParser.BadSpecError): self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs) address_mapper_with_ignore = BuildFileAddressMapper( self.build_file_parser, self.project_tree, build_ignore_patterns=['some']) self.spec_parser = CmdLineSpecParser(self.build_root, address_mapper_with_ignore) self.assert_parsed_list(cmdline_spec_list=['::'], expected=expected_specs)
def __init__(self, root_dir, options, build_config, run_tracker, reporting, build_graph=None, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param BuildGraph build_graph: A BuildGraph instance (for graph reuse, optional). :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast # Will be provided through context.address_mapper.build_ignore_patterns. self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns pants_ignore = self._global_options.pants_ignore or [] self._project_tree = self._get_project_tree(self._global_options.build_file_rev, pants_ignore) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) build_ignore_patterns = self._global_options.ignore_patterns or [] self._address_mapper = BuildFileAddressMapper( self._build_file_parser, self._project_tree, build_ignore_patterns, exclude_target_regexps=self._global_options.exclude_target_regexp ) self._build_graph = self._select_buildgraph(self._global_options.enable_v2_engine, self._global_options.pants_ignore, build_graph)
def setUp(self): """ :API: public """ super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.subprocess_dir = os.path.join(self.build_root, '.pids') self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'pants_subprocessdir': self.subprocess_dir, 'cache_key_gen_version': '0-test', } self.options['cache'] = { 'read_from': [], 'write_to': [], } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.project_tree = FileSystemProjectTree(self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns) self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast self._spec_excludes = self._global_options.spec_excludes self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns self._project_tree = self._get_project_tree(self._global_options.build_file_rev) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._project_tree) self._build_graph = BuildGraph(self._address_mapper) self._spec_parser = CmdLineSpecParser( self._root_dir, self._address_mapper, spec_excludes=self._spec_excludes, exclude_target_regexps=self._global_options.exclude_target_regexp, )
class GoalRunnerFactory(object): def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast self._spec_excludes = self._global_options.spec_excludes self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns self._project_tree = self._get_project_tree(self._global_options.build_file_rev) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._project_tree) self._build_graph = BuildGraph(self._address_mapper) self._spec_parser = CmdLineSpecParser( self._root_dir, self._address_mapper, spec_excludes=self._spec_excludes, exclude_target_regexps=self._global_options.exclude_target_regexp, ) def _get_project_tree(self, build_file_rev): """Creates the project tree for build files for use in a given pants run.""" if build_file_rev: return ScmProjectTree(self._root_dir, get_scm(), build_file_rev) else: return FileSystemProjectTree(self._root_dir) def _expand_goals(self, goals): """Check and populate the requested goals for a given run.""" for goal in goals: try: self._address_mapper.resolve_spec(goal) logger.warning( "Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal) ) except AddressLookupError: pass if self._help_request: help_printer = HelpPrinter(self._options) result = help_printer.print_help() self._exiter(result) self._goals.extend([Goal.by_name(goal) for goal in goals]) def _expand_specs(self, specs, fail_fast): """Populate the BuildGraph and target list from a set of input specs.""" with self._run_tracker.new_workunit(name="parse", labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag)) for spec in specs: for address in self._spec_parser.parse_addresses(spec, fail_fast): self._build_graph.inject_address_closure(address) target = self._build_graph.get_target(address) if tag_filter(target): self._targets.append(target) def _maybe_launch_pantsd(self): """Launches pantsd if configured to do so.""" if self._global_options.enable_pantsd: # Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on. with self._run_tracker.new_workunit(name="pantsd", labels=[WorkUnitLabel.SETUP]): PantsDaemonLauncher.global_instance().maybe_launch() def _is_quiet(self): return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain def _setup_context(self): self._maybe_launch_pantsd() with self._run_tracker.new_workunit(name="setup", labels=[WorkUnitLabel.SETUP]): self._expand_goals(self._requested_goals) self._expand_specs(self._target_specs, self._fail_fast) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self._run_tracker.run_info.add_scm_info() # Update the Reporting settings now that we have options and goal info. invalidation_report = self._reporting.update_reporting( self._global_options, self._is_quiet(), self._run_tracker ) context = Context( options=self._options, run_tracker=self._run_tracker, target_roots=self._targets, requested_goals=self._requested_goals, build_graph=self._build_graph, build_file_parser=self._build_file_parser, address_mapper=self._address_mapper, spec_excludes=self._spec_excludes, invalidation_report=invalidation_report, ) return context, invalidation_report def setup(self): context, invalidation_report = self._setup_context() return GoalRunner( context=context, goals=self._goals, kill_nailguns=self._kill_nailguns, run_tracker=self._run_tracker, invalidation_report=invalidation_report, exiter=self._exiter, )
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot. :API: public """ def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path. :API: public """ if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. :API: public relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_workdir_dir(self, relpath): """Creates a directory under the work directory. :API: public relpath: The relative path to the directory from the work directory. """ path = os.path.join(self.pants_workdir, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. :API: public relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def create_workdir_file(self, relpath, contents='', mode='wb'): """Writes to a file under the work directory. :API: public relpath: The relative path to the file from the work directory. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.pants_workdir, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. :API: public relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') return BuildFile(self.address_mapper._project_tree, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, derived_from=None, synthetic=False, **kwargs): """Creates a target and injects it into the test's build graph. :API: public :param string spec: The target address spec that locates this target. :param type target_type: The concrete target subclass to create this new target from. :param list dependencies: A list of target instances this new target depends on. :param derived_from: The target this new target was derived from. :type derived_from: :class:`pants.build_graph.target.Target` """ address = Address.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] self.build_graph.apply_injectables([target]) self.build_graph.inject_target(target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from, synthetic=synthetic) # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph. # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it. traversables = [target.compute_dependency_specs(payload=target.payload)] for dependency_spec in itertools.chain(*traversables): dependency_address = Address.parse(dependency_spec, relative_to=address.spec_path) dependency_target = self.build_graph.get_target(dependency_address) if not dependency_target: raise ValueError('Tests must make targets for dependency specs ahead of them ' 'being traversed, {} tried to traverse {} which does not exist.' .format(target, dependency_address)) if dependency_target not in target.dependencies: self.build_graph.inject_dependency(dependent=target.address, dependency=dependency_address) target.mark_transitive_invalidation_hash_dirty() return target @property def alias_groups(self): """ :API: public """ return BuildFileAliases(targets={'target': Target}) @property def build_ignore_patterns(self): """ :API: public """ return None def setUp(self): """ :API: public """ super(BaseTest, self).setUp() # Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup. clean_global_runtime_state(reset_subsystem=True) self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.subprocess_dir = os.path.join(self.build_root, '.pids') self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'pants_subprocessdir': self.subprocess_dir, 'cache_key_gen_version': '0-test', } self.options['cache'] = { 'read_from': [], 'write_to': [], } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.project_tree = FileSystemProjectTree(self.build_root) self.reset_build_graph() def buildroot_files(self, relpath=None): """Returns the set of all files under the test build root. :API: public :param string relpath: If supplied, only collect files from this subtree. :returns: All file paths found. :rtype: set """ def scan(): for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or '')): for f in files: yield os.path.relpath(os.path.join(root, f), self.build_root) return set(scan()) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns) self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, for_subsystems=None, options=None, target_roots=None, console_outstream=None, workspace=None, **kwargs): """ :API: public :param dict **kwargs: keyword arguments passed in to `create_options_for_optionables`. """ # Many tests use source root functionality via the SourceRootConfig.global_instance(). # (typically accessed via Target.target_base), so we always set it up, for convenience. optionables = {SourceRootConfig} extra_scopes = set() for_subsystems = for_subsystems or () for subsystem in for_subsystems: if subsystem.options_scope is None: raise TaskError('You must set a scope on your subsystem type before using it in tests.') optionables.add(subsystem) for_task_types = for_task_types or () for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError('You must set a scope on your task type before using it in tests.') optionables.add(task_type) # If task is expected to inherit goal-level options, register those directly on the task, # by subclassing the goal options registrar and settings its scope to the task scope. if issubclass(task_type, GoalOptionsMixin): subclass_name = b'test_{}_{}_{}'.format( task_type.__name__, task_type.goal_options_registrar_cls.options_scope, task_type.options_scope) optionables.add(type(subclass_name, (task_type.goal_options_registrar_cls, ), {b'options_scope': task_type.options_scope})) extra_scopes.update([si.scope for si in task_type.known_scope_infos()]) optionables.update(Subsystem.closure( set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) | self._build_configuration.subsystems())) # Now default the option values and override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. options = options.copy() if options else {} for s, opts in self.options.items(): scoped_opts = options.setdefault(s, {}) scoped_opts.update(opts) fake_options = create_options_for_optionables( optionables, extra_scopes=extra_scopes, options=options, **kwargs) Subsystem.reset(reset_options=True) Subsystem.set_options(fake_options) context = create_context_from_options(fake_options, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) return context def tearDown(self): """ :API: public """ super(BaseTest, self).tearDown() BuildFile.clear_cache() Subsystem.reset() def target(self, spec): """Resolves the given target address to a Target object. :API: public address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = Address.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. :API: public spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec = CmdLineSpecParser(self.build_root).parse_spec(spec) addresses = list(self.address_mapper.scan_specs([spec])) for address in addresses: self.build_graph.inject_address_closure(address) targets = [self.build_graph.get_target(address) for address in addresses] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. :API: public path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources :API: public path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file(path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict(target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), java_sources=('java_sources=[%s],' % ','.join(map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): """ :API: public """ return self.create_library(path, 'resources', name, sources) def assertUnorderedPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, unordered. :API: public """ actual = list(itertools.islice(actual_iter, len(expected))) self.assertEqual(sorted(expected), sorted(actual)) def assertPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, in order. :API: public """ self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected)))) def assertInFile(self, string, file_path): """Verifies that a string appears in a file :API: public """ with open(file_path) as f: content = f.read() self.assertIn(string, content, '"{}" is not in the file {}:\n{}'.format(string, f.name, content)) def get_bootstrap_options(self, cli_options=()): """Retrieves bootstrap options. :param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`. """ # Can't parse any options without a pants.ini. self.create_file('pants.ini') return OptionsBootstrapper(args=cli_options).get_bootstrap_options().for_global_scope() class LoggingRecorder(object): """Simple logging handler to record warnings.""" def __init__(self): self._records = [] self.level = logging.DEBUG def handle(self, record): self._records.append(record) def _messages_for_level(self, levelname): return ['{}: {}'.format(record.name, record.getMessage()) for record in self._records if record.levelname == levelname] def infos(self): return self._messages_for_level('INFO') def warnings(self): return self._messages_for_level('WARNING') @contextmanager def captured_logging(self, level=None): root_logger = logging.getLogger() old_level = root_logger.level root_logger.setLevel(level or logging.NOTSET) handler = self.LoggingRecorder() root_logger.addHandler(handler) try: yield handler finally: root_logger.setLevel(old_level) root_logger.removeHandler(handler)
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot. :API: public """ def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path. :API: public """ if os.path.basename(relpath).startswith("BUILD"): return relpath else: return os.path.join(relpath, "BUILD") def create_dir(self, relpath): """Creates a directory under the buildroot. :API: public relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_workdir_dir(self, relpath): """Creates a directory under the work directory. :API: public relpath: The relative path to the directory from the work directory. """ path = os.path.join(self.pants_workdir, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents="", mode="wb"): """Writes to a file under the buildroot. :API: public relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def create_workdir_file(self, relpath, contents="", mode="wb"): """Writes to a file under the work directory. :API: public relpath: The relative path to the file from the work directory. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.pants_workdir, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. :API: public relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode="a") return BuildFile(self.address_mapper._project_tree, relpath=self.build_path(relpath)) def make_target(self, spec="", target_type=Target, dependencies=None, derived_from=None, synthetic=False, **kwargs): """Creates a target and injects it into the test's build graph. :API: public :param string spec: The target address spec that locates this target. :param type target_type: The concrete target subclass to create this new target from. :param list dependencies: A list of target instances this new target depends on. :param derived_from: The target this new target was derived from. :type derived_from: :class:`pants.build_graph.target.Target` """ address = Address.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] self.build_graph.inject_target( target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from, synthetic=synthetic ) # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph. # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it. for traversable_dependency_spec in target.traversable_dependency_specs: traversable_dependency_address = Address.parse(traversable_dependency_spec, relative_to=address.spec_path) traversable_dependency_target = self.build_graph.get_target(traversable_dependency_address) if not traversable_dependency_target: raise ValueError( "Tests must make targets for traversable dependency specs ahead of them " "being traversed, {} tried to traverse {} which does not exist.".format( target, traversable_dependency_address ) ) if traversable_dependency_target not in target.dependencies: self.build_graph.inject_dependency(dependent=target.address, dependency=traversable_dependency_address) target.mark_transitive_invalidation_hash_dirty() return target @property def alias_groups(self): """ :API: public """ return BuildFileAliases(targets={"target": Target}) @property def build_ignore_patterns(self): """ :API: public """ return None def setUp(self): """ :API: public """ super(BaseTest, self).setUp() # Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup. clean_global_runtime_state(reset_runtracker=False, reset_subsystem=True) self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT")) self.subprocess_dir = os.path.join(self.build_root, ".pids") self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, ".pants.d") safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[""] = { "pants_workdir": self.pants_workdir, "pants_supportdir": os.path.join(self.build_root, "build-support"), "pants_distdir": os.path.join(self.build_root, "dist"), "pants_configdir": os.path.join(self.build_root, "config"), "pants_subprocessdir": self.subprocess_dir, "cache_key_gen_version": "0-test", } self.options["cache"] = {"read_from": [], "write_to": []} BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.project_tree = FileSystemProjectTree(self.build_root) self.reset_build_graph() def buildroot_files(self, relpath=None): """Returns the set of all files under the test build root. :API: public :param string relpath: If supplied, only collect files from this subtree. :returns: All file paths found. :rtype: set """ def scan(): for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or "")): for f in files: yield os.path.relpath(os.path.join(root, f), self.build_root) return set(scan()) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper( self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns ) self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context( self, for_task_types=None, options=None, passthru_args=None, target_roots=None, console_outstream=None, workspace=None, for_subsystems=None, ): """ :API: public """ # Many tests use source root functionality via the SourceRootConfig.global_instance() # (typically accessed via Target.target_base), so we always set it up, for convenience. optionables = {SourceRootConfig} extra_scopes = set() for_subsystems = for_subsystems or () for subsystem in for_subsystems: if subsystem.options_scope is None: raise TaskError("You must set a scope on your subsystem type before using it in tests.") optionables.add(subsystem) for_task_types = for_task_types or () for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError("You must set a scope on your task type before using it in tests.") optionables.add(task_type) extra_scopes.update([si.scope for si in task_type.known_scope_infos()]) optionables.update( Subsystem.closure( set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) | self._build_configuration.subsystems() ) ) # Now default the option values and override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. options = options.copy() if options else {} for s, opts in self.options.items(): scoped_opts = options.setdefault(s, {}) scoped_opts.update(opts) options = create_options_for_optionables(optionables, extra_scopes=extra_scopes, options=options) Subsystem.reset(reset_options=True) Subsystem.set_options(options) context = create_context( options=options, passthru_args=passthru_args, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace, ) return context def tearDown(self): """ :API: public """ super(BaseTest, self).tearDown() BuildFile.clear_cache() Subsystem.reset() def target(self, spec): """Resolves the given target address to a Target object. :API: public address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = Address.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. :API: public spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec = CmdLineSpecParser(self.build_root).parse_spec(spec) addresses = list(self.address_mapper.scan_specs([spec])) for address in addresses: self.build_graph.inject_address_closure(address) targets = [self.build_graph.get_target(address) for address in addresses] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. :API: public path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources :API: public path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file( path, dedent( """ %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) """ % dict( target_type=target_type, name=name, sources=("sources=%s," % repr(sources) if sources else ""), resources=('resources=["%s"],' % kwargs.get("resources") if "resources" in kwargs else ""), java_sources=( "java_sources=[%s]," % ",".join(map(lambda str_target: '"%s"' % str_target, kwargs.get("java_sources"))) if "java_sources" in kwargs else "" ), provides=("provides=%s," % kwargs.get("provides") if "provides" in kwargs else ""), dependencies=("dependencies=%s," % kwargs.get("dependencies") if "dependencies" in kwargs else ""), ) ), ) return self.target("%s:%s" % (path, name)) def create_resources(self, path, name, *sources): """ :API: public """ return self.create_library(path, "resources", name, sources) def assertUnorderedPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, unordered. :API: public """ actual = list(itertools.islice(actual_iter, len(expected))) self.assertEqual(sorted(expected), sorted(actual)) def assertPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, in order. :API: public """ self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected))))
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot. :API: public """ def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path. :API: public """ if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. :API: public relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_workdir_dir(self, relpath): """Creates a directory under the work directory. :API: public relpath: The relative path to the directory from the work directory. """ path = os.path.join(self.pants_workdir, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. :API: public relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def create_workdir_file(self, relpath, contents='', mode='wb'): """Writes to a file under the work directory. :API: public relpath: The relative path to the file from the work directory. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.pants_workdir, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. :API: public relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') return BuildFile(self.address_mapper._project_tree, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, derived_from=None, synthetic=False, **kwargs): """Creates a target and injects it into the test's build graph. :API: public :param string spec: The target address spec that locates this target. :param type target_type: The concrete target subclass to create this new target from. :param list dependencies: A list of target instances this new target depends on. :param derived_from: The target this new target was derived from. :type derived_from: :class:`pants.build_graph.target.Target` """ address = Address.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] self.build_graph.apply_injectables([target]) self.build_graph.inject_target( target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from, synthetic=synthetic) # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph. # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it. traversables = [ target.compute_dependency_specs(payload=target.payload) ] for dependency_spec in itertools.chain(*traversables): dependency_address = Address.parse(dependency_spec, relative_to=address.spec_path) dependency_target = self.build_graph.get_target(dependency_address) if not dependency_target: raise ValueError( 'Tests must make targets for dependency specs ahead of them ' 'being traversed, {} tried to traverse {} which does not exist.' .format(target, dependency_address)) if dependency_target not in target.dependencies: self.build_graph.inject_dependency( dependent=target.address, dependency=dependency_address) target.mark_transitive_invalidation_hash_dirty() return target @property def alias_groups(self): """ :API: public """ return BuildFileAliases(targets={'target': Target}) @property def build_ignore_patterns(self): """ :API: public """ return None def setUp(self): """ :API: public """ super(BaseTest, self).setUp() # Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup. clean_global_runtime_state(reset_subsystem=True) self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.subprocess_dir = os.path.join(self.build_root, '.pids') self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[GLOBAL_SCOPE] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'pants_subprocessdir': self.subprocess_dir, 'cache_key_gen_version': '0-test', } self.options['cache'] = { 'read_from': [], 'write_to': [], } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.project_tree = FileSystemProjectTree(self.build_root) self.reset_build_graph() def buildroot_files(self, relpath=None): """Returns the set of all files under the test build root. :API: public :param string relpath: If supplied, only collect files from this subtree. :returns: All file paths found. :rtype: set """ def scan(): for root, dirs, files in os.walk( os.path.join(self.build_root, relpath or '')): for f in files: yield os.path.relpath(os.path.join(root, f), self.build_root) return set(scan()) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper( self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns) self.build_graph = MutableBuildGraph( address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, for_subsystems=None, options=None, target_roots=None, console_outstream=None, workspace=None, scheduler=None, **kwargs): """ :API: public :param dict **kwargs: keyword arguments passed in to `create_options_for_optionables`. """ # Many tests use source root functionality via the SourceRootConfig.global_instance(). # (typically accessed via Target.target_base), so we always set it up, for convenience. for_subsystems = set(for_subsystems or ()) for subsystem in for_subsystems: if subsystem.options_scope is None: raise TaskError( 'You must set a scope on your subsystem type before using it in tests.' ) optionables = { SourceRootConfig } | self._build_configuration.subsystems() | for_subsystems for_task_types = for_task_types or () for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError( 'You must set a scope on your task type before using it in tests.' ) optionables.add(task_type) # If task is expected to inherit goal-level options, register those directly on the task, # by subclassing the goal options registrar and settings its scope to the task scope. if issubclass(task_type, GoalOptionsMixin): subclass_name = b'test_{}_{}_{}'.format( task_type.__name__, task_type.goal_options_registrar_cls.options_scope, task_type.options_scope) optionables.add( type(subclass_name, (task_type.goal_options_registrar_cls, ), {b'options_scope': task_type.options_scope})) # Now expand to all deps. all_optionables = set() for optionable in optionables: all_optionables.update(si.optionable_cls for si in optionable.known_scope_infos()) # Now default the option values and override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. options = options.copy() if options else {} for s, opts in self.options.items(): scoped_opts = options.setdefault(s, {}) scoped_opts.update(opts) fake_options = create_options_for_optionables(all_optionables, options=options, **kwargs) Subsystem.reset(reset_options=True) Subsystem.set_options(fake_options) context = create_context_from_options( fake_options, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace, scheduler=scheduler) return context def tearDown(self): """ :API: public """ super(BaseTest, self).tearDown() BuildFile.clear_cache() Subsystem.reset() def target(self, spec): """Resolves the given target address to a Target object. :API: public address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = Address.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. :API: public spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec = CmdLineSpecParser(self.build_root).parse_spec(spec) addresses = list(self.address_mapper.scan_specs([spec])) for address in addresses: self.build_graph.inject_address_closure(address) targets = [ self.build_graph.get_target(address) for address in addresses ] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. :API: public path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources :API: public path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file( path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict( target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), java_sources=('java_sources=[%s],' % ','.join( map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): """ :API: public """ return self.create_library(path, 'resources', name, sources) def assertUnorderedPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, unordered. :API: public """ actual = list(itertools.islice(actual_iter, len(expected))) self.assertEqual(sorted(expected), sorted(actual)) def assertPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, in order. :API: public """ self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected)))) def assertInFile(self, string, file_path): """Verifies that a string appears in a file :API: public """ with open(file_path) as f: content = f.read() self.assertIn( string, content, '"{}" is not in the file {}:\n{}'.format( string, f.name, content)) def get_bootstrap_options(self, cli_options=()): """Retrieves bootstrap options. :param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`. """ # Can't parse any options without a pants.ini. self.create_file('pants.ini') return OptionsBootstrapper( args=cli_options).get_bootstrap_options().for_global_scope() class LoggingRecorder(object): """Simple logging handler to record warnings.""" def __init__(self): self._records = [] self.level = logging.DEBUG def handle(self, record): self._records.append(record) def _messages_for_level(self, levelname): return [ '{}: {}'.format(record.name, record.getMessage()) for record in self._records if record.levelname == levelname ] def infos(self): return self._messages_for_level('INFO') def warnings(self): return self._messages_for_level('WARNING') @contextmanager def captured_logging(self, level=None): root_logger = logging.getLogger() old_level = root_logger.level root_logger.setLevel(level or logging.NOTSET) handler = self.LoggingRecorder() root_logger.addHandler(handler) try: yield handler finally: root_logger.setLevel(old_level) root_logger.removeHandler(handler)
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot. :API: public """ def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path. :API: public """ if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. :API: public relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_workdir_dir(self, relpath): """Creates a directory under the work directory. :API: public relpath: The relative path to the directory from the work directory. """ path = os.path.join(self.pants_workdir, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. :API: public relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def create_workdir_file(self, relpath, contents='', mode='wb'): """Writes to a file under the work directory. :API: public relpath: The relative path to the file from the work directory. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.pants_workdir, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. :API: public relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') return BuildFile(self.address_mapper._project_tree, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, derived_from=None, synthetic=False, **kwargs): """Creates a target and injects it into the test's build graph. :API: public :param string spec: The target address spec that locates this target. :param type target_type: The concrete target subclass to create this new target from. :param list dependencies: A list of target instances this new target depends on. :param derived_from: The target this new target was derived from. :type derived_from: :class:`pants.build_graph.target.Target` """ address = Address.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] self.build_graph.inject_target( target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from, synthetic=synthetic) # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph. # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it. for traversable_dependency_spec in target.traversable_dependency_specs: traversable_dependency_address = Address.parse( traversable_dependency_spec, relative_to=address.spec_path) traversable_dependency_target = self.build_graph.get_target( traversable_dependency_address) if not traversable_dependency_target: raise ValueError( 'Tests must make targets for traversable dependency specs ahead of them ' 'being traversed, {} tried to traverse {} which does not exist.' .format(target, traversable_dependency_address)) if traversable_dependency_target not in target.dependencies: self.build_graph.inject_dependency( dependent=target.address, dependency=traversable_dependency_address) target.mark_transitive_invalidation_hash_dirty() return target @property def alias_groups(self): """ :API: public """ return BuildFileAliases(targets={'target': Target}) @property def build_ignore_patterns(self): """ :API: public """ return None def setUp(self): """ :API: public """ super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.subprocess_dir = os.path.join(self.build_root, '.pids') self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'pants_subprocessdir': self.subprocess_dir, 'cache_key_gen_version': '0-test', } self.options['cache'] = { 'read_from': [], 'write_to': [], } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.project_tree = FileSystemProjectTree(self.build_root) self.address_mapper = BuildFileAddressMapper( self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns) self.build_graph = MutableBuildGraph( address_mapper=self.address_mapper) def buildroot_files(self, relpath=None): """Returns the set of all files under the test build root. :API: public :param string relpath: If supplied, only collect files from this subtree. :returns: All file paths found. :rtype: set """ def scan(): for root, dirs, files in os.walk( os.path.join(self.build_root, relpath or '')): for f in files: yield os.path.relpath(os.path.join(root, f), self.build_root) return set(scan()) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper( self.build_file_parser, FileSystemProjectTree(self.build_root)) self.build_graph = MutableBuildGraph( address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, options=None, passthru_args=None, target_roots=None, console_outstream=None, workspace=None, for_subsystems=None): """ :API: public """ # Many tests use source root functionality via the SourceRootConfig.global_instance() # (typically accessed via Target.target_base), so we always set it up, for convenience. optionables = {SourceRootConfig} extra_scopes = set() for_subsystems = for_subsystems or () for subsystem in for_subsystems: if subsystem.options_scope is None: raise TaskError( 'You must set a scope on your subsystem type before using it in tests.' ) optionables.add(subsystem) for_task_types = for_task_types or () for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError( 'You must set a scope on your task type before using it in tests.' ) optionables.add(task_type) extra_scopes.update( [si.scope for si in task_type.known_scope_infos()]) optionables.update( Subsystem.closure( set([ dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter() ]) | self._build_configuration.subsystems())) # Now default the option values and override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. options = options.copy() if options else {} for s, opts in self.options.items(): scoped_opts = options.setdefault(s, {}) scoped_opts.update(opts) options = create_options_for_optionables(optionables, extra_scopes=extra_scopes, options=options) Subsystem.reset(reset_options=True) Subsystem.set_options(options) context = create_context(options=options, passthru_args=passthru_args, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) return context def tearDown(self): """ :API: public """ super(BaseTest, self).tearDown() BuildFile.clear_cache() Subsystem.reset() def target(self, spec): """Resolves the given target address to a Target object. :API: public address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = Address.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. :API: public spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec = CmdLineSpecParser(self.build_root).parse_spec(spec) addresses = list(self.address_mapper.scan_specs([spec])) for address in addresses: self.build_graph.inject_address_closure(address) targets = [ self.build_graph.get_target(address) for address in addresses ] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. :API: public path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources :API: public path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file( path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict( target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), resources=('resources=["%s"],' % kwargs.get('resources') if 'resources' in kwargs else ''), java_sources=('java_sources=[%s],' % ','.join( map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): """ :API: public """ return self.create_library(path, 'resources', name, sources) def assertUnorderedPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, unordered. :API: public """ actual = list(itertools.islice(actual_iter, len(expected))) self.assertEqual(sorted(expected), sorted(actual)) def assertPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, in order. :API: public """ self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected))))
class GoalRunnerFactory(object): def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast self._spec_excludes = self._global_options.spec_excludes self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns self._build_file_type = self._get_buildfile_type( self._global_options.build_file_rev) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type) self._build_graph = BuildGraph(self._address_mapper) self._spec_parser = CmdLineSpecParser( self._root_dir, self._address_mapper, spec_excludes=self._spec_excludes, exclude_target_regexps=self._global_options.exclude_target_regexp) def _get_buildfile_type(self, build_file_rev): """Selects the BuildFile type for use in a given pants run.""" if build_file_rev: ScmBuildFile.set_rev(build_file_rev) ScmBuildFile.set_scm(get_scm()) return ScmBuildFile else: return FilesystemBuildFile def _expand_goals(self, goals): """Check and populate the requested goals for a given run.""" for goal in goals: if self._address_mapper.from_cache(self._root_dir, goal, must_exist=False).file_exists(): logger.warning( "Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}." .format(goal)) if self._help_request: help_printer = HelpPrinter(self._options) help_printer.print_help() self._exiter(0) self._goals.extend([Goal.by_name(goal) for goal in goals]) def _expand_specs(self, specs, fail_fast): """Populate the BuildGraph and target list from a set of input specs.""" with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag)) for spec in specs: for address in self._spec_parser.parse_addresses( spec, fail_fast): self._build_graph.inject_address_closure(address) target = self._build_graph.get_target(address) if tag_filter(target): self._targets.append(target) def _is_quiet(self): return any( goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain def _setup_context(self): # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self._run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self._address_mapper, self._build_file_parser) with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]): self._expand_goals(self._requested_goals) self._expand_specs(self._target_specs, self._fail_fast) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self._run_tracker.run_info.add_scm_info() # Update the Reporting settings now that we have options and goal info. invalidation_report = self._reporting.update_reporting( self._global_options, self._is_quiet(), self._run_tracker) context = Context(options=self._options, run_tracker=self._run_tracker, target_roots=self._targets, requested_goals=self._requested_goals, build_graph=self._build_graph, build_file_parser=self._build_file_parser, address_mapper=self._address_mapper, spec_excludes=self._spec_excludes, invalidation_report=invalidation_report) return context, invalidation_report def setup(self): context, invalidation_report = self._setup_context() return GoalRunner(context=context, goals=self._goals, kill_nailguns=self._kill_nailguns, run_tracker=self._run_tracker, invalidation_report=invalidation_report)
class GoalRunnerFactory(object): def __init__(self, root_dir, options, build_config, run_tracker, reporting, build_graph=None, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param BuildGraph build_graph: A BuildGraph instance (for graph reuse, optional). :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast # Will be provided through context.address_mapper.build_ignore_patterns. self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns pants_ignore = self._global_options.pants_ignore or [] self._project_tree = self._get_project_tree(self._global_options.build_file_rev, pants_ignore) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) build_ignore_patterns = self._global_options.ignore_patterns or [] self._address_mapper = BuildFileAddressMapper( self._build_file_parser, self._project_tree, build_ignore_patterns, exclude_target_regexps=self._global_options.exclude_target_regexp ) self._build_graph = self._select_buildgraph(self._global_options.enable_v2_engine, self._global_options.pants_ignore, build_graph) def _select_buildgraph(self, use_engine, path_ignore_patterns, cached_buildgraph=None): """Selects a BuildGraph to use then constructs and returns it. :param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph. :param list path_ignore_patterns: The path ignore patterns from `--pants-ignore`. :param LegacyBuildGraph cached_buildgraph: A cached graph to reuse, if available. """ if cached_buildgraph is not None: return cached_buildgraph elif use_engine: root_specs = EngineInitializer.parse_commandline_to_spec_roots(options=self._options, build_root=self._root_dir) graph_helper = EngineInitializer.setup_legacy_graph(path_ignore_patterns) return graph_helper.create_graph(root_specs) else: return MutableBuildGraph(self._address_mapper) def _get_project_tree(self, build_file_rev, pants_ignore): """Creates the project tree for build files for use in a given pants run.""" if build_file_rev: return ScmProjectTree(self._root_dir, get_scm(), build_file_rev, pants_ignore) else: return FileSystemProjectTree(self._root_dir, pants_ignore) def _expand_goals(self, goals): """Check and populate the requested goals for a given run.""" for goal in goals: try: self._address_mapper.resolve_spec(goal) logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) except AddressLookupError: pass if self._help_request: help_printer = HelpPrinter(self._options) result = help_printer.print_help() self._exiter(result) self._goals.extend([Goal.by_name(goal) for goal in goals]) def _expand_specs(self, spec_strs, fail_fast): """Populate the BuildGraph and target list from a set of input specs.""" with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag)) # Parse all specs into unique Spec objects. spec_parser = CmdLineSpecParser(self._root_dir) specs = OrderedSet() for spec_str in spec_strs: specs.add(spec_parser.parse_spec(spec_str)) # Then scan them to generate unique Addresses. for address in self._build_graph.inject_specs_closure(specs, fail_fast): target = self._build_graph.get_target(address) if tag_filter(target): self._targets.append(target) def _maybe_launch_pantsd(self): """Launches pantsd if configured to do so.""" if self._global_options.enable_pantsd: # Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on. with self._run_tracker.new_workunit(name='pantsd', labels=[WorkUnitLabel.SETUP]): pantsd_launcher = PantsDaemonLauncher.Factory.global_instance().create(EngineInitializer) pantsd_launcher.maybe_launch() def _is_quiet(self): return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain def _setup_context(self): self._maybe_launch_pantsd() with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]): self._expand_goals(self._requested_goals) self._expand_specs(self._target_specs, self._fail_fast) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self._run_tracker.run_info.add_scm_info() # Update the Reporting settings now that we have options and goal info. invalidation_report = self._reporting.update_reporting(self._global_options, self._is_quiet(), self._run_tracker) context = Context(options=self._options, run_tracker=self._run_tracker, target_roots=self._targets, requested_goals=self._requested_goals, build_graph=self._build_graph, build_file_parser=self._build_file_parser, address_mapper=self._address_mapper, invalidation_report=invalidation_report) return context, invalidation_report def setup(self): context, invalidation_report = self._setup_context() return GoalRunner(context=context, goals=self._goals, kill_nailguns=self._kill_nailguns, run_tracker=self._run_tracker, invalidation_report=invalidation_report, exiter=self._exiter)
def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper( self.build_file_parser, FileSystemProjectTree(self.build_root)) self.build_graph = MutableBuildGraph( address_mapper=self.address_mapper)
class GoalRunnerFactory(object): def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast # Will be provided through context.address_mapper.build_ignore_patterns. self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns self._project_tree = self._get_project_tree( self._global_options.build_file_rev) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) build_ignore_patterns = self._global_options.ignore_patterns or [] self._address_mapper = BuildFileAddressMapper( self._build_file_parser, self._project_tree, build_ignore_patterns, exclude_target_regexps=self._global_options.exclude_target_regexp) self._build_graph = MutableBuildGraph(self._address_mapper) def _get_project_tree(self, build_file_rev): """Creates the project tree for build files for use in a given pants run.""" if build_file_rev: return ScmProjectTree(self._root_dir, get_scm(), build_file_rev) else: return FileSystemProjectTree(self._root_dir) def _expand_goals(self, goals): """Check and populate the requested goals for a given run.""" for goal in goals: try: self._address_mapper.resolve_spec(goal) logger.warning( "Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}." .format(goal)) except AddressLookupError: pass if self._help_request: help_printer = HelpPrinter(self._options) result = help_printer.print_help() self._exiter(result) self._goals.extend([Goal.by_name(goal) for goal in goals]) def _expand_specs(self, spec_strs, fail_fast): """Populate the BuildGraph and target list from a set of input specs.""" with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag)) # Parse all specs into unique Spec objects. spec_parser = CmdLineSpecParser(self._root_dir) specs = OrderedSet() for spec_str in spec_strs: specs.add(spec_parser.parse_spec(spec_str)) # Then scan them to generate unique Addresses. for address in self._build_graph.inject_specs_closure( specs, fail_fast): target = self._build_graph.get_target(address) if tag_filter(target): self._targets.append(target) def _maybe_launch_pantsd(self): """Launches pantsd if configured to do so.""" if self._global_options.enable_pantsd: # Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on. with self._run_tracker.new_workunit(name='pantsd', labels=[WorkUnitLabel.SETUP]): PantsDaemonLauncher.global_instance().maybe_launch() def _is_quiet(self): return any( goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain def _setup_context(self): self._maybe_launch_pantsd() with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]): self._expand_goals(self._requested_goals) self._expand_specs(self._target_specs, self._fail_fast) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self._run_tracker.run_info.add_scm_info() # Update the Reporting settings now that we have options and goal info. invalidation_report = self._reporting.update_reporting( self._global_options, self._is_quiet(), self._run_tracker) context = Context(options=self._options, run_tracker=self._run_tracker, target_roots=self._targets, requested_goals=self._requested_goals, build_graph=self._build_graph, build_file_parser=self._build_file_parser, address_mapper=self._address_mapper, invalidation_report=invalidation_report) return context, invalidation_report def setup(self): context, invalidation_report = self._setup_context() return GoalRunner(context=context, goals=self._goals, kill_nailguns=self._kill_nailguns, run_tracker=self._run_tracker, invalidation_report=invalidation_report, exiter=self._exiter)
def build_file_address_mapper(self): return BuildFileAddressMapper(self.build_file_parser, FileSystemProjectTree(self.build_root), build_ignore_patterns=['subdir'])
def build_file_address_mapper(self): return BuildFileAddressMapper(self.build_file_parser, FileSystemProjectTree(self.build_root))
def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FileSystemProjectTree(self.build_root)) self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns) self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper)
class GoalRunnerFactory(object): def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit): """ :param str root_dir: The root directory of the pants workspace (aka the "build root"). :param Options options: The global, pre-initialized Options instance. :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance. :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance. :param Reporting reporting: The global, pre-initialized Reporting instance. :param func exiter: A function that accepts an exit code value and exits (for tests, Optional). """ self._root_dir = root_dir self._options = options self._build_config = build_config self._run_tracker = run_tracker self._reporting = reporting self._exiter = exiter self._goals = [] self._targets = [] self._requested_goals = self._options.goals self._target_specs = self._options.target_specs self._help_request = self._options.help_request self._global_options = options.for_global_scope() self._tag = self._global_options.tag self._fail_fast = self._global_options.fail_fast self._spec_excludes = self._global_options.spec_excludes self._explain = self._global_options.explain self._kill_nailguns = self._global_options.kill_nailguns self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev) self._build_file_parser = BuildFileParser(self._build_config, self._root_dir) self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type) self._build_graph = BuildGraph(self._address_mapper) self._spec_parser = CmdLineSpecParser( self._root_dir, self._address_mapper, spec_excludes=self._spec_excludes, exclude_target_regexps=self._global_options.exclude_target_regexp ) def _get_buildfile_type(self, build_file_rev): """Selects the BuildFile type for use in a given pants run.""" if build_file_rev: ScmBuildFile.set_rev(build_file_rev) ScmBuildFile.set_scm(get_scm()) return ScmBuildFile else: return FilesystemBuildFile def _expand_goals(self, goals): """Check and populate the requested goals for a given run.""" for goal in goals: if self._address_mapper.from_cache(self._root_dir, goal, must_exist=False).file_exists(): logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be " "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)) if self._help_request: help_printer = HelpPrinter(self._options) help_printer.print_help() self._exiter(0) self._goals.extend([Goal.by_name(goal) for goal in goals]) def _expand_specs(self, specs, fail_fast): """Populate the BuildGraph and target list from a set of input specs.""" with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]): def filter_for_tag(tag): return lambda target: tag in map(str, target.tags) tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag)) for spec in specs: for address in self._spec_parser.parse_addresses(spec, fail_fast): self._build_graph.inject_address_closure(address) target = self._build_graph.get_target(address) if tag_filter(target): self._targets.append(target) def _is_quiet(self): return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain def _setup_context(self): # TODO(John Sirois): Kill when source root registration is lifted out of BUILD files. with self._run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]): source_root_bootstrapper = SourceRootBootstrapper.global_instance() source_root_bootstrapper.bootstrap(self._address_mapper, self._build_file_parser) with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]): self._expand_goals(self._requested_goals) self._expand_specs(self._target_specs, self._fail_fast) # Now that we've parsed the bootstrap BUILD files, and know about the SCM system. self._run_tracker.run_info.add_scm_info() # Update the Reporting settings now that we have options and goal info. invalidation_report = self._reporting.update_reporting(self._global_options, self._is_quiet(), self._run_tracker) context = Context(options=self._options, run_tracker=self._run_tracker, target_roots=self._targets, requested_goals=self._requested_goals, build_graph=self._build_graph, build_file_parser=self._build_file_parser, address_mapper=self._address_mapper, spec_excludes=self._spec_excludes, invalidation_report=invalidation_report) return context, invalidation_report def setup(self): context, invalidation_report = self._setup_context() return GoalRunner(context=context, goals=self._goals, kill_nailguns=self._kill_nailguns, run_tracker=self._run_tracker, invalidation_report=invalidation_report)