Ejemplo n.º 1
0
 def reset_build_graph(self):
     """Start over with a fresh build graph with no targets in it."""
     self.address_mapper = BuildFileAddressMapper(
         self.build_file_parser,
         self.project_tree,
         build_ignore_patterns=self.build_ignore_patterns)
     self.build_graph = MutableBuildGraph(
         address_mapper=self.address_mapper)
Ejemplo n.º 2
0
  def scan(self, root=None):
    """Scans and parses all BUILD files found under ``root``.

    Only BUILD files found under ``root`` are parsed as roots in the graph, but any dependencies of
    targets parsed in the root tree's BUILD files will be followed and this may lead to BUILD files
    outside of ``root`` being parsed and included in the returned build graph.

    :API: public

    :param string root: The path to scan; by default, the build root.
    :returns: A new build graph encapsulating the targets found.
    """
    build_graph = MutableBuildGraph(self.address_mapper)
    for address in self.address_mapper.scan_addresses(root, spec_excludes=self._spec_excludes):
      build_graph.inject_address_closure(address)
    return build_graph
Ejemplo n.º 3
0
    def scan(self, root=None):
        """Scans and parses all BUILD files found under ``root``.

    Only BUILD files found under ``root`` are parsed as roots in the graph, but any dependencies of
    targets parsed in the root tree's BUILD files will be followed and this may lead to BUILD files
    outside of ``root`` being parsed and included in the returned build graph.

    :API: public

    :param string root: The path to scan; by default, the build root.
    :returns: A new build graph encapsulating the targets found.
    """
        build_graph = MutableBuildGraph(self.address_mapper)
        for address in self.address_mapper.scan_addresses(root):
            build_graph.inject_address_closure(address)
        return build_graph
Ejemplo n.º 4
0
    def __init__(self,
                 root_dir,
                 options,
                 build_config,
                 run_tracker,
                 reporting,
                 exiter=sys.exit):
        """
    :param str root_dir: The root directory of the pants workspace (aka the "build root").
    :param Options options: The global, pre-initialized Options instance.
    :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
    :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
    :param Reporting reporting: The global, pre-initialized Reporting instance.
    :param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
    """
        self._root_dir = root_dir
        self._options = options
        self._build_config = build_config
        self._run_tracker = run_tracker
        self._reporting = reporting
        self._exiter = exiter

        self._goals = []
        self._targets = []
        self._requested_goals = self._options.goals
        self._target_specs = self._options.target_specs
        self._help_request = self._options.help_request

        self._global_options = options.for_global_scope()
        self._tag = self._global_options.tag
        self._fail_fast = self._global_options.fail_fast
        # Will be provided through context.address_mapper.build_ignore_patterns.
        self._explain = self._global_options.explain
        self._kill_nailguns = self._global_options.kill_nailguns

        self._project_tree = self._get_project_tree(
            self._global_options.build_file_rev)
        self._build_file_parser = BuildFileParser(self._build_config,
                                                  self._root_dir)
        build_ignore_patterns = self._global_options.ignore_patterns or []
        self._address_mapper = BuildFileAddressMapper(
            self._build_file_parser,
            self._project_tree,
            build_ignore_patterns,
            exclude_target_regexps=self._global_options.exclude_target_regexp)
        self._build_graph = MutableBuildGraph(self._address_mapper)
    def test_create_bad_targets(self):
        with self.assertRaises(TypeError):
            BuildFileAliases(targets={'fred': object()})

        target = Target('fred', Address.parse('a:b'),
                        MutableBuildGraph(address_mapper=None))
        with self.assertRaises(TypeError):
            BuildFileAliases(targets={'fred': target})
Ejemplo n.º 6
0
    def setUp(self):
        """
    :API: public
    """
        super(BaseTest, self).setUp()
        Goal.clear()
        Subsystem.reset()

        self.real_build_root = BuildRoot().path

        self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
        self.subprocess_dir = os.path.join(self.build_root, '.pids')
        self.addCleanup(safe_rmtree, self.build_root)

        self.pants_workdir = os.path.join(self.build_root, '.pants.d')
        safe_mkdir(self.pants_workdir)

        self.options = defaultdict(dict)  # scope -> key-value mapping.
        self.options[''] = {
            'pants_workdir': self.pants_workdir,
            'pants_supportdir': os.path.join(self.build_root, 'build-support'),
            'pants_distdir': os.path.join(self.build_root, 'dist'),
            'pants_configdir': os.path.join(self.build_root, 'config'),
            'pants_subprocessdir': self.subprocess_dir,
            'cache_key_gen_version': '0-test',
        }
        self.options['cache'] = {
            'read_from': [],
            'write_to': [],
        }

        BuildRoot().path = self.build_root
        self.addCleanup(BuildRoot().reset)

        self._build_configuration = BuildConfiguration()
        self._build_configuration.register_aliases(self.alias_groups)
        self.build_file_parser = BuildFileParser(self._build_configuration,
                                                 self.build_root)
        self.project_tree = FileSystemProjectTree(self.build_root)
        self.address_mapper = BuildFileAddressMapper(
            self.build_file_parser,
            self.project_tree,
            build_ignore_patterns=self.build_ignore_patterns)
        self.build_graph = MutableBuildGraph(
            address_mapper=self.address_mapper)
Ejemplo n.º 7
0
    def _init_graph(self,
                    use_engine,
                    pants_ignore_patterns,
                    build_ignore_patterns,
                    exclude_target_regexps,
                    target_specs,
                    workdir,
                    graph_helper=None,
                    subproject_build_roots=None):
        """Determine the BuildGraph, AddressMapper and spec_roots for a given run.

    :param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph.
    :param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'.
    :param list build_ignore_patterns: The build ignore patterns from '--build-ignore',
                                       applied during BUILD file searching.
    :param str workdir: The pants workdir.
    :param list exclude_target_regexps: Regular expressions for targets to be excluded.
    :param list target_specs: The original target specs.
    :param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction,
                                           if available. This would usually come from the daemon.
    :returns: A tuple of (BuildGraph, AddressMapper, opt Scheduler, spec_roots).
    """
        # N.B. Use of the daemon implies use of the v2 engine.
        if graph_helper or use_engine:
            # The daemon may provide a `graph_helper`. If that's present, use it for graph construction.
            if not graph_helper:
                native = Native.create(self._global_options)
                native.set_panic_handler()
                graph_helper = EngineInitializer.setup_legacy_graph(
                    pants_ignore_patterns,
                    workdir,
                    self._global_options.build_file_imports,
                    native=native,
                    build_file_aliases=self._build_config.registered_aliases(),
                    build_ignore_patterns=build_ignore_patterns,
                    exclude_target_regexps=exclude_target_regexps,
                    subproject_roots=subproject_build_roots,
                    include_trace_on_error=self._options.for_global_scope(
                    ).print_exception_stacktrace)

            target_roots = TargetRoots.create(
                options=self._options,
                build_root=self._root_dir,
                change_calculator=graph_helper.change_calculator)
            graph, address_mapper = graph_helper.create_build_graph(
                target_roots, self._root_dir)
            return graph, address_mapper, graph_helper.scheduler, target_roots.as_specs(
            )
        else:
            spec_roots = TargetRoots.parse_specs(target_specs, self._root_dir)
            address_mapper = BuildFileAddressMapper(
                self._build_file_parser,
                get_project_tree(self._global_options), build_ignore_patterns,
                exclude_target_regexps, subproject_build_roots)
            return MutableBuildGraph(
                address_mapper), address_mapper, None, spec_roots
Ejemplo n.º 8
0
  def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
    """
    :param str root_dir: The root directory of the pants workspace (aka the "build root").
    :param Options options: The global, pre-initialized Options instance.
    :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
    :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
    :param Reporting reporting: The global, pre-initialized Reporting instance.
    :param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
    """
    self._root_dir = root_dir
    self._options = options
    self._build_config = build_config
    self._run_tracker = run_tracker
    self._reporting = reporting
    self._exiter = exiter

    self._goals = []
    self._targets = []
    self._requested_goals = self._options.goals
    self._target_specs = self._options.target_specs
    self._help_request = self._options.help_request

    self._global_options = options.for_global_scope()
    self._tag = self._global_options.tag
    self._fail_fast = self._global_options.fail_fast
    # Will be provided through context.address_mapper.build_ignore_patterns.
    self._spec_excludes = None
    self._explain = self._global_options.explain
    self._kill_nailguns = self._global_options.kill_nailguns

    self._project_tree = self._get_project_tree(self._global_options.build_file_rev)
    self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
    build_ignore_patterns = self._global_options.ignore_patterns or []
    build_ignore_patterns.extend(BuildFile._spec_excludes_to_gitignore_syntax(self._root_dir,
                                                                              self._global_options.spec_excludes))
    self._address_mapper = BuildFileAddressMapper(
      self._build_file_parser,
      self._project_tree,
      build_ignore_patterns,
      exclude_target_regexps=self._global_options.exclude_target_regexp
    )
    self._build_graph = MutableBuildGraph(self._address_mapper)
Ejemplo n.º 9
0
    def _select_buildgraph(self,
                           use_engine,
                           path_ignore_patterns,
                           cached_buildgraph=None):
        """Selects a BuildGraph to use then constructs and returns it.

    :param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph.
    :param list path_ignore_patterns: The path ignore patterns from `--pants-ignore`.
    :param LegacyBuildGraph cached_buildgraph: A cached graph to reuse, if available.
    """
        if cached_buildgraph is not None:
            return cached_buildgraph
        elif use_engine:
            root_specs = EngineInitializer.parse_commandline_to_spec_roots(
                options=self._options, build_root=self._root_dir)
            graph_helper = EngineInitializer.setup_legacy_graph(
                path_ignore_patterns)
            return graph_helper.create_graph(root_specs)
        else:
            return MutableBuildGraph(self._address_mapper)
Ejemplo n.º 10
0
  def setUp(self):
    """
    :API: public
    """
    super(BaseTest, self).setUp()
    Goal.clear()
    Subsystem.reset()

    self.real_build_root = BuildRoot().path

    self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
    self.subprocess_dir = os.path.join(self.build_root, '.pids')
    self.addCleanup(safe_rmtree, self.build_root)

    self.pants_workdir = os.path.join(self.build_root, '.pants.d')
    safe_mkdir(self.pants_workdir)

    self.options = defaultdict(dict)  # scope -> key-value mapping.
    self.options[''] = {
      'pants_workdir': self.pants_workdir,
      'pants_supportdir': os.path.join(self.build_root, 'build-support'),
      'pants_distdir': os.path.join(self.build_root, 'dist'),
      'pants_configdir': os.path.join(self.build_root, 'config'),
      'pants_subprocessdir': self.subprocess_dir,
      'cache_key_gen_version': '0-test',
    }
    self.options['cache'] = {
      'read_from': [],
      'write_to': [],
    }

    BuildRoot().path = self.build_root
    self.addCleanup(BuildRoot().reset)

    self._build_configuration = BuildConfiguration()
    self._build_configuration.register_aliases(self.alias_groups)
    self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
    self.project_tree = FileSystemProjectTree(self.build_root)
    self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree,
                                                 build_ignore_patterns=self.build_ignore_patterns)
    self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
Ejemplo n.º 11
0
class BaseTest(unittest.TestCase):
    """A baseclass useful for tests requiring a temporary buildroot.

  :API: public

  """
    def build_path(self, relpath):
        """Returns the canonical BUILD file path for the given relative build path.

    :API: public
    """
        if os.path.basename(relpath).startswith('BUILD'):
            return relpath
        else:
            return os.path.join(relpath, 'BUILD')

    def create_dir(self, relpath):
        """Creates a directory under the buildroot.

    :API: public

    relpath: The relative path to the directory from the build root.
    """
        path = os.path.join(self.build_root, relpath)
        safe_mkdir(path)
        return path

    def create_workdir_dir(self, relpath):
        """Creates a directory under the work directory.

    :API: public

    relpath: The relative path to the directory from the work directory.
    """
        path = os.path.join(self.pants_workdir, relpath)
        safe_mkdir(path)
        return path

    def create_file(self, relpath, contents='', mode='wb'):
        """Writes to a file under the buildroot.

    :API: public

    relpath:  The relative path to the file from the build root.
    contents: A string containing the contents of the file - '' by default..
    mode:     The mode to write to the file in - over-write by default.
    """
        path = os.path.join(self.build_root, relpath)
        with safe_open(path, mode=mode) as fp:
            fp.write(contents)
        return path

    def create_workdir_file(self, relpath, contents='', mode='wb'):
        """Writes to a file under the work directory.

    :API: public

    relpath:  The relative path to the file from the work directory.
    contents: A string containing the contents of the file - '' by default..
    mode:     The mode to write to the file in - over-write by default.
    """
        path = os.path.join(self.pants_workdir, relpath)
        with safe_open(path, mode=mode) as fp:
            fp.write(contents)
        return path

    def add_to_build_file(self, relpath, target):
        """Adds the given target specification to the BUILD file at relpath.

    :API: public

    relpath: The relative path to the BUILD file from the build root.
    target:  A string containing the target definition as it would appear in a BUILD file.
    """
        self.create_file(self.build_path(relpath), target, mode='a')
        return BuildFile(self.address_mapper._project_tree,
                         relpath=self.build_path(relpath))

    def make_target(self,
                    spec='',
                    target_type=Target,
                    dependencies=None,
                    derived_from=None,
                    synthetic=False,
                    **kwargs):
        """Creates a target and injects it into the test's build graph.

    :API: public

    :param string spec: The target address spec that locates this target.
    :param type target_type: The concrete target subclass to create this new target from.
    :param list dependencies: A list of target instances this new target depends on.
    :param derived_from: The target this new target was derived from.
    :type derived_from: :class:`pants.build_graph.target.Target`
    """
        address = Address.parse(spec)
        target = target_type(name=address.target_name,
                             address=address,
                             build_graph=self.build_graph,
                             **kwargs)
        dependencies = dependencies or []

        self.build_graph.apply_injectables([target])
        self.build_graph.inject_target(
            target,
            dependencies=[dep.address for dep in dependencies],
            derived_from=derived_from,
            synthetic=synthetic)

        # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
        # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
        traversables = [
            target.compute_dependency_specs(payload=target.payload)
        ]

        for dependency_spec in itertools.chain(*traversables):
            dependency_address = Address.parse(dependency_spec,
                                               relative_to=address.spec_path)
            dependency_target = self.build_graph.get_target(dependency_address)
            if not dependency_target:
                raise ValueError(
                    'Tests must make targets for dependency specs ahead of them '
                    'being traversed, {} tried to traverse {} which does not exist.'
                    .format(target, dependency_address))
            if dependency_target not in target.dependencies:
                self.build_graph.inject_dependency(
                    dependent=target.address, dependency=dependency_address)
                target.mark_transitive_invalidation_hash_dirty()

        return target

    @property
    def alias_groups(self):
        """
    :API: public
    """
        return BuildFileAliases(targets={'target': Target})

    @property
    def build_ignore_patterns(self):
        """
    :API: public
    """
        return None

    def setUp(self):
        """
    :API: public
    """
        super(BaseTest, self).setUp()
        # Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup.
        clean_global_runtime_state(reset_subsystem=True)

        self.real_build_root = BuildRoot().path

        self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
        self.subprocess_dir = os.path.join(self.build_root, '.pids')
        self.addCleanup(safe_rmtree, self.build_root)

        self.pants_workdir = os.path.join(self.build_root, '.pants.d')
        safe_mkdir(self.pants_workdir)

        self.options = defaultdict(dict)  # scope -> key-value mapping.
        self.options[GLOBAL_SCOPE] = {
            'pants_workdir': self.pants_workdir,
            'pants_supportdir': os.path.join(self.build_root, 'build-support'),
            'pants_distdir': os.path.join(self.build_root, 'dist'),
            'pants_configdir': os.path.join(self.build_root, 'config'),
            'pants_subprocessdir': self.subprocess_dir,
            'cache_key_gen_version': '0-test',
        }
        self.options['cache'] = {
            'read_from': [],
            'write_to': [],
        }

        BuildRoot().path = self.build_root
        self.addCleanup(BuildRoot().reset)

        self._build_configuration = BuildConfiguration()
        self._build_configuration.register_aliases(self.alias_groups)
        self.build_file_parser = BuildFileParser(self._build_configuration,
                                                 self.build_root)
        self.project_tree = FileSystemProjectTree(self.build_root)
        self.reset_build_graph()

    def buildroot_files(self, relpath=None):
        """Returns the set of all files under the test build root.

    :API: public

    :param string relpath: If supplied, only collect files from this subtree.
    :returns: All file paths found.
    :rtype: set
    """
        def scan():
            for root, dirs, files in os.walk(
                    os.path.join(self.build_root, relpath or '')):
                for f in files:
                    yield os.path.relpath(os.path.join(root, f),
                                          self.build_root)

        return set(scan())

    def reset_build_graph(self):
        """Start over with a fresh build graph with no targets in it."""
        self.address_mapper = BuildFileAddressMapper(
            self.build_file_parser,
            self.project_tree,
            build_ignore_patterns=self.build_ignore_patterns)
        self.build_graph = MutableBuildGraph(
            address_mapper=self.address_mapper)

    def set_options_for_scope(self, scope, **kwargs):
        self.options[scope].update(kwargs)

    def context(self,
                for_task_types=None,
                for_subsystems=None,
                options=None,
                target_roots=None,
                console_outstream=None,
                workspace=None,
                scheduler=None,
                **kwargs):
        """
    :API: public

    :param dict **kwargs: keyword arguments passed in to `create_options_for_optionables`.
    """
        # Many tests use source root functionality via the SourceRootConfig.global_instance().
        # (typically accessed via Target.target_base), so we always set it up, for convenience.
        for_subsystems = set(for_subsystems or ())
        for subsystem in for_subsystems:
            if subsystem.options_scope is None:
                raise TaskError(
                    'You must set a scope on your subsystem type before using it in tests.'
                )

        optionables = {
            SourceRootConfig
        } | self._build_configuration.subsystems() | for_subsystems

        for_task_types = for_task_types or ()
        for task_type in for_task_types:
            scope = task_type.options_scope
            if scope is None:
                raise TaskError(
                    'You must set a scope on your task type before using it in tests.'
                )
            optionables.add(task_type)
            # If task is expected to inherit goal-level options, register those directly on the task,
            # by subclassing the goal options registrar and settings its scope to the task scope.
            if issubclass(task_type, GoalOptionsMixin):
                subclass_name = b'test_{}_{}_{}'.format(
                    task_type.__name__,
                    task_type.goal_options_registrar_cls.options_scope,
                    task_type.options_scope)
                optionables.add(
                    type(subclass_name,
                         (task_type.goal_options_registrar_cls, ),
                         {b'options_scope': task_type.options_scope}))

        # Now expand to all deps.
        all_optionables = set()
        for optionable in optionables:
            all_optionables.update(si.optionable_cls
                                   for si in optionable.known_scope_infos())

        # Now default the option values and override with any caller-specified values.
        # TODO(benjy): Get rid of the options arg, and require tests to call set_options.
        options = options.copy() if options else {}
        for s, opts in self.options.items():
            scoped_opts = options.setdefault(s, {})
            scoped_opts.update(opts)

        fake_options = create_options_for_optionables(all_optionables,
                                                      options=options,
                                                      **kwargs)

        Subsystem.reset(reset_options=True)
        Subsystem.set_options(fake_options)

        context = create_context_from_options(
            fake_options,
            target_roots=target_roots,
            build_graph=self.build_graph,
            build_file_parser=self.build_file_parser,
            address_mapper=self.address_mapper,
            console_outstream=console_outstream,
            workspace=workspace,
            scheduler=scheduler)
        return context

    def tearDown(self):
        """
    :API: public
    """
        super(BaseTest, self).tearDown()
        BuildFile.clear_cache()
        Subsystem.reset()

    def target(self, spec):
        """Resolves the given target address to a Target object.

    :API: public

    address: The BUILD target address to resolve.

    Returns the corresponding Target or else None if the address does not point to a defined Target.
    """
        address = Address.parse(spec)
        self.build_graph.inject_address_closure(address)
        return self.build_graph.get_target(address)

    def targets(self, spec):
        """Resolves a target spec to one or more Target objects.

    :API: public

    spec: Either BUILD target address or else a target glob using the siblings ':' or
          descendants '::' suffixes.

    Returns the set of all Targets found.
    """

        spec = CmdLineSpecParser(self.build_root).parse_spec(spec)
        addresses = list(self.address_mapper.scan_specs([spec]))
        for address in addresses:
            self.build_graph.inject_address_closure(address)
        targets = [
            self.build_graph.get_target(address) for address in addresses
        ]
        return targets

    def create_files(self, path, files):
        """Writes to a file under the buildroot with contents same as file name.

    :API: public

     path:  The relative path to the file from the build root.
     files: List of file names.
    """
        for f in files:
            self.create_file(os.path.join(path, f), contents=f)

    def create_library(self, path, target_type, name, sources=None, **kwargs):
        """Creates a library target of given type at the BUILD file at path with sources

    :API: public

     path: The relative path to the BUILD file from the build root.
     target_type: valid pants target type.
     name: Name of the library target.
     sources: List of source file at the path relative to path.
     **kwargs: Optional attributes that can be set for any library target.
       Currently it includes support for resources, java_sources, provides
       and dependencies.
    """
        if sources:
            self.create_files(path, sources)
        self.add_to_build_file(
            path,
            dedent('''
          %(target_type)s(name='%(name)s',
            %(sources)s
            %(java_sources)s
            %(provides)s
            %(dependencies)s
          )
        ''' % dict(
                target_type=target_type,
                name=name,
                sources=('sources=%s,' % repr(sources) if sources else ''),
                java_sources=('java_sources=[%s],' % ','.join(
                    map(lambda str_target: '"%s"' % str_target,
                        kwargs.get('java_sources')))
                              if 'java_sources' in kwargs else ''),
                provides=('provides=%s,' % kwargs.get('provides')
                          if 'provides' in kwargs else ''),
                dependencies=('dependencies=%s,' % kwargs.get('dependencies')
                              if 'dependencies' in kwargs else ''),
            )))
        return self.target('%s:%s' % (path, name))

    def create_resources(self, path, name, *sources):
        """
    :API: public
    """
        return self.create_library(path, 'resources', name, sources)

    def assertUnorderedPrefixEqual(self, expected, actual_iter):
        """Consumes len(expected) items from the given iter, and asserts that they match, unordered.

    :API: public
    """
        actual = list(itertools.islice(actual_iter, len(expected)))
        self.assertEqual(sorted(expected), sorted(actual))

    def assertPrefixEqual(self, expected, actual_iter):
        """Consumes len(expected) items from the given iter, and asserts that they match, in order.

    :API: public
    """
        self.assertEqual(expected,
                         list(itertools.islice(actual_iter, len(expected))))

    def assertInFile(self, string, file_path):
        """Verifies that a string appears in a file

    :API: public
    """

        with open(file_path) as f:
            content = f.read()
            self.assertIn(
                string, content, '"{}" is not in the file {}:\n{}'.format(
                    string, f.name, content))

    def get_bootstrap_options(self, cli_options=()):
        """Retrieves bootstrap options.

    :param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`.
    """
        # Can't parse any options without a pants.ini.
        self.create_file('pants.ini')
        return OptionsBootstrapper(
            args=cli_options).get_bootstrap_options().for_global_scope()

    class LoggingRecorder(object):
        """Simple logging handler to record warnings."""
        def __init__(self):
            self._records = []
            self.level = logging.DEBUG

        def handle(self, record):
            self._records.append(record)

        def _messages_for_level(self, levelname):
            return [
                '{}: {}'.format(record.name, record.getMessage())
                for record in self._records if record.levelname == levelname
            ]

        def infos(self):
            return self._messages_for_level('INFO')

        def warnings(self):
            return self._messages_for_level('WARNING')

    @contextmanager
    def captured_logging(self, level=None):
        root_logger = logging.getLogger()

        old_level = root_logger.level
        root_logger.setLevel(level or logging.NOTSET)

        handler = self.LoggingRecorder()
        root_logger.addHandler(handler)
        try:
            yield handler
        finally:
            root_logger.setLevel(old_level)
            root_logger.removeHandler(handler)
Ejemplo n.º 12
0
class GoalRunnerFactory(object):
  def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
    """
    :param str root_dir: The root directory of the pants workspace (aka the "build root").
    :param Options options: The global, pre-initialized Options instance.
    :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
    :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
    :param Reporting reporting: The global, pre-initialized Reporting instance.
    :param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
    """
    self._root_dir = root_dir
    self._options = options
    self._build_config = build_config
    self._run_tracker = run_tracker
    self._reporting = reporting
    self._exiter = exiter

    self._goals = []
    self._targets = []
    self._requested_goals = self._options.goals
    self._target_specs = self._options.target_specs
    self._help_request = self._options.help_request

    self._global_options = options.for_global_scope()
    self._tag = self._global_options.tag
    self._fail_fast = self._global_options.fail_fast
    # Will be provided through context.address_mapper.build_ignore_patterns.
    self._explain = self._global_options.explain
    self._kill_nailguns = self._global_options.kill_nailguns

    pants_ignore = self._global_options.pants_ignore or []
    self._project_tree = self._get_project_tree(self._global_options.build_file_rev, pants_ignore)
    self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
    build_ignore_patterns = self._global_options.ignore_patterns or []
    self._address_mapper = BuildFileAddressMapper(
      self._build_file_parser,
      self._project_tree,
      build_ignore_patterns,
      exclude_target_regexps=self._global_options.exclude_target_regexp
    )
    self._build_graph = MutableBuildGraph(self._address_mapper)

  def _get_project_tree(self, build_file_rev, pants_ignore):
    """Creates the project tree for build files for use in a given pants run."""
    if build_file_rev:
      return ScmProjectTree(self._root_dir, get_scm(), build_file_rev, pants_ignore)
    else:
      return FileSystemProjectTree(self._root_dir, pants_ignore)

  def _expand_goals(self, goals):
    """Check and populate the requested goals for a given run."""
    for goal in goals:
      try:
        self._address_mapper.resolve_spec(goal)
        logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be "
                       "a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
      except AddressLookupError:
        pass

    if self._help_request:
      help_printer = HelpPrinter(self._options)
      result = help_printer.print_help()
      self._exiter(result)

    self._goals.extend([Goal.by_name(goal) for goal in goals])

  def _expand_specs(self, spec_strs, fail_fast):
    """Populate the BuildGraph and target list from a set of input specs."""
    with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]):
      def filter_for_tag(tag):
        return lambda target: tag in map(str, target.tags)

      tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag))

      # Parse all specs into unique Spec objects.
      spec_parser = CmdLineSpecParser(self._root_dir)
      specs = OrderedSet()
      for spec_str in spec_strs:
        specs.add(spec_parser.parse_spec(spec_str))

      # Then scan them to generate unique Addresses.
      for address in self._build_graph.inject_specs_closure(specs, fail_fast):
        target = self._build_graph.get_target(address)
        if tag_filter(target):
          self._targets.append(target)

  def _maybe_launch_pantsd(self):
    """Launches pantsd if configured to do so."""
    if self._global_options.enable_pantsd:
      # Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on.
      with self._run_tracker.new_workunit(name='pantsd', labels=[WorkUnitLabel.SETUP]):
        PantsDaemonLauncher.global_instance().maybe_launch()

  def _is_quiet(self):
    return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain

  def _setup_context(self):
    self._maybe_launch_pantsd()

    with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]):
      self._expand_goals(self._requested_goals)
      self._expand_specs(self._target_specs, self._fail_fast)

      # Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
      self._run_tracker.run_info.add_scm_info()

      # Update the Reporting settings now that we have options and goal info.
      invalidation_report = self._reporting.update_reporting(self._global_options,
                                                             self._is_quiet(),
                                                             self._run_tracker)

      context = Context(options=self._options,
                        run_tracker=self._run_tracker,
                        target_roots=self._targets,
                        requested_goals=self._requested_goals,
                        build_graph=self._build_graph,
                        build_file_parser=self._build_file_parser,
                        address_mapper=self._address_mapper,
                        invalidation_report=invalidation_report)

    return context, invalidation_report

  def setup(self):
    context, invalidation_report = self._setup_context()
    return GoalRunner(context=context,
                      goals=self._goals,
                      kill_nailguns=self._kill_nailguns,
                      run_tracker=self._run_tracker,
                      invalidation_report=invalidation_report,
                      exiter=self._exiter)
Ejemplo n.º 13
0
class BaseTest(unittest.TestCase):
    """A baseclass useful for tests requiring a temporary buildroot.

  :API: public

  """
    def build_path(self, relpath):
        """Returns the canonical BUILD file path for the given relative build path.

    :API: public
    """
        if os.path.basename(relpath).startswith('BUILD'):
            return relpath
        else:
            return os.path.join(relpath, 'BUILD')

    def create_dir(self, relpath):
        """Creates a directory under the buildroot.

    :API: public

    relpath: The relative path to the directory from the build root.
    """
        path = os.path.join(self.build_root, relpath)
        safe_mkdir(path)
        return path

    def create_workdir_dir(self, relpath):
        """Creates a directory under the work directory.

    :API: public

    relpath: The relative path to the directory from the work directory.
    """
        path = os.path.join(self.pants_workdir, relpath)
        safe_mkdir(path)
        return path

    def create_file(self, relpath, contents='', mode='wb'):
        """Writes to a file under the buildroot.

    :API: public

    relpath:  The relative path to the file from the build root.
    contents: A string containing the contents of the file - '' by default..
    mode:     The mode to write to the file in - over-write by default.
    """
        path = os.path.join(self.build_root, relpath)
        with safe_open(path, mode=mode) as fp:
            fp.write(contents)
        return path

    def create_workdir_file(self, relpath, contents='', mode='wb'):
        """Writes to a file under the work directory.

    :API: public

    relpath:  The relative path to the file from the work directory.
    contents: A string containing the contents of the file - '' by default..
    mode:     The mode to write to the file in - over-write by default.
    """
        path = os.path.join(self.pants_workdir, relpath)
        with safe_open(path, mode=mode) as fp:
            fp.write(contents)
        return path

    def add_to_build_file(self, relpath, target):
        """Adds the given target specification to the BUILD file at relpath.

    :API: public

    relpath: The relative path to the BUILD file from the build root.
    target:  A string containing the target definition as it would appear in a BUILD file.
    """
        self.create_file(self.build_path(relpath), target, mode='a')
        return BuildFile(self.address_mapper._project_tree,
                         relpath=self.build_path(relpath))

    def make_target(self,
                    spec='',
                    target_type=Target,
                    dependencies=None,
                    derived_from=None,
                    synthetic=False,
                    **kwargs):
        """Creates a target and injects it into the test's build graph.

    :API: public

    :param string spec: The target address spec that locates this target.
    :param type target_type: The concrete target subclass to create this new target from.
    :param list dependencies: A list of target instances this new target depends on.
    :param derived_from: The target this new target was derived from.
    :type derived_from: :class:`pants.build_graph.target.Target`
    """
        address = Address.parse(spec)
        target = target_type(name=address.target_name,
                             address=address,
                             build_graph=self.build_graph,
                             **kwargs)
        dependencies = dependencies or []

        self.build_graph.inject_target(
            target,
            dependencies=[dep.address for dep in dependencies],
            derived_from=derived_from,
            synthetic=synthetic)

        # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
        # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
        for traversable_dependency_spec in target.traversable_dependency_specs:
            traversable_dependency_address = Address.parse(
                traversable_dependency_spec, relative_to=address.spec_path)
            traversable_dependency_target = self.build_graph.get_target(
                traversable_dependency_address)
            if not traversable_dependency_target:
                raise ValueError(
                    'Tests must make targets for traversable dependency specs ahead of them '
                    'being traversed, {} tried to traverse {} which does not exist.'
                    .format(target, traversable_dependency_address))
            if traversable_dependency_target not in target.dependencies:
                self.build_graph.inject_dependency(
                    dependent=target.address,
                    dependency=traversable_dependency_address)
                target.mark_transitive_invalidation_hash_dirty()

        return target

    @property
    def alias_groups(self):
        """
    :API: public
    """
        return BuildFileAliases(targets={'target': Target})

    @property
    def build_ignore_patterns(self):
        """
    :API: public
    """
        return None

    def setUp(self):
        """
    :API: public
    """
        super(BaseTest, self).setUp()
        Goal.clear()
        Subsystem.reset()

        self.real_build_root = BuildRoot().path

        self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
        self.subprocess_dir = os.path.join(self.build_root, '.pids')
        self.addCleanup(safe_rmtree, self.build_root)

        self.pants_workdir = os.path.join(self.build_root, '.pants.d')
        safe_mkdir(self.pants_workdir)

        self.options = defaultdict(dict)  # scope -> key-value mapping.
        self.options[''] = {
            'pants_workdir': self.pants_workdir,
            'pants_supportdir': os.path.join(self.build_root, 'build-support'),
            'pants_distdir': os.path.join(self.build_root, 'dist'),
            'pants_configdir': os.path.join(self.build_root, 'config'),
            'pants_subprocessdir': self.subprocess_dir,
            'cache_key_gen_version': '0-test',
        }
        self.options['cache'] = {
            'read_from': [],
            'write_to': [],
        }

        BuildRoot().path = self.build_root
        self.addCleanup(BuildRoot().reset)

        self._build_configuration = BuildConfiguration()
        self._build_configuration.register_aliases(self.alias_groups)
        self.build_file_parser = BuildFileParser(self._build_configuration,
                                                 self.build_root)
        self.project_tree = FileSystemProjectTree(self.build_root)
        self.address_mapper = BuildFileAddressMapper(
            self.build_file_parser,
            self.project_tree,
            build_ignore_patterns=self.build_ignore_patterns)
        self.build_graph = MutableBuildGraph(
            address_mapper=self.address_mapper)

    def buildroot_files(self, relpath=None):
        """Returns the set of all files under the test build root.

    :API: public

    :param string relpath: If supplied, only collect files from this subtree.
    :returns: All file paths found.
    :rtype: set
    """
        def scan():
            for root, dirs, files in os.walk(
                    os.path.join(self.build_root, relpath or '')):
                for f in files:
                    yield os.path.relpath(os.path.join(root, f),
                                          self.build_root)

        return set(scan())

    def reset_build_graph(self):
        """Start over with a fresh build graph with no targets in it."""
        self.address_mapper = BuildFileAddressMapper(
            self.build_file_parser, FileSystemProjectTree(self.build_root))
        self.build_graph = MutableBuildGraph(
            address_mapper=self.address_mapper)

    def set_options_for_scope(self, scope, **kwargs):
        self.options[scope].update(kwargs)

    def context(self,
                for_task_types=None,
                options=None,
                passthru_args=None,
                target_roots=None,
                console_outstream=None,
                workspace=None,
                for_subsystems=None):
        """
    :API: public
    """
        # Many tests use source root functionality via the SourceRootConfig.global_instance()
        # (typically accessed via Target.target_base), so we always set it up, for convenience.
        optionables = {SourceRootConfig}
        extra_scopes = set()

        for_subsystems = for_subsystems or ()
        for subsystem in for_subsystems:
            if subsystem.options_scope is None:
                raise TaskError(
                    'You must set a scope on your subsystem type before using it in tests.'
                )
            optionables.add(subsystem)

        for_task_types = for_task_types or ()
        for task_type in for_task_types:
            scope = task_type.options_scope
            if scope is None:
                raise TaskError(
                    'You must set a scope on your task type before using it in tests.'
                )
            optionables.add(task_type)
            extra_scopes.update(
                [si.scope for si in task_type.known_scope_infos()])
            optionables.update(
                Subsystem.closure(
                    set([
                        dep.subsystem_cls
                        for dep in task_type.subsystem_dependencies_iter()
                    ]) | self._build_configuration.subsystems()))

        # Now default the option values and override with any caller-specified values.
        # TODO(benjy): Get rid of the options arg, and require tests to call set_options.
        options = options.copy() if options else {}
        for s, opts in self.options.items():
            scoped_opts = options.setdefault(s, {})
            scoped_opts.update(opts)

        options = create_options_for_optionables(optionables,
                                                 extra_scopes=extra_scopes,
                                                 options=options)

        Subsystem.reset(reset_options=True)
        Subsystem.set_options(options)

        context = create_context(options=options,
                                 passthru_args=passthru_args,
                                 target_roots=target_roots,
                                 build_graph=self.build_graph,
                                 build_file_parser=self.build_file_parser,
                                 address_mapper=self.address_mapper,
                                 console_outstream=console_outstream,
                                 workspace=workspace)
        return context

    def tearDown(self):
        """
    :API: public
    """
        super(BaseTest, self).tearDown()
        BuildFile.clear_cache()
        Subsystem.reset()

    def target(self, spec):
        """Resolves the given target address to a Target object.

    :API: public

    address: The BUILD target address to resolve.

    Returns the corresponding Target or else None if the address does not point to a defined Target.
    """
        address = Address.parse(spec)
        self.build_graph.inject_address_closure(address)
        return self.build_graph.get_target(address)

    def targets(self, spec):
        """Resolves a target spec to one or more Target objects.

    :API: public

    spec: Either BUILD target address or else a target glob using the siblings ':' or
          descendants '::' suffixes.

    Returns the set of all Targets found.
    """

        spec = CmdLineSpecParser(self.build_root).parse_spec(spec)
        addresses = list(self.address_mapper.scan_specs([spec]))
        for address in addresses:
            self.build_graph.inject_address_closure(address)
        targets = [
            self.build_graph.get_target(address) for address in addresses
        ]
        return targets

    def create_files(self, path, files):
        """Writes to a file under the buildroot with contents same as file name.

    :API: public

     path:  The relative path to the file from the build root.
     files: List of file names.
    """
        for f in files:
            self.create_file(os.path.join(path, f), contents=f)

    def create_library(self, path, target_type, name, sources=None, **kwargs):
        """Creates a library target of given type at the BUILD file at path with sources

    :API: public

     path: The relative path to the BUILD file from the build root.
     target_type: valid pants target type.
     name: Name of the library target.
     sources: List of source file at the path relative to path.
     **kwargs: Optional attributes that can be set for any library target.
       Currently it includes support for resources, java_sources, provides
       and dependencies.
    """
        if sources:
            self.create_files(path, sources)
        self.add_to_build_file(
            path,
            dedent('''
          %(target_type)s(name='%(name)s',
            %(sources)s
            %(resources)s
            %(java_sources)s
            %(provides)s
            %(dependencies)s
          )
        ''' % dict(
                target_type=target_type,
                name=name,
                sources=('sources=%s,' % repr(sources) if sources else ''),
                resources=('resources=["%s"],' % kwargs.get('resources')
                           if 'resources' in kwargs else ''),
                java_sources=('java_sources=[%s],' % ','.join(
                    map(lambda str_target: '"%s"' % str_target,
                        kwargs.get('java_sources')))
                              if 'java_sources' in kwargs else ''),
                provides=('provides=%s,' % kwargs.get('provides')
                          if 'provides' in kwargs else ''),
                dependencies=('dependencies=%s,' % kwargs.get('dependencies')
                              if 'dependencies' in kwargs else ''),
            )))
        return self.target('%s:%s' % (path, name))

    def create_resources(self, path, name, *sources):
        """
    :API: public
    """
        return self.create_library(path, 'resources', name, sources)

    def assertUnorderedPrefixEqual(self, expected, actual_iter):
        """Consumes len(expected) items from the given iter, and asserts that they match, unordered.

    :API: public
    """
        actual = list(itertools.islice(actual_iter, len(expected)))
        self.assertEqual(sorted(expected), sorted(actual))

    def assertPrefixEqual(self, expected, actual_iter):
        """Consumes len(expected) items from the given iter, and asserts that they match, in order.

    :API: public
    """
        self.assertEqual(expected,
                         list(itertools.islice(actual_iter, len(expected))))
Ejemplo n.º 14
0
 def reset_build_graph(self):
     """Start over with a fresh build graph with no targets in it."""
     self.address_mapper = BuildFileAddressMapper(
         self.build_file_parser, FileSystemProjectTree(self.build_root))
     self.build_graph = MutableBuildGraph(
         address_mapper=self.address_mapper)
Ejemplo n.º 15
0
 def reset_build_graph(self):
   """Start over with a fresh build graph with no targets in it."""
   self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FileSystemProjectTree(self.build_root))
   self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
Ejemplo n.º 16
0
class GoalRunnerFactory(object):
    def __init__(self,
                 root_dir,
                 options,
                 build_config,
                 run_tracker,
                 reporting,
                 exiter=sys.exit):
        """
    :param str root_dir: The root directory of the pants workspace (aka the "build root").
    :param Options options: The global, pre-initialized Options instance.
    :param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
    :param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
    :param Reporting reporting: The global, pre-initialized Reporting instance.
    :param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
    """
        self._root_dir = root_dir
        self._options = options
        self._build_config = build_config
        self._run_tracker = run_tracker
        self._reporting = reporting
        self._exiter = exiter

        self._goals = []
        self._targets = []
        self._requested_goals = self._options.goals
        self._target_specs = self._options.target_specs
        self._help_request = self._options.help_request

        self._global_options = options.for_global_scope()
        self._tag = self._global_options.tag
        self._fail_fast = self._global_options.fail_fast
        # Will be provided through context.address_mapper.build_ignore_patterns.
        self._explain = self._global_options.explain
        self._kill_nailguns = self._global_options.kill_nailguns

        self._project_tree = self._get_project_tree(
            self._global_options.build_file_rev)
        self._build_file_parser = BuildFileParser(self._build_config,
                                                  self._root_dir)
        build_ignore_patterns = self._global_options.ignore_patterns or []
        self._address_mapper = BuildFileAddressMapper(
            self._build_file_parser,
            self._project_tree,
            build_ignore_patterns,
            exclude_target_regexps=self._global_options.exclude_target_regexp)
        self._build_graph = MutableBuildGraph(self._address_mapper)

    def _get_project_tree(self, build_file_rev):
        """Creates the project tree for build files for use in a given pants run."""
        if build_file_rev:
            return ScmProjectTree(self._root_dir, get_scm(), build_file_rev)
        else:
            return FileSystemProjectTree(self._root_dir)

    def _expand_goals(self, goals):
        """Check and populate the requested goals for a given run."""
        for goal in goals:
            try:
                self._address_mapper.resolve_spec(goal)
                logger.warning(
                    "Command-line argument '{0}' is ambiguous and was assumed to be "
                    "a goal. If this is incorrect, disambiguate it with ./{0}."
                    .format(goal))
            except AddressLookupError:
                pass

        if self._help_request:
            help_printer = HelpPrinter(self._options)
            result = help_printer.print_help()
            self._exiter(result)

        self._goals.extend([Goal.by_name(goal) for goal in goals])

    def _expand_specs(self, spec_strs, fail_fast):
        """Populate the BuildGraph and target list from a set of input specs."""
        with self._run_tracker.new_workunit(name='parse',
                                            labels=[WorkUnitLabel.SETUP]):

            def filter_for_tag(tag):
                return lambda target: tag in map(str, target.tags)

            tag_filter = wrap_filters(create_filters(self._tag,
                                                     filter_for_tag))

            # Parse all specs into unique Spec objects.
            spec_parser = CmdLineSpecParser(self._root_dir)
            specs = OrderedSet()
            for spec_str in spec_strs:
                specs.add(spec_parser.parse_spec(spec_str))

            # Then scan them to generate unique Addresses.
            for address in self._build_graph.inject_specs_closure(
                    specs, fail_fast):
                target = self._build_graph.get_target(address)
                if tag_filter(target):
                    self._targets.append(target)

    def _maybe_launch_pantsd(self):
        """Launches pantsd if configured to do so."""
        if self._global_options.enable_pantsd:
            # Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on.
            with self._run_tracker.new_workunit(name='pantsd',
                                                labels=[WorkUnitLabel.SETUP]):
                PantsDaemonLauncher.global_instance().maybe_launch()

    def _is_quiet(self):
        return any(
            goal.has_task_of_type(QuietTaskMixin)
            for goal in self._goals) or self._explain

    def _setup_context(self):
        self._maybe_launch_pantsd()

        with self._run_tracker.new_workunit(name='setup',
                                            labels=[WorkUnitLabel.SETUP]):
            self._expand_goals(self._requested_goals)
            self._expand_specs(self._target_specs, self._fail_fast)

            # Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
            self._run_tracker.run_info.add_scm_info()

            # Update the Reporting settings now that we have options and goal info.
            invalidation_report = self._reporting.update_reporting(
                self._global_options, self._is_quiet(), self._run_tracker)

            context = Context(options=self._options,
                              run_tracker=self._run_tracker,
                              target_roots=self._targets,
                              requested_goals=self._requested_goals,
                              build_graph=self._build_graph,
                              build_file_parser=self._build_file_parser,
                              address_mapper=self._address_mapper,
                              invalidation_report=invalidation_report)

        return context, invalidation_report

    def setup(self):
        context, invalidation_report = self._setup_context()
        return GoalRunner(context=context,
                          goals=self._goals,
                          kill_nailguns=self._kill_nailguns,
                          run_tracker=self._run_tracker,
                          invalidation_report=invalidation_report,
                          exiter=self._exiter)
Ejemplo n.º 17
0
 def reset_build_graph(self):
   """Start over with a fresh build graph with no targets in it."""
   self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree,
                                                build_ignore_patterns=self.build_ignore_patterns)
   self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
Ejemplo n.º 18
0
class BaseTest(unittest.TestCase):
    """A baseclass useful for tests requiring a temporary buildroot.

  :API: public

  """

    def build_path(self, relpath):
        """Returns the canonical BUILD file path for the given relative build path.

    :API: public
    """
        if os.path.basename(relpath).startswith("BUILD"):
            return relpath
        else:
            return os.path.join(relpath, "BUILD")

    def create_dir(self, relpath):
        """Creates a directory under the buildroot.

    :API: public

    relpath: The relative path to the directory from the build root.
    """
        path = os.path.join(self.build_root, relpath)
        safe_mkdir(path)
        return path

    def create_workdir_dir(self, relpath):
        """Creates a directory under the work directory.

    :API: public

    relpath: The relative path to the directory from the work directory.
    """
        path = os.path.join(self.pants_workdir, relpath)
        safe_mkdir(path)
        return path

    def create_file(self, relpath, contents="", mode="wb"):
        """Writes to a file under the buildroot.

    :API: public

    relpath:  The relative path to the file from the build root.
    contents: A string containing the contents of the file - '' by default..
    mode:     The mode to write to the file in - over-write by default.
    """
        path = os.path.join(self.build_root, relpath)
        with safe_open(path, mode=mode) as fp:
            fp.write(contents)
        return path

    def create_workdir_file(self, relpath, contents="", mode="wb"):
        """Writes to a file under the work directory.

    :API: public

    relpath:  The relative path to the file from the work directory.
    contents: A string containing the contents of the file - '' by default..
    mode:     The mode to write to the file in - over-write by default.
    """
        path = os.path.join(self.pants_workdir, relpath)
        with safe_open(path, mode=mode) as fp:
            fp.write(contents)
        return path

    def add_to_build_file(self, relpath, target):
        """Adds the given target specification to the BUILD file at relpath.

    :API: public

    relpath: The relative path to the BUILD file from the build root.
    target:  A string containing the target definition as it would appear in a BUILD file.
    """
        self.create_file(self.build_path(relpath), target, mode="a")
        return BuildFile(self.address_mapper._project_tree, relpath=self.build_path(relpath))

    def make_target(self, spec="", target_type=Target, dependencies=None, derived_from=None, synthetic=False, **kwargs):
        """Creates a target and injects it into the test's build graph.

    :API: public

    :param string spec: The target address spec that locates this target.
    :param type target_type: The concrete target subclass to create this new target from.
    :param list dependencies: A list of target instances this new target depends on.
    :param derived_from: The target this new target was derived from.
    :type derived_from: :class:`pants.build_graph.target.Target`
    """
        address = Address.parse(spec)
        target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs)
        dependencies = dependencies or []

        self.build_graph.inject_target(
            target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from, synthetic=synthetic
        )

        # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
        # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
        for traversable_dependency_spec in target.traversable_dependency_specs:
            traversable_dependency_address = Address.parse(traversable_dependency_spec, relative_to=address.spec_path)
            traversable_dependency_target = self.build_graph.get_target(traversable_dependency_address)
            if not traversable_dependency_target:
                raise ValueError(
                    "Tests must make targets for traversable dependency specs ahead of them "
                    "being traversed, {} tried to traverse {} which does not exist.".format(
                        target, traversable_dependency_address
                    )
                )
            if traversable_dependency_target not in target.dependencies:
                self.build_graph.inject_dependency(dependent=target.address, dependency=traversable_dependency_address)
                target.mark_transitive_invalidation_hash_dirty()

        return target

    @property
    def alias_groups(self):
        """
    :API: public
    """
        return BuildFileAliases(targets={"target": Target})

    @property
    def build_ignore_patterns(self):
        """
    :API: public
    """
        return None

    def setUp(self):
        """
    :API: public
    """
        super(BaseTest, self).setUp()
        # Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup.
        clean_global_runtime_state(reset_runtracker=False, reset_subsystem=True)

        self.real_build_root = BuildRoot().path

        self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT"))
        self.subprocess_dir = os.path.join(self.build_root, ".pids")
        self.addCleanup(safe_rmtree, self.build_root)

        self.pants_workdir = os.path.join(self.build_root, ".pants.d")
        safe_mkdir(self.pants_workdir)

        self.options = defaultdict(dict)  # scope -> key-value mapping.
        self.options[""] = {
            "pants_workdir": self.pants_workdir,
            "pants_supportdir": os.path.join(self.build_root, "build-support"),
            "pants_distdir": os.path.join(self.build_root, "dist"),
            "pants_configdir": os.path.join(self.build_root, "config"),
            "pants_subprocessdir": self.subprocess_dir,
            "cache_key_gen_version": "0-test",
        }
        self.options["cache"] = {"read_from": [], "write_to": []}

        BuildRoot().path = self.build_root
        self.addCleanup(BuildRoot().reset)

        self._build_configuration = BuildConfiguration()
        self._build_configuration.register_aliases(self.alias_groups)
        self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
        self.project_tree = FileSystemProjectTree(self.build_root)
        self.reset_build_graph()

    def buildroot_files(self, relpath=None):
        """Returns the set of all files under the test build root.

    :API: public

    :param string relpath: If supplied, only collect files from this subtree.
    :returns: All file paths found.
    :rtype: set
    """

        def scan():
            for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or "")):
                for f in files:
                    yield os.path.relpath(os.path.join(root, f), self.build_root)

        return set(scan())

    def reset_build_graph(self):
        """Start over with a fresh build graph with no targets in it."""
        self.address_mapper = BuildFileAddressMapper(
            self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns
        )
        self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)

    def set_options_for_scope(self, scope, **kwargs):
        self.options[scope].update(kwargs)

    def context(
        self,
        for_task_types=None,
        options=None,
        passthru_args=None,
        target_roots=None,
        console_outstream=None,
        workspace=None,
        for_subsystems=None,
    ):
        """
    :API: public
    """
        # Many tests use source root functionality via the SourceRootConfig.global_instance()
        # (typically accessed via Target.target_base), so we always set it up, for convenience.
        optionables = {SourceRootConfig}
        extra_scopes = set()

        for_subsystems = for_subsystems or ()
        for subsystem in for_subsystems:
            if subsystem.options_scope is None:
                raise TaskError("You must set a scope on your subsystem type before using it in tests.")
            optionables.add(subsystem)

        for_task_types = for_task_types or ()
        for task_type in for_task_types:
            scope = task_type.options_scope
            if scope is None:
                raise TaskError("You must set a scope on your task type before using it in tests.")
            optionables.add(task_type)
            extra_scopes.update([si.scope for si in task_type.known_scope_infos()])
            optionables.update(
                Subsystem.closure(
                    set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()])
                    | self._build_configuration.subsystems()
                )
            )

        # Now default the option values and override with any caller-specified values.
        # TODO(benjy): Get rid of the options arg, and require tests to call set_options.
        options = options.copy() if options else {}
        for s, opts in self.options.items():
            scoped_opts = options.setdefault(s, {})
            scoped_opts.update(opts)

        options = create_options_for_optionables(optionables, extra_scopes=extra_scopes, options=options)

        Subsystem.reset(reset_options=True)
        Subsystem.set_options(options)

        context = create_context(
            options=options,
            passthru_args=passthru_args,
            target_roots=target_roots,
            build_graph=self.build_graph,
            build_file_parser=self.build_file_parser,
            address_mapper=self.address_mapper,
            console_outstream=console_outstream,
            workspace=workspace,
        )
        return context

    def tearDown(self):
        """
    :API: public
    """
        super(BaseTest, self).tearDown()
        BuildFile.clear_cache()
        Subsystem.reset()

    def target(self, spec):
        """Resolves the given target address to a Target object.

    :API: public

    address: The BUILD target address to resolve.

    Returns the corresponding Target or else None if the address does not point to a defined Target.
    """
        address = Address.parse(spec)
        self.build_graph.inject_address_closure(address)
        return self.build_graph.get_target(address)

    def targets(self, spec):
        """Resolves a target spec to one or more Target objects.

    :API: public

    spec: Either BUILD target address or else a target glob using the siblings ':' or
          descendants '::' suffixes.

    Returns the set of all Targets found.
    """

        spec = CmdLineSpecParser(self.build_root).parse_spec(spec)
        addresses = list(self.address_mapper.scan_specs([spec]))
        for address in addresses:
            self.build_graph.inject_address_closure(address)
        targets = [self.build_graph.get_target(address) for address in addresses]
        return targets

    def create_files(self, path, files):
        """Writes to a file under the buildroot with contents same as file name.

    :API: public

     path:  The relative path to the file from the build root.
     files: List of file names.
    """
        for f in files:
            self.create_file(os.path.join(path, f), contents=f)

    def create_library(self, path, target_type, name, sources=None, **kwargs):
        """Creates a library target of given type at the BUILD file at path with sources

    :API: public

     path: The relative path to the BUILD file from the build root.
     target_type: valid pants target type.
     name: Name of the library target.
     sources: List of source file at the path relative to path.
     **kwargs: Optional attributes that can be set for any library target.
       Currently it includes support for resources, java_sources, provides
       and dependencies.
    """
        if sources:
            self.create_files(path, sources)
        self.add_to_build_file(
            path,
            dedent(
                """
          %(target_type)s(name='%(name)s',
            %(sources)s
            %(resources)s
            %(java_sources)s
            %(provides)s
            %(dependencies)s
          )
        """
                % dict(
                    target_type=target_type,
                    name=name,
                    sources=("sources=%s," % repr(sources) if sources else ""),
                    resources=('resources=["%s"],' % kwargs.get("resources") if "resources" in kwargs else ""),
                    java_sources=(
                        "java_sources=[%s],"
                        % ",".join(map(lambda str_target: '"%s"' % str_target, kwargs.get("java_sources")))
                        if "java_sources" in kwargs
                        else ""
                    ),
                    provides=("provides=%s," % kwargs.get("provides") if "provides" in kwargs else ""),
                    dependencies=("dependencies=%s," % kwargs.get("dependencies") if "dependencies" in kwargs else ""),
                )
            ),
        )
        return self.target("%s:%s" % (path, name))

    def create_resources(self, path, name, *sources):
        """
    :API: public
    """
        return self.create_library(path, "resources", name, sources)

    def assertUnorderedPrefixEqual(self, expected, actual_iter):
        """Consumes len(expected) items from the given iter, and asserts that they match, unordered.

    :API: public
    """
        actual = list(itertools.islice(actual_iter, len(expected)))
        self.assertEqual(sorted(expected), sorted(actual))

    def assertPrefixEqual(self, expected, actual_iter):
        """Consumes len(expected) items from the given iter, and asserts that they match, in order.

    :API: public
    """
        self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected))))
Ejemplo n.º 19
0
class BaseTest(unittest.TestCase):
  """A baseclass useful for tests requiring a temporary buildroot.

  :API: public

  """

  def build_path(self, relpath):
    """Returns the canonical BUILD file path for the given relative build path.

    :API: public
    """
    if os.path.basename(relpath).startswith('BUILD'):
      return relpath
    else:
      return os.path.join(relpath, 'BUILD')

  def create_dir(self, relpath):
    """Creates a directory under the buildroot.

    :API: public

    relpath: The relative path to the directory from the build root.
    """
    path = os.path.join(self.build_root, relpath)
    safe_mkdir(path)
    return path

  def create_workdir_dir(self, relpath):
    """Creates a directory under the work directory.

    :API: public

    relpath: The relative path to the directory from the work directory.
    """
    path = os.path.join(self.pants_workdir, relpath)
    safe_mkdir(path)
    return path

  def create_file(self, relpath, contents='', mode='wb'):
    """Writes to a file under the buildroot.

    :API: public

    relpath:  The relative path to the file from the build root.
    contents: A string containing the contents of the file - '' by default..
    mode:     The mode to write to the file in - over-write by default.
    """
    path = os.path.join(self.build_root, relpath)
    with safe_open(path, mode=mode) as fp:
      fp.write(contents)
    return path

  def create_workdir_file(self, relpath, contents='', mode='wb'):
    """Writes to a file under the work directory.

    :API: public

    relpath:  The relative path to the file from the work directory.
    contents: A string containing the contents of the file - '' by default..
    mode:     The mode to write to the file in - over-write by default.
    """
    path = os.path.join(self.pants_workdir, relpath)
    with safe_open(path, mode=mode) as fp:
      fp.write(contents)
    return path

  def add_to_build_file(self, relpath, target):
    """Adds the given target specification to the BUILD file at relpath.

    :API: public

    relpath: The relative path to the BUILD file from the build root.
    target:  A string containing the target definition as it would appear in a BUILD file.
    """
    self.create_file(self.build_path(relpath), target, mode='a')
    return BuildFile(self.address_mapper._project_tree, relpath=self.build_path(relpath))

  def make_target(self,
                  spec='',
                  target_type=Target,
                  dependencies=None,
                  derived_from=None,
                  synthetic=False,
                  **kwargs):
    """Creates a target and injects it into the test's build graph.

    :API: public

    :param string spec: The target address spec that locates this target.
    :param type target_type: The concrete target subclass to create this new target from.
    :param list dependencies: A list of target instances this new target depends on.
    :param derived_from: The target this new target was derived from.
    :type derived_from: :class:`pants.build_graph.target.Target`
    """
    address = Address.parse(spec)
    target = target_type(name=address.target_name,
                         address=address,
                         build_graph=self.build_graph,
                         **kwargs)
    dependencies = dependencies or []

    self.build_graph.apply_injectables([target])
    self.build_graph.inject_target(target,
                                   dependencies=[dep.address for dep in dependencies],
                                   derived_from=derived_from,
                                   synthetic=synthetic)

    # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
    # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
    traversables = [target.compute_dependency_specs(payload=target.payload)]

    for dependency_spec in itertools.chain(*traversables):
      dependency_address = Address.parse(dependency_spec, relative_to=address.spec_path)
      dependency_target = self.build_graph.get_target(dependency_address)
      if not dependency_target:
        raise ValueError('Tests must make targets for dependency specs ahead of them '
                         'being traversed, {} tried to traverse {} which does not exist.'
                         .format(target, dependency_address))
      if dependency_target not in target.dependencies:
        self.build_graph.inject_dependency(dependent=target.address,
                                           dependency=dependency_address)
        target.mark_transitive_invalidation_hash_dirty()

    return target

  @property
  def alias_groups(self):
    """
    :API: public
    """
    return BuildFileAliases(targets={'target': Target})

  @property
  def build_ignore_patterns(self):
    """
    :API: public
    """
    return None

  def setUp(self):
    """
    :API: public
    """
    super(BaseTest, self).setUp()
    # Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup.
    clean_global_runtime_state(reset_subsystem=True)

    self.real_build_root = BuildRoot().path

    self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
    self.subprocess_dir = os.path.join(self.build_root, '.pids')
    self.addCleanup(safe_rmtree, self.build_root)

    self.pants_workdir = os.path.join(self.build_root, '.pants.d')
    safe_mkdir(self.pants_workdir)

    self.options = defaultdict(dict)  # scope -> key-value mapping.
    self.options[''] = {
      'pants_workdir': self.pants_workdir,
      'pants_supportdir': os.path.join(self.build_root, 'build-support'),
      'pants_distdir': os.path.join(self.build_root, 'dist'),
      'pants_configdir': os.path.join(self.build_root, 'config'),
      'pants_subprocessdir': self.subprocess_dir,
      'cache_key_gen_version': '0-test',
    }
    self.options['cache'] = {
      'read_from': [],
      'write_to': [],
    }

    BuildRoot().path = self.build_root
    self.addCleanup(BuildRoot().reset)

    self._build_configuration = BuildConfiguration()
    self._build_configuration.register_aliases(self.alias_groups)
    self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
    self.project_tree = FileSystemProjectTree(self.build_root)
    self.reset_build_graph()

  def buildroot_files(self, relpath=None):
    """Returns the set of all files under the test build root.

    :API: public

    :param string relpath: If supplied, only collect files from this subtree.
    :returns: All file paths found.
    :rtype: set
    """
    def scan():
      for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or '')):
        for f in files:
          yield os.path.relpath(os.path.join(root, f), self.build_root)
    return set(scan())

  def reset_build_graph(self):
    """Start over with a fresh build graph with no targets in it."""
    self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree,
                                                 build_ignore_patterns=self.build_ignore_patterns)
    self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)

  def set_options_for_scope(self, scope, **kwargs):
    self.options[scope].update(kwargs)

  def context(self, for_task_types=None, for_subsystems=None, options=None,
              target_roots=None, console_outstream=None, workspace=None,
              **kwargs):
    """
    :API: public

    :param dict **kwargs: keyword arguments passed in to `create_options_for_optionables`.
    """
    # Many tests use source root functionality via the SourceRootConfig.global_instance().
    # (typically accessed via Target.target_base), so we always set it up, for convenience.
    optionables = {SourceRootConfig}
    extra_scopes = set()

    for_subsystems = for_subsystems or ()
    for subsystem in for_subsystems:
      if subsystem.options_scope is None:
        raise TaskError('You must set a scope on your subsystem type before using it in tests.')
      optionables.add(subsystem)

    for_task_types = for_task_types or ()
    for task_type in for_task_types:
      scope = task_type.options_scope
      if scope is None:
        raise TaskError('You must set a scope on your task type before using it in tests.')
      optionables.add(task_type)
      # If task is expected to inherit goal-level options, register those directly on the task,
      # by subclassing the goal options registrar and settings its scope to the task scope.
      if issubclass(task_type, GoalOptionsMixin):
        subclass_name = b'test_{}_{}_{}'.format(
          task_type.__name__, task_type.goal_options_registrar_cls.options_scope,
          task_type.options_scope)
        optionables.add(type(subclass_name, (task_type.goal_options_registrar_cls, ),
                             {b'options_scope': task_type.options_scope}))

      extra_scopes.update([si.scope for si in task_type.known_scope_infos()])
      optionables.update(Subsystem.closure(
        set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) |
            self._build_configuration.subsystems()))

    # Now default the option values and override with any caller-specified values.
    # TODO(benjy): Get rid of the options arg, and require tests to call set_options.
    options = options.copy() if options else {}
    for s, opts in self.options.items():
      scoped_opts = options.setdefault(s, {})
      scoped_opts.update(opts)

    fake_options = create_options_for_optionables(
      optionables, extra_scopes=extra_scopes, options=options, **kwargs)

    Subsystem.reset(reset_options=True)
    Subsystem.set_options(fake_options)

    context = create_context_from_options(fake_options,
                                          target_roots=target_roots,
                                          build_graph=self.build_graph,
                                          build_file_parser=self.build_file_parser,
                                          address_mapper=self.address_mapper,
                                          console_outstream=console_outstream,
                                          workspace=workspace)
    return context

  def tearDown(self):
    """
    :API: public
    """
    super(BaseTest, self).tearDown()
    BuildFile.clear_cache()
    Subsystem.reset()

  def target(self, spec):
    """Resolves the given target address to a Target object.

    :API: public

    address: The BUILD target address to resolve.

    Returns the corresponding Target or else None if the address does not point to a defined Target.
    """
    address = Address.parse(spec)
    self.build_graph.inject_address_closure(address)
    return self.build_graph.get_target(address)

  def targets(self, spec):
    """Resolves a target spec to one or more Target objects.

    :API: public

    spec: Either BUILD target address or else a target glob using the siblings ':' or
          descendants '::' suffixes.

    Returns the set of all Targets found.
    """

    spec = CmdLineSpecParser(self.build_root).parse_spec(spec)
    addresses = list(self.address_mapper.scan_specs([spec]))
    for address in addresses:
      self.build_graph.inject_address_closure(address)
    targets = [self.build_graph.get_target(address) for address in addresses]
    return targets

  def create_files(self, path, files):
    """Writes to a file under the buildroot with contents same as file name.

    :API: public

     path:  The relative path to the file from the build root.
     files: List of file names.
    """
    for f in files:
      self.create_file(os.path.join(path, f), contents=f)

  def create_library(self, path, target_type, name, sources=None, **kwargs):
    """Creates a library target of given type at the BUILD file at path with sources

    :API: public

     path: The relative path to the BUILD file from the build root.
     target_type: valid pants target type.
     name: Name of the library target.
     sources: List of source file at the path relative to path.
     **kwargs: Optional attributes that can be set for any library target.
       Currently it includes support for resources, java_sources, provides
       and dependencies.
    """
    if sources:
      self.create_files(path, sources)
    self.add_to_build_file(path, dedent('''
          %(target_type)s(name='%(name)s',
            %(sources)s
            %(java_sources)s
            %(provides)s
            %(dependencies)s
          )
        ''' % dict(target_type=target_type,
                   name=name,
                   sources=('sources=%s,' % repr(sources)
                              if sources else ''),
                   java_sources=('java_sources=[%s],'
                                 % ','.join(map(lambda str_target: '"%s"' % str_target,
                                                kwargs.get('java_sources')))
                                 if 'java_sources' in kwargs else ''),
                   provides=('provides=%s,' % kwargs.get('provides')
                              if 'provides' in kwargs else ''),
                   dependencies=('dependencies=%s,' % kwargs.get('dependencies')
                              if 'dependencies' in kwargs else ''),
                   )))
    return self.target('%s:%s' % (path, name))

  def create_resources(self, path, name, *sources):
    """
    :API: public
    """
    return self.create_library(path, 'resources', name, sources)

  def assertUnorderedPrefixEqual(self, expected, actual_iter):
    """Consumes len(expected) items from the given iter, and asserts that they match, unordered.

    :API: public
    """
    actual = list(itertools.islice(actual_iter, len(expected)))
    self.assertEqual(sorted(expected), sorted(actual))

  def assertPrefixEqual(self, expected, actual_iter):
    """Consumes len(expected) items from the given iter, and asserts that they match, in order.

    :API: public
    """
    self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected))))

  def assertInFile(self, string, file_path):
    """Verifies that a string appears in a file

    :API: public
    """

    with open(file_path) as f:
      content = f.read()
      self.assertIn(string, content, '"{}" is not in the file {}:\n{}'.format(string, f.name, content))

  def get_bootstrap_options(self, cli_options=()):
    """Retrieves bootstrap options.

    :param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`.
    """
    # Can't parse any options without a pants.ini.
    self.create_file('pants.ini')
    return OptionsBootstrapper(args=cli_options).get_bootstrap_options().for_global_scope()

  class LoggingRecorder(object):
    """Simple logging handler to record warnings."""

    def __init__(self):
      self._records = []
      self.level = logging.DEBUG

    def handle(self, record):
      self._records.append(record)

    def _messages_for_level(self, levelname):
      return ['{}: {}'.format(record.name, record.getMessage())
              for record in self._records if record.levelname == levelname]

    def infos(self):
      return self._messages_for_level('INFO')

    def warnings(self):
      return self._messages_for_level('WARNING')

  @contextmanager
  def captured_logging(self, level=None):
    root_logger = logging.getLogger()

    old_level = root_logger.level
    root_logger.setLevel(level or logging.NOTSET)

    handler = self.LoggingRecorder()
    root_logger.addHandler(handler)
    try:
      yield handler
    finally:
      root_logger.setLevel(old_level)
      root_logger.removeHandler(handler)