def setUp(self): super(BaseTestWithParser, self).setUp() build_configuration = BuildConfiguration() build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(build_configuration, self.build_root)
def setUp(self): super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'cache_key_gen_version': '0-test', } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) # We need a pants.ini, even if empty. get_buildroot() uses its presence. self.create_file('pants.ini') self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def load_backend( build_configuration: BuildConfiguration, backend_package: str, is_v1_backend: bool ) -> None: """Installs the given backend package into the build configuration. :param build_configuration: the BuildConfiguration to install the backend plugin into. :param backend_package: the package name containing the backend plugin register module that provides the plugin entrypoints. :param is_v1_backend: Is this a v1 or v2 backend. :raises: :class:``pants.base.exceptions.BuildConfigurationError`` if there is a problem loading the build configuration. """ backend_module = backend_package + ".register" try: module = importlib.import_module(backend_module) except ImportError as ex: traceback.print_exc() raise BackendConfigurationError(f"Failed to load the {backend_module} backend: {ex!r}") def invoke_entrypoint(name): entrypoint = getattr(module, name, lambda: None) try: return entrypoint() except TypeError as e: traceback.print_exc() raise BackendConfigurationError( f"Entrypoint {name} in {backend_module} must be a zero-arg callable: {e!r}" ) # While the Target API is a V2 concept, we expect V1 plugin authors to still write Target # API bindings. So, we end up using this entry point regardless of V1 vs. V2. targets = invoke_entrypoint("targets2") if targets: build_configuration.register_targets(targets) if is_v1_backend: invoke_entrypoint("register_goals") subsystems = invoke_entrypoint("global_subsystems") if subsystems: build_configuration.register_optionables(subsystems) # For now, `build_file_aliases` is still V1-only. TBD what entry-point we use for # `objects` and `context_aware_object_factories`. build_file_aliases = invoke_entrypoint("build_file_aliases") if build_file_aliases: build_configuration.register_aliases(build_file_aliases) else: rules = invoke_entrypoint("rules") if rules: build_configuration.register_rules(rules) build_file_aliases2 = invoke_entrypoint("build_file_aliases2") if build_file_aliases2: build_configuration.register_aliases(build_file_aliases2)
async def find_owners( build_configuration: BuildConfiguration, address_mapper: AddressMapper, changed_request: ChangedRequest, ) -> ChangedAddresses: owners = await Get[Owners](OwnersRequest(sources=changed_request.sources)) # If the ChangedRequest does not require dependees, then we're done. if changed_request.include_dependees == IncludeDependeesOption.NONE: return ChangedAddresses(owners.addresses) # Otherwise: find dependees. all_addresses = await Get[Addresses](AddressSpecs( (DescendantAddresses(""), ))) all_structs = [ s.value for s in await MultiGet(Get[HydratedStruct](Address, a) for a in all_addresses) ] bfa = build_configuration.registered_aliases() graph = _DependentGraph.from_iterable( target_types_from_build_file_aliases(bfa), address_mapper, all_structs) if changed_request.include_dependees == IncludeDependeesOption.DIRECT: return ChangedAddresses( Addresses(graph.dependents_of_addresses(owners.addresses))) return ChangedAddresses( Addresses(graph.transitive_dependents_of_addresses(owners.addresses)))
def setUp(self): """ :API: public """ super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.subprocess_dir = os.path.join(self.build_root, '.pids') self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'pants_subprocessdir': self.subprocess_dir, 'cache_key_gen_version': '0-test', } self.options['cache'] = { 'read_from': [], 'write_to': [], } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.project_tree = FileSystemProjectTree(self.build_root) self.address_mapper = BuildFileAddressMapper( self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns) self.build_graph = MutableBuildGraph( address_mapper=self.address_mapper)
async def find_owners(build_configuration: BuildConfiguration, address_mapper: AddressMapper, owners_request: OwnersRequest) -> BuildFileAddresses: sources_set = OrderedSet(owners_request.sources) dirs_set = OrderedSet(dirname(source) for source in sources_set) # Walk up the buildroot looking for targets that would conceivably claim changed sources. candidate_specs = tuple(AscendantAddresses(directory=d) for d in dirs_set) candidate_targets = await Get(HydratedTargets, Specs(candidate_specs)) # Match the source globs against the expanded candidate targets. def owns_any_source(legacy_target): """Given a `HydratedTarget` instance, check if it owns the given source file.""" target_kwargs = legacy_target.adaptor.kwargs() # Handle `sources`-declaring targets. # NB: Deleted files can only be matched against the 'filespec' (ie, `PathGlobs`) for a target, # so we don't actually call `fileset.matches` here. # TODO: This matching logic should be implemented using the rust `fs` crate for two reasons: # 1) having two implementations isn't great # 2) we're expanding sources via HydratedTarget, but it isn't necessary to do that to match target_sources = target_kwargs.get('sources', None) if target_sources and any_matches_filespec(sources_set, target_sources.filespec): return True return False direct_owners = tuple( ht.adaptor.address for ht in candidate_targets if LegacyAddressMapper.any_is_declaring_file( ht.adaptor.address, sources_set) or owns_any_source(ht)) # If the OwnersRequest does not require dependees, then we're done. if owners_request.include_dependees == 'none': return BuildFileAddresses(direct_owners) else: # Otherwise: find dependees. all_addresses = await Get(BuildFileAddresses, Specs((DescendantAddresses(''), ))) all_hydrated_structs = await MultiGet( Get(HydratedStruct, Address, a.to_address()) for a in all_addresses) all_structs = [hs.value for hs in all_hydrated_structs] bfa = build_configuration.registered_aliases() graph = _DependentGraph.from_iterable( target_types_from_build_file_aliases(bfa), address_mapper, all_structs) if owners_request.include_dependees == 'direct': return BuildFileAddresses( tuple(graph.dependents_of_addresses(direct_owners))) else: assert owners_request.include_dependees == 'transitive' return BuildFileAddresses( tuple(graph.transitive_dependents_of_addresses(direct_owners)))
def __init__( self, *, rules: Optional[Iterable] = None, target_types: Optional[Iterable[Type[Target]]] = None, objects: Optional[Dict[str, Any]] = None, context_aware_object_factories: Optional[Dict[str, Any]] = None, ) -> None: self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT")) safe_mkdir(self.build_root, clean=True) safe_mkdir(self.pants_workdir) BuildRoot().path = self.build_root # TODO: Redesign rule registration for tests to be more ergonomic and to make this less # special-cased. all_rules = ( *(rules or ()), *source_root.rules(), *pants_environment.rules(), QueryRule(WrappedTarget, (Address,)), ) build_config_builder = BuildConfiguration.Builder() build_config_builder.register_aliases( BuildFileAliases( objects=objects, context_aware_object_factories=context_aware_object_factories ) ) build_config_builder.register_rules(all_rules) build_config_builder.register_target_types(target_types or ()) self.build_config = build_config_builder.create() options_bootstrapper = create_options_bootstrapper() global_options = options_bootstrapper.bootstrap_options.for_global_scope() local_store_dir = global_options.local_store_dir local_execution_root_dir = global_options.local_execution_root_dir named_caches_dir = global_options.named_caches_dir graph_session = EngineInitializer.setup_graph_extended( pants_ignore_patterns=[], use_gitignore=False, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, native=Native(), options_bootstrapper=options_bootstrapper, build_root=self.build_root, build_configuration=self.build_config, execution_options=ExecutionOptions.from_bootstrap_options(global_options), ).new_session( build_id="buildid_for_test", session_values=SessionValues( {OptionsBootstrapper: options_bootstrapper, PantsEnvironment: PantsEnvironment()} ), should_report_workunits=True, ) self.scheduler = graph_session.scheduler_session
def load_plugins_and_backends(plugins, working_set, backends): """Load named plugins and source backends :param list<str> plugins: Plugins to load (see `load_plugins`). :param WorkingSet working_set: A pkg_resources.WorkingSet to load plugins from. :param list<str> backends: Source backends to load (see `load_build_configuration_from_source`). """ build_configuration = BuildConfiguration() load_plugins(build_configuration, plugins or [], working_set) load_build_configuration_from_source(build_configuration, additional_backends=backends or []) return build_configuration
def setUp(self): """ :API: public """ super(BaseTest, self).setUp() # Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup. clean_global_runtime_state(reset_runtracker=False, reset_subsystem=True) self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.subprocess_dir = os.path.join(self.build_root, '.pids') self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'pants_subprocessdir': self.subprocess_dir, 'cache_key_gen_version': '0-test', } self.options['cache'] = { 'read_from': [], 'write_to': [], } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.project_tree = FileSystemProjectTree(self.build_root) self.reset_build_graph()
def load_backends_and_plugins(plugins, working_set, backends, build_configuration=None): """Load named plugins and source backends :param list<str> plugins: Plugins to load (see `load_plugins`). Plugins are loaded after backends. :param WorkingSet working_set: A pkg_resources.WorkingSet to load plugins from. :param list<str> backends: Source backends to load (see `load_build_configuration_from_source`). """ build_configuration = build_configuration or BuildConfiguration() load_build_configuration_from_source(build_configuration, backends) load_plugins(build_configuration, plugins or [], working_set) return build_configuration
def _load_plugins(self): # Add any extra paths to python path (e.g., for loading extra source backends). for path in self._bootstrap_options.pythonpath: if path not in sys.path: sys.path.append(path) pkg_resources.fixup_namespace_packages(path) # Load plugins and backends. return load_backends_and_plugins( self._bootstrap_options.plugins, self._bootstrap_options.plugins2, self._working_set, self._bootstrap_options.backend_packages, self._bootstrap_options.backend_packages2, BuildConfiguration())
def load_backend(build_configuration: BuildConfiguration, backend_package: str, is_v1_backend: bool) -> None: """Installs the given backend package into the build configuration. :param build_configuration: the BuildConfiguration to install the backend plugin into. :param backend_package: the package name containing the backend plugin register module that provides the plugin entrypoints. :param is_v1_backend: Is this a v1 or v2 backend. :raises: :class:``pants.base.exceptions.BuildConfigurationError`` if there is a problem loading the build configuration. """ backend_module = backend_package + ".register" try: module = importlib.import_module(backend_module) except ImportError as ex: traceback.print_exc() raise BackendConfigurationError( f"Failed to load the {backend_module} backend: {ex!r}") def invoke_entrypoint(name): entrypoint = getattr(module, name, lambda: None) try: return entrypoint() except TypeError as e: traceback.print_exc() raise BackendConfigurationError( f"Entrypoint {name} in {backend_module} must be a zero-arg callable: {e!r}" ) if is_v1_backend: invoke_entrypoint("register_goals") # TODO: Might v2 plugins need to register global subsystems? Hopefully not. subsystems = invoke_entrypoint("global_subsystems") if subsystems: build_configuration.register_optionables(subsystems) # The v2 target API is still TBD, so we keep build_file_aliases as a v1-only thing. # Having thus no overlap between v1 and v2 backend entrypoints makes things much simpler. # TODO: Revisit, ideally with a v2-only entry point, once the v2 target API is a thing. build_file_aliases = invoke_entrypoint("build_file_aliases") if build_file_aliases: build_configuration.register_aliases(build_file_aliases) else: rules = invoke_entrypoint("rules") if rules: build_configuration.register_rules(rules) build_file_aliases2 = invoke_entrypoint("build_file_aliases2") if build_file_aliases2: build_configuration.register_aliases(build_file_aliases2)
def load_backend(build_configuration: BuildConfiguration, backend_package: str, is_v1_backend: bool) -> None: """Installs the given backend package into the build configuration. :param build_configuration: the BuildConfiguration to install the backend plugin into. :param backend_package: the package name containing the backend plugin register module that provides the plugin entrypoints. :param is_v1_backend: Is this a v1 or v2 backend. :raises: :class:``pants.base.exceptions.BuildConfigurationError`` if there is a problem loading the build configuration. """ backend_module = backend_package + ".register" try: module = importlib.import_module(backend_module) except ImportError as ex: traceback.print_exc() raise BackendConfigurationError( f"Failed to load the {backend_module} backend: {ex!r}") def invoke_entrypoint(name): entrypoint = getattr(module, name, lambda: None) try: return entrypoint() except TypeError as e: traceback.print_exc() raise BackendConfigurationError( f"Entrypoint {name} in {backend_module} must be a zero-arg callable: {e!r}" ) # See the comment in `load_plugins` for why we load both `target_types` and # `build_file_aliases` in both V1 and V2. target_types = invoke_entrypoint("target_types") if target_types: build_configuration.register_target_types(target_types) build_file_aliases = invoke_entrypoint("build_file_aliases") if build_file_aliases: build_configuration.register_aliases(build_file_aliases) if is_v1_backend: invoke_entrypoint("register_goals") subsystems = invoke_entrypoint("global_subsystems") if subsystems: build_configuration.register_optionables(subsystems) else: rules = invoke_entrypoint("rules") if rules: build_configuration.register_rules(rules)
def load_backends_and_plugins( plugins: List[str], working_set: WorkingSet, backends: List[str], bc_builder: Optional[BuildConfiguration.Builder] = None, ) -> BuildConfiguration: """Load named plugins and source backends. :param plugins: v2 plugins to load. :param working_set: A pkg_resources.WorkingSet to load plugins from. :param backends: v2 backends to load. :param bc_builder: The BuildConfiguration (for adding aliases). """ bc_builder = bc_builder or BuildConfiguration.Builder() load_build_configuration_from_source(bc_builder, backends) load_plugins(bc_builder, plugins, working_set) return bc_builder.create()
def create_bootstrap_scheduler( options_bootstrapper: OptionsBootstrapper, executor: PyExecutor | None = None) -> BootstrapScheduler: bc_builder = BuildConfiguration.Builder() # To load plugins, we only need access to the Python/PEX rules. load_build_configuration_from_source(bc_builder, ["pants.backend.python"]) # And to plugin-loading-specific rules. bc_builder.register_rules("_dummy_for_bootstrapping_", plugin_resolver_rules()) # We allow unrecognized options to defer any option error handling until post-bootstrap. bc_builder.allow_unknown_options() return BootstrapScheduler( EngineInitializer.setup_graph( options_bootstrapper.bootstrap_options.for_global_scope(), bc_builder.create(), DynamicRemoteOptions.disabled(), executor, ).scheduler)
def create_bootstrap_scheduler( options_bootstrapper: OptionsBootstrapper, executor: Optional[PyExecutor] = None, ) -> BootstrapScheduler: bc_builder = BuildConfiguration.Builder() # To load plugins, we only need access to the Python/PEX rules. load_build_configuration_from_source(bc_builder, ["pants.backend.python"]) # And to plugin-loading-specific rules. bc_builder.register_rules(plugin_resolver_rules()) # We allow unrecognized options to defer any option error handling until post-bootstrap. bc_builder.allow_unknown_options() return BootstrapScheduler( EngineInitializer.setup_graph( options_bootstrapper, bc_builder.create(), executor=executor, # TODO: We use the default execution options to avoid invoking remote execution auth # plugins. They should be loaded via rules using the bootstrap Scheduler in the future. execution_options=DEFAULT_EXECUTION_OPTIONS, ).scheduler)
def setUp(self): """ :API: public """ super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.subprocess_dir = os.path.join(self.build_root, '.pids') self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'pants_subprocessdir': self.subprocess_dir, 'cache_key_gen_version': '0-test', } self.options['cache'] = { 'read_from': [], 'write_to': [], } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.project_tree = FileSystemProjectTree(self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns) self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
def setUp(self): """ :API: public """ super(BaseTest, self).setUp() # Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup. clean_global_runtime_state(reset_subsystem=True) self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.subprocess_dir = os.path.join(self.build_root, '.pids') self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'pants_subprocessdir': self.subprocess_dir, 'cache_key_gen_version': '0-test', } self.options['cache'] = { 'read_from': [], 'write_to': [], } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.project_tree = FileSystemProjectTree(self.build_root) self.reset_build_graph()
def setUp(self): """ :API: public """ super(BaseTest, self).setUp() # Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup. clean_global_runtime_state(reset_runtracker=False, reset_subsystem=True) self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT")) self.subprocess_dir = os.path.join(self.build_root, ".pids") self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, ".pants.d") safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[""] = { "pants_workdir": self.pants_workdir, "pants_supportdir": os.path.join(self.build_root, "build-support"), "pants_distdir": os.path.join(self.build_root, "dist"), "pants_configdir": os.path.join(self.build_root, "config"), "pants_subprocessdir": self.subprocess_dir, "cache_key_gen_version": "0-test", } self.options["cache"] = {"read_from": [], "write_to": []} BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.project_tree = FileSystemProjectTree(self.build_root) self.reset_build_graph()
def load_backend(build_configuration: BuildConfiguration, backend_package: str) -> None: """Installs the given backend package into the build configuration. :param build_configuration: the BuildConfiguration to install the backend plugin into. :param backend_package: the package name containing the backend plugin register module that provides the plugin entrypoints. :raises: :class:``pants.base.exceptions.BuildConfigurationError`` if there is a problem loading the build configuration.""" backend_module = backend_package + '.register' try: module = importlib.import_module(backend_module) except ImportError as e: traceback.print_exc() raise BackendConfigurationError(f'Failed to load the {backend_module} backend: {e!r}') def invoke_entrypoint(name): entrypoint = getattr(module, name, lambda: None) try: return entrypoint() except TypeError as e: traceback.print_exc() raise BackendConfigurationError( f'Entrypoint {name} in {backend_module} must be a zero-arg callable: {e!r}' ) build_file_aliases = invoke_entrypoint('build_file_aliases') if build_file_aliases: build_configuration.register_aliases(build_file_aliases) subsystems = invoke_entrypoint('global_subsystems') if subsystems: build_configuration.register_optionables(subsystems) rules = invoke_entrypoint('rules') if rules: build_configuration.register_rules(rules) invoke_entrypoint('register_goals')
def build_config(cls): build_config = BuildConfiguration() build_config.register_aliases(cls.alias_groups()) build_config.register_rules(cls.rules()) return build_config
def setUp(self) -> None: self.bc_builder = BuildConfiguration.Builder()
def build_config(cls): build_config = BuildConfiguration.Builder() build_config.register_aliases(cls.alias_groups()) build_config.register_rules(cls.rules()) build_config.register_target_types(cls.target_types()) return build_config.create()
def setUp(self): self.build_configuration = BuildConfiguration()
class BuildConfigurationTest(unittest.TestCase): def setUp(self): self.build_configuration = BuildConfiguration() def _register_aliases(self, **kwargs): self.build_configuration.register_aliases(BuildFileAliases(**kwargs)) def test_register_bad(self): with self.assertRaises(TypeError): self.build_configuration.register_aliases(42) def test_register_target_alias(self): class Fred(Target): pass self._register_aliases(targets={'fred': Fred}) aliases = self.build_configuration.registered_aliases() self.assertEqual({}, aliases.target_macro_factories) self.assertEqual({}, aliases.objects) self.assertEqual({}, aliases.context_aware_object_factories) self.assertEqual(dict(fred=Fred), aliases.target_types) with self._create_mock_build_file('fred') as build_file: parse_state = self.build_configuration.initialize_parse_state( build_file) self.assertEqual(0, len(parse_state.objects)) self.assertEqual(1, len(parse_state.parse_globals)) target_call_proxy = parse_state.parse_globals['fred'] target_call_proxy(name='jake') self.assertEqual(1, len(parse_state.objects)) target_proxy = parse_state.objects[0] self.assertEqual('jake', target_proxy.addressed_name) self.assertEqual(Fred, target_proxy.addressed_type) def test_register_target_macro_facory(self): class Fred(Target): pass class FredMacro(TargetMacro): def __init__(self, parse_context): self._parse_context = parse_context def expand(self, *args, **kwargs): return self._parse_context.create_object( Fred, name='frog', dependencies=[kwargs['name']]) class FredFactory(TargetMacro.Factory): @property def target_types(self): return {Fred} def macro(self, parse_context): return FredMacro(parse_context) factory = FredFactory() self._register_aliases(targets={'fred': factory}) aliases = self.build_configuration.registered_aliases() self.assertEqual({}, aliases.target_types) self.assertEqual({}, aliases.objects) self.assertEqual({}, aliases.context_aware_object_factories) self.assertEqual(dict(fred=factory), aliases.target_macro_factories) with self._create_mock_build_file('fred') as build_file: parse_state = self.build_configuration.initialize_parse_state( build_file) self.assertEqual(0, len(parse_state.objects)) self.assertEqual(1, len(parse_state.parse_globals)) target_call_proxy = parse_state.parse_globals['fred'] target_call_proxy(name='jake') self.assertEqual(1, len(parse_state.objects)) target_proxy = parse_state.objects[0] self.assertEqual('frog', target_proxy.addressed_name) self.assertEqual(Fred, target_proxy.addressed_type) self.assertEqual(['jake'], target_proxy.dependency_specs) def test_register_exposed_object(self): self._register_aliases(objects={'jane': 42}) aliases = self.build_configuration.registered_aliases() self.assertEqual({}, aliases.target_types) self.assertEqual({}, aliases.target_macro_factories) self.assertEqual({}, aliases.context_aware_object_factories) self.assertEqual(dict(jane=42), aliases.objects) with self._create_mock_build_file('jane') as build_file: parse_state = self.build_configuration.initialize_parse_state( build_file) self.assertEqual(0, len(parse_state.objects)) self.assertEqual(1, len(parse_state.parse_globals)) self.assertEqual(42, parse_state.parse_globals['jane']) def test_register_exposed_context_aware_function(self): self.do_test_exposed_context_aware_function( lambda context: lambda: context.rel_path) self.do_test_exposed_context_aware_function( lambda context=None: lambda: context.rel_path) def test_register_union_rules(self): # Two calls to register_rules should merge relevant unions. @union class Base: pass class A: pass class B: pass self.build_configuration.register_rules([UnionRule(Base, A)]) self.build_configuration.register_rules([UnionRule(Base, B)]) self.assertEqual(set(self.build_configuration.union_rules()[Base]), {A, B}) def george_method(self, parse_context): return lambda: parse_context.rel_path def test_register_exposed_context_aware_method(self): self.do_test_exposed_context_aware_function(self.george_method) @classmethod def george_classmethod(cls, parse_context): return lambda: parse_context.rel_path def test_register_exposed_context_aware_classmethod(self): self.do_test_exposed_context_aware_function(self.george_classmethod) @staticmethod def george_staticmethod(parse_context): return lambda: parse_context.rel_path def test_register_exposed_context_aware_staticmethod(self): self.do_test_exposed_context_aware_function(self.george_staticmethod) def do_test_exposed_context_aware_function(self, func, *args, **kwargs): with self.do_test_exposed_context_aware_object( func) as context_aware_object: self.assertEqual('george', context_aware_object(*args, **kwargs)) def test_register_exposed_context_aware_class(self): class George: def __init__(self, parse_context): self._parse_context = parse_context def honorific(self): return len(self._parse_context.rel_path) with self.do_test_exposed_context_aware_object( George) as context_aware_object: self.assertEqual(6, context_aware_object.honorific()) @contextmanager def do_test_exposed_context_aware_object(self, context_aware_object_factory): self._register_aliases(context_aware_object_factories={ 'george': context_aware_object_factory }) aliases = self.build_configuration.registered_aliases() self.assertEqual({}, aliases.target_types) self.assertEqual({}, aliases.target_macro_factories) self.assertEqual({}, aliases.objects) self.assertEqual(dict(george=context_aware_object_factory), aliases.context_aware_object_factories) with temporary_dir() as root: build_file_path = os.path.join(root, 'george', 'BUILD') touch(build_file_path) build_file = BuildFile(FileSystemProjectTree(root), 'george/BUILD') parse_state = self.build_configuration.initialize_parse_state( build_file) self.assertEqual(0, len(parse_state.objects)) self.assertEqual(1, len(parse_state.parse_globals)) yield parse_state.parse_globals['george'] @contextmanager def _create_mock_build_file(self, dirname): with temporary_dir() as root: os.mkdir(os.path.join(root, dirname)) touch(os.path.join(root, dirname, 'BUILD')) yield BuildFile(FileSystemProjectTree(root), os.path.join(dirname, 'BUILD'))
def setup_legacy_graph_extended( pants_ignore_patterns: List[str], use_gitignore: bool, local_store_dir: str, local_execution_root_dir: str, named_caches_dir: str, build_file_prelude_globs: Tuple[str, ...], options_bootstrapper: OptionsBootstrapper, build_configuration: BuildConfiguration, execution_options: ExecutionOptions, build_root: Optional[str] = None, native: Optional[Native] = None, glob_match_error_behavior: GlobMatchErrorBehavior = GlobMatchErrorBehavior.warn, build_ignore_patterns=None, exclude_target_regexps=None, subproject_roots=None, include_trace_on_error: bool = True, ) -> LegacyGraphScheduler: """Construct and return the components necessary for LegacyBuildGraph construction. :param local_store_dir: The directory to use for storing the engine's LMDB store in. :param local_execution_root_dir: The directory to use for local execution sandboxes. :param named_caches_dir: The base directory for named cache storage. :param build_file_prelude_globs: Globs to match files to be prepended to all BUILD files. :param build_root: A path to be used as the build root. If None, then default is used. :param native: An instance of the native-engine subsystem. :param options_bootstrapper: A `OptionsBootstrapper` object containing bootstrap options. :param build_configuration: The `BuildConfiguration` object to get build file aliases from. :param glob_match_error_behavior: How to behave if a glob specified for a target's sources or bundles does not expand to anything. :param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD files, usually taken from the '--build-ignore' global option. :param list exclude_target_regexps: A list of regular expressions for excluding targets. :param list subproject_roots: Paths that correspond with embedded build roots under the current build root. :param include_trace_on_error: If True, when an error occurs, the error message will include the graph trace. :param execution_options: Option values for (remote) process execution. """ build_root = build_root or get_buildroot() build_configuration = build_configuration or BuildConfigInitializer.get( options_bootstrapper) bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope( ) build_file_aliases = build_configuration.registered_aliases() rules = build_configuration.rules() registered_target_types = RegisteredTargetTypes.create( build_configuration.target_types()) symbol_table = _legacy_symbol_table(build_file_aliases, registered_target_types) execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS # Register "literal" subjects required for these rules. parser = LegacyPythonCallbacksParser(symbol_table, build_file_aliases) address_mapper = AddressMapper( parser=parser, prelude_glob_patterns=build_file_prelude_globs, build_ignore_patterns=build_ignore_patterns, exclude_target_regexps=exclude_target_regexps, subproject_roots=subproject_roots, ) @rule def glob_match_error_behavior_singleton() -> GlobMatchErrorBehavior: return glob_match_error_behavior @rule def build_configuration_singleton() -> BuildConfiguration: return build_configuration @rule def symbol_table_singleton() -> SymbolTable: return symbol_table @rule def registered_target_types_singleton() -> RegisteredTargetTypes: return registered_target_types @rule def union_membership_singleton() -> UnionMembership: return UnionMembership(build_configuration.union_rules()) @rule def build_root_singleton() -> BuildRoot: return cast(BuildRoot, BuildRoot.instance) # Create a Scheduler containing graph and filesystem rules, with no installed goals. The # LegacyBuildGraph will explicitly request the products it needs. rules = ( RootRule(Console), glob_match_error_behavior_singleton, build_configuration_singleton, symbol_table_singleton, registered_target_types_singleton, union_membership_singleton, build_root_singleton, *interactive_runner.rules(), *graph.rules(), *options_parsing.rules(), *process.rules(), *target.rules(), *create_legacy_graph_tasks(), *create_fs_rules(), *create_platform_rules(), *create_graph_rules(address_mapper), *structs_rules(), *changed_rules(), *binary_tool_rules(), *binary_util_rules(), *rules, ) goal_map = EngineInitializer._make_goal_map_from_rules(rules) union_rules = build_configuration.union_rules() scheduler = Scheduler( native=native, ignore_patterns=pants_ignore_patterns, use_gitignore=use_gitignore, build_root=build_root, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, rules=rules, union_rules=union_rules, execution_options=execution_options, include_trace_on_error=include_trace_on_error, visualize_to_dir=bootstrap_options.native_engine_visualize_to, ) return LegacyGraphScheduler(scheduler, build_file_aliases, goal_map)
def setUp(self): self.bc_builder = BuildConfiguration.Builder() self.working_set = WorkingSet() for entry in working_set.entries: self.working_set.add_entry(entry)
def build_config(cls): build_config = BuildConfiguration() build_config.register_aliases(cls.alias_groups()) return build_config
class LoaderTest(unittest.TestCase): def setUp(self): self.build_configuration = BuildConfiguration() self.working_set = WorkingSet() for entry in working_set.entries: self.working_set.add_entry(entry) def tearDown(self): Goal.clear() @contextmanager def create_register(self, build_file_aliases=None, register_goals=None, global_subsystems=None, module_name='register'): package_name = b'__test_package_{0}'.format(uuid.uuid4().hex) self.assertFalse(package_name in sys.modules) package_module = types.ModuleType(package_name) sys.modules[package_name] = package_module try: register_module_fqn = b'{0}.{1}'.format(package_name, module_name) register_module = types.ModuleType(register_module_fqn) setattr(package_module, module_name, register_module) sys.modules[register_module_fqn] = register_module def register_entrypoint(function_name, function): if function: setattr(register_module, function_name, function) register_entrypoint('build_file_aliases', build_file_aliases) register_entrypoint('global_subsystems', global_subsystems) register_entrypoint('register_goals', register_goals) yield package_name finally: del sys.modules[package_name] def assert_empty_aliases(self): registered_aliases = self.build_configuration.registered_aliases() self.assertEqual(0, len(registered_aliases.target_types)) self.assertEqual(0, len(registered_aliases.target_macro_factories)) self.assertEqual(0, len(registered_aliases.objects)) self.assertEqual(0, len(registered_aliases.context_aware_object_factories)) self.assertEqual(self.build_configuration.subsystems(), set()) def test_load_valid_empty(self): with self.create_register() as backend_package: load_backend(self.build_configuration, backend_package) self.assert_empty_aliases() def test_load_valid_partial_aliases(self): aliases = BuildFileAliases(targets={'bob': DummyTarget}, objects={'obj1': DummyObject1, 'obj2': DummyObject2}) with self.create_register(build_file_aliases=lambda: aliases) as backend_package: load_backend(self.build_configuration, backend_package) registered_aliases = self.build_configuration.registered_aliases() self.assertEqual(DummyTarget, registered_aliases.target_types['bob']) self.assertEqual(DummyObject1, registered_aliases.objects['obj1']) self.assertEqual(DummyObject2, registered_aliases.objects['obj2']) self.assertEqual(self.build_configuration.subsystems(), {DummySubsystem1, DummySubsystem2}) def test_load_valid_partial_goals(self): def register_goals(): Goal.by_name('jack').install(TaskRegistrar('jill', DummyTask)) with self.create_register(register_goals=register_goals) as backend_package: Goal.clear() self.assertEqual(0, len(Goal.all())) load_backend(self.build_configuration, backend_package) self.assert_empty_aliases() self.assertEqual(1, len(Goal.all())) task_names = Goal.by_name('jack').ordered_task_names() self.assertEqual(1, len(task_names)) task_name = task_names[0] self.assertEqual('jill', task_name) def test_load_invalid_entrypoint(self): def build_file_aliases(bad_arg): return BuildFileAliases() with self.create_register(build_file_aliases=build_file_aliases) as backend_package: with self.assertRaises(BuildConfigurationError): load_backend(self.build_configuration, backend_package) def test_load_invalid_module(self): with self.create_register(module_name='register2') as backend_package: with self.assertRaises(BuildConfigurationError): load_backend(self.build_configuration, backend_package) def test_load_missing_plugin(self): with self.assertRaises(PluginNotFound): self.load_plugins(['Foobar']) def get_mock_plugin(self, name, version, reg=None, alias=None, after=None): """Make a fake Distribution (optionally with entry points) Note the entry points do not actually point to code in the returned distribution -- the distribution does not even have a location and does not contain any code, just metadata. A module is synthesized on the fly and installed into sys.modules under a random name. If optional entry point callables are provided, those are added as methods to the module and their name (foo/bar/baz in fake module) is added as the requested entry point to the mocked metadata added to the returned dist. :param string name: project_name for distribution (see pkg_resources) :param string version: version for distribution (see pkg_resources) :param callable reg: Optional callable for goal registration entry point :param callable alias: Optional callable for build_file_aliases entry point :param callable after: Optional callable for load_after list entry point """ plugin_pkg = b'demoplugin{0}'.format(uuid.uuid4().hex) pkg = types.ModuleType(plugin_pkg) sys.modules[plugin_pkg] = pkg module_name = b'{0}.{1}'.format(plugin_pkg, 'demo') plugin = types.ModuleType(module_name) setattr(pkg, 'demo', plugin) sys.modules[module_name] = plugin metadata = {} entry_lines = [] if reg is not None: setattr(plugin, 'foo', reg) entry_lines.append('register_goals = {}:foo\n'.format(module_name)) if alias is not None: setattr(plugin, 'bar', alias) entry_lines.append('build_file_aliases = {}:bar\n'.format(module_name)) if after is not None: setattr(plugin, 'baz', after) entry_lines.append('load_after = {}:baz\n'.format(module_name)) if entry_lines: entry_data = '[pantsbuild.plugin]\n{}\n'.format('\n'.join(entry_lines)) metadata = {'entry_points.txt': entry_data} return Distribution(project_name=name, version=version, metadata=MockMetadata(metadata)) def load_plugins(self, plugins): load_plugins(self.build_configuration, plugins, self.working_set) def test_plugin_load_and_order(self): d1 = self.get_mock_plugin('demo1', '0.0.1', after=lambda: ['demo2']) d2 = self.get_mock_plugin('demo2', '0.0.3') self.working_set.add(d1) # Attempting to load 'demo1' then 'demo2' should fail as 'demo1' requires 'after'=['demo2']. with self.assertRaises(PluginLoadOrderError): self.load_plugins(['demo1', 'demo2']) # Attempting to load 'demo2' first should fail as it is not (yet) installed. with self.assertRaises(PluginNotFound): self.load_plugins(['demo2', 'demo1']) # Installing demo2 and then loading in correct order should work though. self.working_set.add(d2) self.load_plugins(['demo2>=0.0.2', 'demo1']) # But asking for a bad (not installed) version fails. with self.assertRaises(VersionConflict): self.load_plugins(['demo2>=0.0.5']) def test_plugin_installs_goal(self): def reg_goal(): Goal.by_name('plugindemo').install(TaskRegistrar('foo', DummyTask)) self.working_set.add(self.get_mock_plugin('regdemo', '0.0.1', reg=reg_goal)) # Start without the custom goal. self.assertEqual(0, len(Goal.by_name('plugindemo').ordered_task_names())) # Load plugin which registers custom goal. self.load_plugins(['regdemo']) # Now the custom goal exists. self.assertEqual(1, len(Goal.by_name('plugindemo').ordered_task_names())) self.assertEqual('foo', Goal.by_name('plugindemo').ordered_task_names()[0]) def test_plugin_installs_alias(self): def reg_alias(): return BuildFileAliases(targets={'pluginalias': DummyTarget}, objects={'FROMPLUGIN1': DummyObject1, 'FROMPLUGIN2': DummyObject2}) self.working_set.add(self.get_mock_plugin('aliasdemo', '0.0.1', alias=reg_alias)) # Start with no aliases. self.assert_empty_aliases() # Now load the plugin which defines aliases. self.load_plugins(['aliasdemo']) # Aliases now exist. registered_aliases = self.build_configuration.registered_aliases() self.assertEqual(DummyTarget, registered_aliases.target_types['pluginalias']) self.assertEqual(DummyObject1, registered_aliases.objects['FROMPLUGIN1']) self.assertEqual(DummyObject2, registered_aliases.objects['FROMPLUGIN2']) self.assertEqual(self.build_configuration.subsystems(), {DummySubsystem1, DummySubsystem2}) def test_subsystems(self): def global_subsystems(): return {DummySubsystem1, DummySubsystem2} with self.create_register(global_subsystems=global_subsystems) as backend_package: load_backend(self.build_configuration, backend_package) self.assertEqual(self.build_configuration.subsystems(), {DummySubsystem1, DummySubsystem2})
def __init__( self, *, rules: Iterable | None = None, target_types: Iterable[type[Target]] | None = None, objects: dict[str, Any] | None = None, context_aware_object_factories: dict[str, Any] | None = None, isolated_local_store: bool = False, preserve_tmpdirs: bool = False, ca_certs_path: str | None = None, bootstrap_args: Iterable[str] = (), ) -> None: bootstrap_args = [*bootstrap_args] root_dir: Path | None = None if preserve_tmpdirs: root_dir = Path(mkdtemp(prefix="RuleRunner.")) print( f"Preserving rule runner temporary directories at {root_dir}.", file=sys.stderr) bootstrap_args.extend([ "--no-process-execution-local-cleanup", f"--local-execution-root-dir={root_dir}", ]) build_root = (root_dir / "BUILD_ROOT").resolve() build_root.mkdir() self.build_root = str(build_root) else: self.build_root = os.path.realpath( safe_mkdtemp(prefix="_BUILD_ROOT")) safe_mkdir(self.pants_workdir) BuildRoot().path = self.build_root # TODO: Redesign rule registration for tests to be more ergonomic and to make this less # special-cased. all_rules = ( *(rules or ()), *source_root.rules(), QueryRule(WrappedTarget, [Address]), QueryRule(UnionMembership, []), ) build_config_builder = BuildConfiguration.Builder() build_config_builder.register_aliases( BuildFileAliases( objects=objects, context_aware_object_factories=context_aware_object_factories)) build_config_builder.register_rules("_dummy_for_test_", all_rules) build_config_builder.register_target_types("_dummy_for_test_", target_types or ()) self.build_config = build_config_builder.create() self.environment = CompleteEnvironment({}) self.options_bootstrapper = create_options_bootstrapper( args=bootstrap_args) options = self.options_bootstrapper.full_options(self.build_config) global_options = self.options_bootstrapper.bootstrap_options.for_global_scope( ) dynamic_remote_options, _ = DynamicRemoteOptions.from_options( options, self.environment) local_store_options = LocalStoreOptions.from_options(global_options) if isolated_local_store: if root_dir: lmdb_store_dir = root_dir / "lmdb_store" lmdb_store_dir.mkdir() store_dir = str(lmdb_store_dir) else: store_dir = safe_mkdtemp(prefix="lmdb_store.") local_store_options = dataclasses.replace(local_store_options, store_dir=store_dir) local_execution_root_dir = global_options.local_execution_root_dir named_caches_dir = global_options.named_caches_dir graph_session = EngineInitializer.setup_graph_extended( pants_ignore_patterns=GlobalOptions.compute_pants_ignore( self.build_root, global_options), use_gitignore=False, local_store_options=local_store_options, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, build_root=self.build_root, build_configuration=self.build_config, executor=_EXECUTOR, execution_options=ExecutionOptions.from_options( global_options, dynamic_remote_options), ca_certs_path=ca_certs_path, engine_visualize_to=None, ).new_session( build_id="buildid_for_test", session_values=SessionValues({ OptionsBootstrapper: self.options_bootstrapper, CompleteEnvironment: self.environment, }), ) self.scheduler = graph_session.scheduler_session
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot. :API: public """ def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path. :API: public """ if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. :API: public relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_workdir_dir(self, relpath): """Creates a directory under the work directory. :API: public relpath: The relative path to the directory from the work directory. """ path = os.path.join(self.pants_workdir, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. :API: public relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def create_workdir_file(self, relpath, contents='', mode='wb'): """Writes to a file under the work directory. :API: public relpath: The relative path to the file from the work directory. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.pants_workdir, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. :API: public relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') return BuildFile(self.address_mapper._project_tree, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, derived_from=None, synthetic=False, **kwargs): """Creates a target and injects it into the test's build graph. :API: public :param string spec: The target address spec that locates this target. :param type target_type: The concrete target subclass to create this new target from. :param list dependencies: A list of target instances this new target depends on. :param derived_from: The target this new target was derived from. :type derived_from: :class:`pants.build_graph.target.Target` """ address = Address.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] self.build_graph.apply_injectables([target]) self.build_graph.inject_target(target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from, synthetic=synthetic) # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph. # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it. traversables = [target.compute_dependency_specs(payload=target.payload)] for dependency_spec in itertools.chain(*traversables): dependency_address = Address.parse(dependency_spec, relative_to=address.spec_path) dependency_target = self.build_graph.get_target(dependency_address) if not dependency_target: raise ValueError('Tests must make targets for dependency specs ahead of them ' 'being traversed, {} tried to traverse {} which does not exist.' .format(target, dependency_address)) if dependency_target not in target.dependencies: self.build_graph.inject_dependency(dependent=target.address, dependency=dependency_address) target.mark_transitive_invalidation_hash_dirty() return target @property def alias_groups(self): """ :API: public """ return BuildFileAliases(targets={'target': Target}) @property def build_ignore_patterns(self): """ :API: public """ return None def setUp(self): """ :API: public """ super(BaseTest, self).setUp() # Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup. clean_global_runtime_state(reset_subsystem=True) self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.subprocess_dir = os.path.join(self.build_root, '.pids') self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'pants_subprocessdir': self.subprocess_dir, 'cache_key_gen_version': '0-test', } self.options['cache'] = { 'read_from': [], 'write_to': [], } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.project_tree = FileSystemProjectTree(self.build_root) self.reset_build_graph() def buildroot_files(self, relpath=None): """Returns the set of all files under the test build root. :API: public :param string relpath: If supplied, only collect files from this subtree. :returns: All file paths found. :rtype: set """ def scan(): for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or '')): for f in files: yield os.path.relpath(os.path.join(root, f), self.build_root) return set(scan()) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns) self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, for_subsystems=None, options=None, target_roots=None, console_outstream=None, workspace=None, **kwargs): """ :API: public :param dict **kwargs: keyword arguments passed in to `create_options_for_optionables`. """ # Many tests use source root functionality via the SourceRootConfig.global_instance(). # (typically accessed via Target.target_base), so we always set it up, for convenience. optionables = {SourceRootConfig} extra_scopes = set() for_subsystems = for_subsystems or () for subsystem in for_subsystems: if subsystem.options_scope is None: raise TaskError('You must set a scope on your subsystem type before using it in tests.') optionables.add(subsystem) for_task_types = for_task_types or () for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError('You must set a scope on your task type before using it in tests.') optionables.add(task_type) # If task is expected to inherit goal-level options, register those directly on the task, # by subclassing the goal options registrar and settings its scope to the task scope. if issubclass(task_type, GoalOptionsMixin): subclass_name = b'test_{}_{}_{}'.format( task_type.__name__, task_type.goal_options_registrar_cls.options_scope, task_type.options_scope) optionables.add(type(subclass_name, (task_type.goal_options_registrar_cls, ), {b'options_scope': task_type.options_scope})) extra_scopes.update([si.scope for si in task_type.known_scope_infos()]) optionables.update(Subsystem.closure( set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) | self._build_configuration.subsystems())) # Now default the option values and override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. options = options.copy() if options else {} for s, opts in self.options.items(): scoped_opts = options.setdefault(s, {}) scoped_opts.update(opts) fake_options = create_options_for_optionables( optionables, extra_scopes=extra_scopes, options=options, **kwargs) Subsystem.reset(reset_options=True) Subsystem.set_options(fake_options) context = create_context_from_options(fake_options, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) return context def tearDown(self): """ :API: public """ super(BaseTest, self).tearDown() BuildFile.clear_cache() Subsystem.reset() def target(self, spec): """Resolves the given target address to a Target object. :API: public address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = Address.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. :API: public spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec = CmdLineSpecParser(self.build_root).parse_spec(spec) addresses = list(self.address_mapper.scan_specs([spec])) for address in addresses: self.build_graph.inject_address_closure(address) targets = [self.build_graph.get_target(address) for address in addresses] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. :API: public path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources :API: public path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file(path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict(target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), java_sources=('java_sources=[%s],' % ','.join(map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): """ :API: public """ return self.create_library(path, 'resources', name, sources) def assertUnorderedPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, unordered. :API: public """ actual = list(itertools.islice(actual_iter, len(expected))) self.assertEqual(sorted(expected), sorted(actual)) def assertPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, in order. :API: public """ self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected)))) def assertInFile(self, string, file_path): """Verifies that a string appears in a file :API: public """ with open(file_path) as f: content = f.read() self.assertIn(string, content, '"{}" is not in the file {}:\n{}'.format(string, f.name, content)) def get_bootstrap_options(self, cli_options=()): """Retrieves bootstrap options. :param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`. """ # Can't parse any options without a pants.ini. self.create_file('pants.ini') return OptionsBootstrapper(args=cli_options).get_bootstrap_options().for_global_scope() class LoggingRecorder(object): """Simple logging handler to record warnings.""" def __init__(self): self._records = [] self.level = logging.DEBUG def handle(self, record): self._records.append(record) def _messages_for_level(self, levelname): return ['{}: {}'.format(record.name, record.getMessage()) for record in self._records if record.levelname == levelname] def infos(self): return self._messages_for_level('INFO') def warnings(self): return self._messages_for_level('WARNING') @contextmanager def captured_logging(self, level=None): root_logger = logging.getLogger() old_level = root_logger.level root_logger.setLevel(level or logging.NOTSET) handler = self.LoggingRecorder() root_logger.addHandler(handler) try: yield handler finally: root_logger.setLevel(old_level) root_logger.removeHandler(handler)
def setUp(self): self.build_configuration = BuildConfiguration() self.working_set = WorkingSet() for entry in working_set.entries: self.working_set.add_entry(entry)
class BuildConfigurationTest(unittest.TestCase): def setUp(self): self.build_configuration = BuildConfiguration() def _register_aliases(self, **kwargs): self.build_configuration.register_aliases(BuildFileAliases(**kwargs)) def test_register_bad(self): with self.assertRaises(TypeError): self.build_configuration.register_aliases(42) def test_register_target_alias(self): class Fred(Target): pass self._register_aliases(targets={'fred': Fred}) aliases = self.build_configuration.registered_aliases() self.assertEqual({}, aliases.target_macro_factories) self.assertEqual({}, aliases.objects) self.assertEqual({}, aliases.context_aware_object_factories) self.assertEqual(dict(fred=Fred), aliases.target_types) with self._create_mock_build_file('fred') as build_file: parse_state = self.build_configuration.initialize_parse_state(build_file) self.assertEqual(0, len(parse_state.objects)) self.assertEqual(1, len(parse_state.parse_globals)) target_call_proxy = parse_state.parse_globals['fred'] target_call_proxy(name='jake') self.assertEqual(1, len(parse_state.objects)) target_proxy = parse_state.objects[0] self.assertEqual('jake', target_proxy.addressed_name) self.assertEqual(Fred, target_proxy.addressed_type) def test_register_target_macro_facory(self): class Fred(Target): pass class FredMacro(TargetMacro): def __init__(self, parse_context): self._parse_context = parse_context def expand(self, *args, **kwargs): return self._parse_context.create_object(Fred, name='frog', dependencies=[kwargs['name']]) class FredFactory(TargetMacro.Factory): @property def target_types(self): return {Fred} def macro(self, parse_context): return FredMacro(parse_context) factory = FredFactory() self._register_aliases(targets={'fred': factory}) aliases = self.build_configuration.registered_aliases() self.assertEqual({}, aliases.target_types) self.assertEqual({}, aliases.objects) self.assertEqual({}, aliases.context_aware_object_factories) self.assertEqual(dict(fred=factory), aliases.target_macro_factories) with self._create_mock_build_file('fred') as build_file: parse_state = self.build_configuration.initialize_parse_state(build_file) self.assertEqual(0, len(parse_state.objects)) self.assertEqual(1, len(parse_state.parse_globals)) target_call_proxy = parse_state.parse_globals['fred'] target_call_proxy(name='jake') self.assertEqual(1, len(parse_state.objects)) target_proxy = parse_state.objects[0] self.assertEqual('frog', target_proxy.addressed_name) self.assertEqual(Fred, target_proxy.addressed_type) self.assertEqual(['jake'], target_proxy.dependency_specs) def test_register_exposed_object(self): self._register_aliases(objects={'jane': 42}) aliases = self.build_configuration.registered_aliases() self.assertEqual({}, aliases.target_types) self.assertEqual({}, aliases.target_macro_factories) self.assertEqual({}, aliases.context_aware_object_factories) self.assertEqual(dict(jane=42), aliases.objects) with self._create_mock_build_file('jane') as build_file: parse_state = self.build_configuration.initialize_parse_state(build_file) self.assertEqual(0, len(parse_state.objects)) self.assertEqual(1, len(parse_state.parse_globals)) self.assertEqual(42, parse_state.parse_globals['jane']) def test_register_exposed_context_aware_function(self): self.do_test_exposed_context_aware_function(lambda context: lambda: context.rel_path) self.do_test_exposed_context_aware_function(lambda context=None: lambda: context.rel_path) def george_method(self, parse_context): return lambda: parse_context.rel_path def test_register_exposed_context_aware_method(self): self.do_test_exposed_context_aware_function(self.george_method) @classmethod def george_classmethod(cls, parse_context): return lambda: parse_context.rel_path def test_register_exposed_context_aware_classmethod(self): self.do_test_exposed_context_aware_function(self.george_classmethod) @staticmethod def george_staticmethod(parse_context): return lambda: parse_context.rel_path def test_register_exposed_context_aware_staticmethod(self): self.do_test_exposed_context_aware_function(self.george_staticmethod) def do_test_exposed_context_aware_function(self, func, *args, **kwargs): with self.do_test_exposed_context_aware_object(func) as context_aware_object: self.assertEqual('george', context_aware_object(*args, **kwargs)) def test_register_exposed_context_aware_class(self): class George(object): def __init__(self, parse_context): self._parse_context = parse_context def honorific(self): return len(self._parse_context.rel_path) with self.do_test_exposed_context_aware_object(George) as context_aware_object: self.assertEqual(6, context_aware_object.honorific()) @contextmanager def do_test_exposed_context_aware_object(self, context_aware_object_factory): self._register_aliases(context_aware_object_factories={'george': context_aware_object_factory}) aliases = self.build_configuration.registered_aliases() self.assertEqual({}, aliases.target_types) self.assertEqual({}, aliases.target_macro_factories) self.assertEqual({}, aliases.objects) self.assertEqual(dict(george=context_aware_object_factory), aliases.context_aware_object_factories) with temporary_dir() as root: build_file_path = os.path.join(root, 'george', 'BUILD') touch(build_file_path) build_file = BuildFile(FileSystemProjectTree(root), 'george/BUILD') parse_state = self.build_configuration.initialize_parse_state(build_file) self.assertEqual(0, len(parse_state.objects)) self.assertEqual(1, len(parse_state.parse_globals)) yield parse_state.parse_globals['george'] @contextmanager def _create_mock_build_file(self, dirname): with temporary_dir() as root: os.mkdir(os.path.join(root, dirname)) touch(os.path.join(root, dirname, 'BUILD')) yield BuildFile(FileSystemProjectTree(root), os.path.join(dirname, 'BUILD'))
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot. :API: public """ def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path. :API: public """ if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. :API: public relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_workdir_dir(self, relpath): """Creates a directory under the work directory. :API: public relpath: The relative path to the directory from the work directory. """ path = os.path.join(self.pants_workdir, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. :API: public relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def create_workdir_file(self, relpath, contents='', mode='wb'): """Writes to a file under the work directory. :API: public relpath: The relative path to the file from the work directory. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.pants_workdir, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. :API: public relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') return BuildFile(self.address_mapper._project_tree, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, derived_from=None, synthetic=False, **kwargs): """Creates a target and injects it into the test's build graph. :API: public :param string spec: The target address spec that locates this target. :param type target_type: The concrete target subclass to create this new target from. :param list dependencies: A list of target instances this new target depends on. :param derived_from: The target this new target was derived from. :type derived_from: :class:`pants.build_graph.target.Target` """ address = Address.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] self.build_graph.apply_injectables([target]) self.build_graph.inject_target( target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from, synthetic=synthetic) # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph. # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it. traversables = [ target.compute_dependency_specs(payload=target.payload) ] for dependency_spec in itertools.chain(*traversables): dependency_address = Address.parse(dependency_spec, relative_to=address.spec_path) dependency_target = self.build_graph.get_target(dependency_address) if not dependency_target: raise ValueError( 'Tests must make targets for dependency specs ahead of them ' 'being traversed, {} tried to traverse {} which does not exist.' .format(target, dependency_address)) if dependency_target not in target.dependencies: self.build_graph.inject_dependency( dependent=target.address, dependency=dependency_address) target.mark_transitive_invalidation_hash_dirty() return target @property def alias_groups(self): """ :API: public """ return BuildFileAliases(targets={'target': Target}) @property def build_ignore_patterns(self): """ :API: public """ return None def setUp(self): """ :API: public """ super(BaseTest, self).setUp() # Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup. clean_global_runtime_state(reset_subsystem=True) self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.subprocess_dir = os.path.join(self.build_root, '.pids') self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[GLOBAL_SCOPE] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'pants_subprocessdir': self.subprocess_dir, 'cache_key_gen_version': '0-test', } self.options['cache'] = { 'read_from': [], 'write_to': [], } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.project_tree = FileSystemProjectTree(self.build_root) self.reset_build_graph() def buildroot_files(self, relpath=None): """Returns the set of all files under the test build root. :API: public :param string relpath: If supplied, only collect files from this subtree. :returns: All file paths found. :rtype: set """ def scan(): for root, dirs, files in os.walk( os.path.join(self.build_root, relpath or '')): for f in files: yield os.path.relpath(os.path.join(root, f), self.build_root) return set(scan()) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper( self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns) self.build_graph = MutableBuildGraph( address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, for_subsystems=None, options=None, target_roots=None, console_outstream=None, workspace=None, scheduler=None, **kwargs): """ :API: public :param dict **kwargs: keyword arguments passed in to `create_options_for_optionables`. """ # Many tests use source root functionality via the SourceRootConfig.global_instance(). # (typically accessed via Target.target_base), so we always set it up, for convenience. for_subsystems = set(for_subsystems or ()) for subsystem in for_subsystems: if subsystem.options_scope is None: raise TaskError( 'You must set a scope on your subsystem type before using it in tests.' ) optionables = { SourceRootConfig } | self._build_configuration.subsystems() | for_subsystems for_task_types = for_task_types or () for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError( 'You must set a scope on your task type before using it in tests.' ) optionables.add(task_type) # If task is expected to inherit goal-level options, register those directly on the task, # by subclassing the goal options registrar and settings its scope to the task scope. if issubclass(task_type, GoalOptionsMixin): subclass_name = b'test_{}_{}_{}'.format( task_type.__name__, task_type.goal_options_registrar_cls.options_scope, task_type.options_scope) optionables.add( type(subclass_name, (task_type.goal_options_registrar_cls, ), {b'options_scope': task_type.options_scope})) # Now expand to all deps. all_optionables = set() for optionable in optionables: all_optionables.update(si.optionable_cls for si in optionable.known_scope_infos()) # Now default the option values and override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. options = options.copy() if options else {} for s, opts in self.options.items(): scoped_opts = options.setdefault(s, {}) scoped_opts.update(opts) fake_options = create_options_for_optionables(all_optionables, options=options, **kwargs) Subsystem.reset(reset_options=True) Subsystem.set_options(fake_options) context = create_context_from_options( fake_options, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace, scheduler=scheduler) return context def tearDown(self): """ :API: public """ super(BaseTest, self).tearDown() BuildFile.clear_cache() Subsystem.reset() def target(self, spec): """Resolves the given target address to a Target object. :API: public address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = Address.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. :API: public spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec = CmdLineSpecParser(self.build_root).parse_spec(spec) addresses = list(self.address_mapper.scan_specs([spec])) for address in addresses: self.build_graph.inject_address_closure(address) targets = [ self.build_graph.get_target(address) for address in addresses ] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. :API: public path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources :API: public path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file( path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict( target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), java_sources=('java_sources=[%s],' % ','.join( map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): """ :API: public """ return self.create_library(path, 'resources', name, sources) def assertUnorderedPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, unordered. :API: public """ actual = list(itertools.islice(actual_iter, len(expected))) self.assertEqual(sorted(expected), sorted(actual)) def assertPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, in order. :API: public """ self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected)))) def assertInFile(self, string, file_path): """Verifies that a string appears in a file :API: public """ with open(file_path) as f: content = f.read() self.assertIn( string, content, '"{}" is not in the file {}:\n{}'.format( string, f.name, content)) def get_bootstrap_options(self, cli_options=()): """Retrieves bootstrap options. :param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`. """ # Can't parse any options without a pants.ini. self.create_file('pants.ini') return OptionsBootstrapper( args=cli_options).get_bootstrap_options().for_global_scope() class LoggingRecorder(object): """Simple logging handler to record warnings.""" def __init__(self): self._records = [] self.level = logging.DEBUG def handle(self, record): self._records.append(record) def _messages_for_level(self, levelname): return [ '{}: {}'.format(record.name, record.getMessage()) for record in self._records if record.levelname == levelname ] def infos(self): return self._messages_for_level('INFO') def warnings(self): return self._messages_for_level('WARNING') @contextmanager def captured_logging(self, level=None): root_logger = logging.getLogger() old_level = root_logger.level root_logger.setLevel(level or logging.NOTSET) handler = self.LoggingRecorder() root_logger.addHandler(handler) try: yield handler finally: root_logger.setLevel(old_level) root_logger.removeHandler(handler)
class BaseTest(unittest.TestCase): """A baseclass useful for tests requiring a temporary buildroot.""" def build_path(self, relpath): """Returns the canonical BUILD file path for the given relative build path.""" if os.path.basename(relpath).startswith('BUILD'): return relpath else: return os.path.join(relpath, 'BUILD') def create_dir(self, relpath): """Creates a directory under the buildroot. relpath: The relative path to the directory from the build root. """ path = os.path.join(self.build_root, relpath) safe_mkdir(path) return path def create_workdir_dir(self, relpath): """Creates a directory under the work directory. relpath: The relative path to the directory from the work directory. """ path = os.path.join(self.pants_workdir, relpath) safe_mkdir(path) return path def create_file(self, relpath, contents='', mode='wb'): """Writes to a file under the buildroot. relpath: The relative path to the file from the build root. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.build_root, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def create_workdir_file(self, relpath, contents='', mode='wb'): """Writes to a file under the work directory. relpath: The relative path to the file from the work directory. contents: A string containing the contents of the file - '' by default.. mode: The mode to write to the file in - over-write by default. """ path = os.path.join(self.pants_workdir, relpath) with safe_open(path, mode=mode) as fp: fp.write(contents) return path def add_to_build_file(self, relpath, target): """Adds the given target specification to the BUILD file at relpath. relpath: The relative path to the BUILD file from the build root. target: A string containing the target definition as it would appear in a BUILD file. """ self.create_file(self.build_path(relpath), target, mode='a') cls = self.address_mapper._build_file_type return cls(root_dir=self.build_root, relpath=self.build_path(relpath)) def make_target(self, spec='', target_type=Target, dependencies=None, derived_from=None, **kwargs): """Creates a target and injects it into the test's build graph. :param string spec: The target address spec that locates this target. :param type target_type: The concrete target subclass to create this new target from. :param list dependencies: A list of target instances this new target depends on. :param derived_from: The target this new target was derived from. :type derived_from: :class:`pants.build_graph.target.Target` """ address = Address.parse(spec) target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs) dependencies = dependencies or [] self.build_graph.inject_target(target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from) # TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph. # Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it. for traversable_dependency_spec in target.traversable_dependency_specs: traversable_dependency_address = Address.parse(traversable_dependency_spec, relative_to=address.spec_path) traversable_dependency_target = self.build_graph.get_target(traversable_dependency_address) if not traversable_dependency_target: raise ValueError('Tests must make targets for traversable dependency specs ahead of them ' 'being traversed, {} tried to traverse {} which does not exist.' .format(target, traversable_dependency_address)) if traversable_dependency_target not in target.dependencies: self.build_graph.inject_dependency(dependent=target.address, dependency=traversable_dependency_address) target.mark_transitive_invalidation_hash_dirty() return target @property def alias_groups(self): return BuildFileAliases(targets={'target': Target}) def setUp(self): super(BaseTest, self).setUp() Goal.clear() Subsystem.reset() self.real_build_root = BuildRoot().path self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT')) self.addCleanup(safe_rmtree, self.build_root) self.pants_workdir = os.path.join(self.build_root, '.pants.d') safe_mkdir(self.pants_workdir) self.options = defaultdict(dict) # scope -> key-value mapping. self.options[''] = { 'pants_workdir': self.pants_workdir, 'pants_supportdir': os.path.join(self.build_root, 'build-support'), 'pants_distdir': os.path.join(self.build_root, 'dist'), 'pants_configdir': os.path.join(self.build_root, 'config'), 'cache_key_gen_version': '0-test', } BuildRoot().path = self.build_root self.addCleanup(BuildRoot().reset) # We need a pants.ini, even if empty. get_buildroot() uses its presence. self.create_file('pants.ini') self._build_configuration = BuildConfiguration() self._build_configuration.register_aliases(self.alias_groups) self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root) self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def buildroot_files(self, relpath=None): """Returns the set of all files under the test build root. :param string relpath: If supplied, only collect files from this subtree. :returns: All file paths found. :rtype: set """ def scan(): for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or '')): for f in files: yield os.path.relpath(os.path.join(root, f), self.build_root) return set(scan()) def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper) def set_options_for_scope(self, scope, **kwargs): self.options[scope].update(kwargs) def context(self, for_task_types=None, options=None, passthru_args=None, target_roots=None, console_outstream=None, workspace=None, for_subsystems=None): # Many tests use source root functionality via the SourceRootConfig.global_instance() # (typically accessed via Target.target_base), so we always set it up, for convenience. optionables = {SourceRootConfig} extra_scopes = set() for_subsystems = for_subsystems or () for subsystem in for_subsystems: if subsystem.options_scope is None: raise TaskError('You must set a scope on your subsystem type before using it in tests.') optionables.add(subsystem) for_task_types = for_task_types or () for task_type in for_task_types: scope = task_type.options_scope if scope is None: raise TaskError('You must set a scope on your task type before using it in tests.') optionables.add(task_type) extra_scopes.update([si.scope for si in task_type.known_scope_infos()]) optionables.update(Subsystem.closure( set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) | self._build_configuration.subsystems())) # Now default the option values and override with any caller-specified values. # TODO(benjy): Get rid of the options arg, and require tests to call set_options. options = options.copy() if options else {} for s, opts in self.options.items(): scoped_opts = options.setdefault(s, {}) scoped_opts.update(opts) options = create_options_for_optionables(optionables, extra_scopes=extra_scopes, options=options) Subsystem._options = options context = create_context(options=options, passthru_args=passthru_args, target_roots=target_roots, build_graph=self.build_graph, build_file_parser=self.build_file_parser, address_mapper=self.address_mapper, console_outstream=console_outstream, workspace=workspace) return context def tearDown(self): super(BaseTest, self).tearDown() FilesystemBuildFile.clear_cache() Subsystem.reset() def target(self, spec): """Resolves the given target address to a Target object. address: The BUILD target address to resolve. Returns the corresponding Target or else None if the address does not point to a defined Target. """ address = Address.parse(spec) self.build_graph.inject_address_closure(address) return self.build_graph.get_target(address) def targets(self, spec): """Resolves a target spec to one or more Target objects. spec: Either BUILD target address or else a target glob using the siblings ':' or descendants '::' suffixes. Returns the set of all Targets found. """ spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper) addresses = list(spec_parser.parse_addresses(spec)) for address in addresses: self.build_graph.inject_address_closure(address) targets = [self.build_graph.get_target(address) for address in addresses] return targets def create_files(self, path, files): """Writes to a file under the buildroot with contents same as file name. path: The relative path to the file from the build root. files: List of file names. """ for f in files: self.create_file(os.path.join(path, f), contents=f) def create_library(self, path, target_type, name, sources=None, **kwargs): """Creates a library target of given type at the BUILD file at path with sources path: The relative path to the BUILD file from the build root. target_type: valid pants target type. name: Name of the library target. sources: List of source file at the path relative to path. **kwargs: Optional attributes that can be set for any library target. Currently it includes support for resources, java_sources, provides and dependencies. """ if sources: self.create_files(path, sources) self.add_to_build_file(path, dedent(''' %(target_type)s(name='%(name)s', %(sources)s %(resources)s %(java_sources)s %(provides)s %(dependencies)s ) ''' % dict(target_type=target_type, name=name, sources=('sources=%s,' % repr(sources) if sources else ''), resources=('resources=["%s"],' % kwargs.get('resources') if 'resources' in kwargs else ''), java_sources=('java_sources=[%s],' % ','.join(map(lambda str_target: '"%s"' % str_target, kwargs.get('java_sources'))) if 'java_sources' in kwargs else ''), provides=('provides=%s,' % kwargs.get('provides') if 'provides' in kwargs else ''), dependencies=('dependencies=%s,' % kwargs.get('dependencies') if 'dependencies' in kwargs else ''), ))) return self.target('%s:%s' % (path, name)) def create_resources(self, path, name, *sources): return self.create_library(path, 'resources', name, sources) def assertUnorderedPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, unordered.""" actual = list(itertools.islice(actual_iter, len(expected))) self.assertEqual(sorted(expected), sorted(actual)) def assertPrefixEqual(self, expected, actual_iter): """Consumes len(expected) items from the given iter, and asserts that they match, in order.""" self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected))))
class BuildConfigurationTest(unittest.TestCase): def setUp(self): self.build_configuration = BuildConfiguration() def _register_aliases(self, **kwargs): self.build_configuration.register_aliases(BuildFileAliases(**kwargs)) def test_register_bad(self): with self.assertRaises(TypeError): self.build_configuration.register_aliases(42) def test_register_target_alias(self): class Fred(Target): pass self._register_aliases(targets={'fred': Fred}) aliases = self.build_configuration.registered_aliases() self.assertEqual({}, aliases.target_macro_factories) self.assertEqual({}, aliases.objects) self.assertEqual({}, aliases.context_aware_object_factories) self.assertEqual(dict(fred=Fred), aliases.target_types) build_file = FilesystemBuildFile('/tmp', 'fred', must_exist=False) parse_state = self.build_configuration.initialize_parse_state( build_file) self.assertEqual(0, len(parse_state.registered_addressable_instances)) self.assertEqual(1, len(parse_state.parse_globals)) target_call_proxy = parse_state.parse_globals['fred'] target_call_proxy(name='jake') self.assertEqual(1, len(parse_state.registered_addressable_instances)) name, target_proxy = parse_state.registered_addressable_instances.pop() self.assertEqual('jake', target_proxy.addressed_name) self.assertEqual(Fred, target_proxy.addressed_type) def test_register_target_macro_facory(self): class Fred(Target): pass class FredMacro(TargetMacro): def __init__(self, parse_context): self._parse_context = parse_context def expand(self, *args, **kwargs): return self._parse_context.create_object( Fred, name='frog', dependencies=[kwargs['name']]) class FredFactory(TargetMacro.Factory): @property def target_types(self): return {Fred} def macro(self, parse_context): return FredMacro(parse_context) factory = FredFactory() self._register_aliases(targets={'fred': factory}) aliases = self.build_configuration.registered_aliases() self.assertEqual({}, aliases.target_types) self.assertEqual({}, aliases.objects) self.assertEqual({}, aliases.context_aware_object_factories) self.assertEqual(dict(fred=factory), aliases.target_macro_factories) build_file = FilesystemBuildFile('/tmp', 'fred', must_exist=False) parse_state = self.build_configuration.initialize_parse_state( build_file) self.assertEqual(0, len(parse_state.registered_addressable_instances)) self.assertEqual(1, len(parse_state.parse_globals)) target_call_proxy = parse_state.parse_globals['fred'] target_call_proxy(name='jake') self.assertEqual(1, len(parse_state.registered_addressable_instances)) name, target_proxy = parse_state.registered_addressable_instances.pop() self.assertEqual('frog', target_proxy.addressed_name) self.assertEqual(Fred, target_proxy.addressed_type) self.assertEqual(['jake'], target_proxy.dependency_specs) def test_register_exposed_object(self): self._register_aliases(objects={'jane': 42}) aliases = self.build_configuration.registered_aliases() self.assertEqual({}, aliases.target_types) self.assertEqual({}, aliases.target_macro_factories) self.assertEqual({}, aliases.context_aware_object_factories) self.assertEqual(dict(jane=42), aliases.objects) build_file = FilesystemBuildFile('/tmp', 'jane', must_exist=False) parse_state = self.build_configuration.initialize_parse_state( build_file) self.assertEqual(0, len(parse_state.registered_addressable_instances)) self.assertEqual(1, len(parse_state.parse_globals)) self.assertEqual(42, parse_state.parse_globals['jane']) def test_register_exposed_context_aware_function(self): self.do_test_exposed_context_aware_function( lambda context: lambda: context.rel_path) self.do_test_exposed_context_aware_function( lambda context=None: lambda: context.rel_path) def george_method(self, parse_context): return lambda: parse_context.rel_path def test_register_exposed_context_aware_method(self): self.do_test_exposed_context_aware_function(self.george_method) @classmethod def george_classmethod(cls, parse_context): return lambda: parse_context.rel_path def test_register_exposed_context_aware_classmethod(self): self.do_test_exposed_context_aware_function(self.george_classmethod) @staticmethod def george_staticmethod(parse_context): return lambda: parse_context.rel_path def test_register_exposed_context_aware_staticmethod(self): self.do_test_exposed_context_aware_function(self.george_staticmethod) def do_test_exposed_context_aware_function(self, func, *args, **kwargs): with self.do_test_exposed_context_aware_object( func) as context_aware_object: self.assertEqual('george', context_aware_object(*args, **kwargs)) def test_register_exposed_context_aware_class(self): class George(object): def __init__(self, parse_context): self._parse_context = parse_context def honorific(self): return len(self._parse_context.rel_path) with self.do_test_exposed_context_aware_object( George) as context_aware_object: self.assertEqual(6, context_aware_object.honorific()) @contextmanager def do_test_exposed_context_aware_object(self, context_aware_object_factory): self._register_aliases(context_aware_object_factories={ 'george': context_aware_object_factory }) aliases = self.build_configuration.registered_aliases() self.assertEqual({}, aliases.target_types) self.assertEqual({}, aliases.target_macro_factories) self.assertEqual({}, aliases.objects) self.assertEqual(dict(george=context_aware_object_factory), aliases.context_aware_object_factories) with temporary_dir() as root: build_file_path = os.path.join(root, 'george', 'BUILD') touch(build_file_path) build_file = FilesystemBuildFile(root, 'george') parse_state = self.build_configuration.initialize_parse_state( build_file) self.assertEqual(0, len(parse_state.registered_addressable_instances)) self.assertEqual(1, len(parse_state.parse_globals)) yield parse_state.parse_globals['george']