def setUp(self):
super(BaseTestWithParser, self).setUp()
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration,
self.build_root)
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'cache_key_gen_version': '0-test',
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
# We need a pants.ini, even if empty. get_buildroot() uses its presence.
self.create_file('pants.ini')
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def load_backend(
build_configuration: BuildConfiguration, backend_package: str, is_v1_backend: bool
) -> None:
"""Installs the given backend package into the build configuration.
:param build_configuration: the BuildConfiguration to install the backend plugin into.
:param backend_package: the package name containing the backend plugin register module that
provides the plugin entrypoints.
:param is_v1_backend: Is this a v1 or v2 backend.
:raises: :class:``pants.base.exceptions.BuildConfigurationError`` if there is a problem loading
the build configuration.
"""
backend_module = backend_package + ".register"
try:
module = importlib.import_module(backend_module)
except ImportError as ex:
traceback.print_exc()
raise BackendConfigurationError(f"Failed to load the {backend_module} backend: {ex!r}")
def invoke_entrypoint(name):
entrypoint = getattr(module, name, lambda: None)
try:
return entrypoint()
except TypeError as e:
traceback.print_exc()
raise BackendConfigurationError(
f"Entrypoint {name} in {backend_module} must be a zero-arg callable: {e!r}"
)
# While the Target API is a V2 concept, we expect V1 plugin authors to still write Target
# API bindings. So, we end up using this entry point regardless of V1 vs. V2.
targets = invoke_entrypoint("targets2")
if targets:
build_configuration.register_targets(targets)
if is_v1_backend:
invoke_entrypoint("register_goals")
subsystems = invoke_entrypoint("global_subsystems")
if subsystems:
build_configuration.register_optionables(subsystems)
# For now, `build_file_aliases` is still V1-only. TBD what entry-point we use for
# `objects` and `context_aware_object_factories`.
build_file_aliases = invoke_entrypoint("build_file_aliases")
if build_file_aliases:
build_configuration.register_aliases(build_file_aliases)
else:
rules = invoke_entrypoint("rules")
if rules:
build_configuration.register_rules(rules)
build_file_aliases2 = invoke_entrypoint("build_file_aliases2")
if build_file_aliases2:
build_configuration.register_aliases(build_file_aliases2)
async def find_owners(
build_configuration: BuildConfiguration,
address_mapper: AddressMapper,
changed_request: ChangedRequest,
) -> ChangedAddresses:
owners = await Get[Owners](OwnersRequest(sources=changed_request.sources))
# If the ChangedRequest does not require dependees, then we're done.
if changed_request.include_dependees == IncludeDependeesOption.NONE:
return ChangedAddresses(owners.addresses)
# Otherwise: find dependees.
all_addresses = await Get[Addresses](AddressSpecs(
(DescendantAddresses(""), )))
all_structs = [
s.value for s in await MultiGet(Get[HydratedStruct](Address, a)
for a in all_addresses)
]
bfa = build_configuration.registered_aliases()
graph = _DependentGraph.from_iterable(
target_types_from_build_file_aliases(bfa), address_mapper, all_structs)
if changed_request.include_dependees == IncludeDependeesOption.DIRECT:
return ChangedAddresses(
Addresses(graph.dependents_of_addresses(owners.addresses)))
return ChangedAddresses(
Addresses(graph.transitive_dependents_of_addresses(owners.addresses)))
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.subprocess_dir = os.path.join(self.build_root, '.pids')
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'pants_subprocessdir': self.subprocess_dir,
'cache_key_gen_version': '0-test',
}
self.options['cache'] = {
'read_from': [],
'write_to': [],
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration,
self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.address_mapper = BuildFileAddressMapper(
self.build_file_parser,
self.project_tree,
build_ignore_patterns=self.build_ignore_patterns)
self.build_graph = MutableBuildGraph(
address_mapper=self.address_mapper)
async def find_owners(build_configuration: BuildConfiguration,
address_mapper: AddressMapper,
owners_request: OwnersRequest) -> BuildFileAddresses:
sources_set = OrderedSet(owners_request.sources)
dirs_set = OrderedSet(dirname(source) for source in sources_set)
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
candidate_specs = tuple(AscendantAddresses(directory=d) for d in dirs_set)
candidate_targets = await Get(HydratedTargets, Specs(candidate_specs))
# Match the source globs against the expanded candidate targets.
def owns_any_source(legacy_target):
"""Given a `HydratedTarget` instance, check if it owns the given source file."""
target_kwargs = legacy_target.adaptor.kwargs()
# Handle `sources`-declaring targets.
# NB: Deleted files can only be matched against the 'filespec' (ie, `PathGlobs`) for a target,
# so we don't actually call `fileset.matches` here.
# TODO: This matching logic should be implemented using the rust `fs` crate for two reasons:
# 1) having two implementations isn't great
# 2) we're expanding sources via HydratedTarget, but it isn't necessary to do that to match
target_sources = target_kwargs.get('sources', None)
if target_sources and any_matches_filespec(sources_set,
target_sources.filespec):
return True
return False
direct_owners = tuple(
ht.adaptor.address for ht in candidate_targets
if LegacyAddressMapper.any_is_declaring_file(
ht.adaptor.address, sources_set) or owns_any_source(ht))
# If the OwnersRequest does not require dependees, then we're done.
if owners_request.include_dependees == 'none':
return BuildFileAddresses(direct_owners)
else:
# Otherwise: find dependees.
all_addresses = await Get(BuildFileAddresses,
Specs((DescendantAddresses(''), )))
all_hydrated_structs = await MultiGet(
Get(HydratedStruct, Address, a.to_address())
for a in all_addresses)
all_structs = [hs.value for hs in all_hydrated_structs]
bfa = build_configuration.registered_aliases()
graph = _DependentGraph.from_iterable(
target_types_from_build_file_aliases(bfa), address_mapper,
all_structs)
if owners_request.include_dependees == 'direct':
return BuildFileAddresses(
tuple(graph.dependents_of_addresses(direct_owners)))
else:
assert owners_request.include_dependees == 'transitive'
return BuildFileAddresses(
tuple(graph.transitive_dependents_of_addresses(direct_owners)))
def __init__(
self,
*,
rules: Optional[Iterable] = None,
target_types: Optional[Iterable[Type[Target]]] = None,
objects: Optional[Dict[str, Any]] = None,
context_aware_object_factories: Optional[Dict[str, Any]] = None,
) -> None:
self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT"))
safe_mkdir(self.build_root, clean=True)
safe_mkdir(self.pants_workdir)
BuildRoot().path = self.build_root
# TODO: Redesign rule registration for tests to be more ergonomic and to make this less
# special-cased.
all_rules = (
*(rules or ()),
*source_root.rules(),
*pants_environment.rules(),
QueryRule(WrappedTarget, (Address,)),
)
build_config_builder = BuildConfiguration.Builder()
build_config_builder.register_aliases(
BuildFileAliases(
objects=objects, context_aware_object_factories=context_aware_object_factories
)
)
build_config_builder.register_rules(all_rules)
build_config_builder.register_target_types(target_types or ())
self.build_config = build_config_builder.create()
options_bootstrapper = create_options_bootstrapper()
global_options = options_bootstrapper.bootstrap_options.for_global_scope()
local_store_dir = global_options.local_store_dir
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
graph_session = EngineInitializer.setup_graph_extended(
pants_ignore_patterns=[],
use_gitignore=False,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
native=Native(),
options_bootstrapper=options_bootstrapper,
build_root=self.build_root,
build_configuration=self.build_config,
execution_options=ExecutionOptions.from_bootstrap_options(global_options),
).new_session(
build_id="buildid_for_test",
session_values=SessionValues(
{OptionsBootstrapper: options_bootstrapper, PantsEnvironment: PantsEnvironment()}
),
should_report_workunits=True,
)
self.scheduler = graph_session.scheduler_session
def load_plugins_and_backends(plugins, working_set, backends): """Load named plugins and source backends :param list<str> plugins: Plugins to load (see `load_plugins`). :param WorkingSet working_set: A pkg_resources.WorkingSet to load plugins from. :param list<str> backends: Source backends to load (see `load_build_configuration_from_source`). """ build_configuration = BuildConfiguration() load_plugins(build_configuration, plugins or [], working_set) load_build_configuration_from_source(build_configuration, additional_backends=backends or []) return build_configuration
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
# Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup.
clean_global_runtime_state(reset_runtracker=False,
reset_subsystem=True)
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.subprocess_dir = os.path.join(self.build_root, '.pids')
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'pants_subprocessdir': self.subprocess_dir,
'cache_key_gen_version': '0-test',
}
self.options['cache'] = {
'read_from': [],
'write_to': [],
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration,
self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.reset_build_graph()
def load_backends_and_plugins(plugins, working_set, backends, build_configuration=None):
"""Load named plugins and source backends
:param list<str> plugins: Plugins to load (see `load_plugins`). Plugins are loaded after
backends.
:param WorkingSet working_set: A pkg_resources.WorkingSet to load plugins from.
:param list<str> backends: Source backends to load (see `load_build_configuration_from_source`).
"""
build_configuration = build_configuration or BuildConfiguration()
load_build_configuration_from_source(build_configuration, backends)
load_plugins(build_configuration, plugins or [], working_set)
return build_configuration
def _load_plugins(self):
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in self._bootstrap_options.pythonpath:
if path not in sys.path:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
return load_backends_and_plugins(
self._bootstrap_options.plugins, self._bootstrap_options.plugins2,
self._working_set, self._bootstrap_options.backend_packages,
self._bootstrap_options.backend_packages2, BuildConfiguration())
def load_backend(build_configuration: BuildConfiguration, backend_package: str,
is_v1_backend: bool) -> None:
"""Installs the given backend package into the build configuration.
:param build_configuration: the BuildConfiguration to install the backend plugin into.
:param backend_package: the package name containing the backend plugin register module that
provides the plugin entrypoints.
:param is_v1_backend: Is this a v1 or v2 backend.
:raises: :class:``pants.base.exceptions.BuildConfigurationError`` if there is a problem loading
the build configuration.
"""
backend_module = backend_package + ".register"
try:
module = importlib.import_module(backend_module)
except ImportError as ex:
traceback.print_exc()
raise BackendConfigurationError(
f"Failed to load the {backend_module} backend: {ex!r}")
def invoke_entrypoint(name):
entrypoint = getattr(module, name, lambda: None)
try:
return entrypoint()
except TypeError as e:
traceback.print_exc()
raise BackendConfigurationError(
f"Entrypoint {name} in {backend_module} must be a zero-arg callable: {e!r}"
)
if is_v1_backend:
invoke_entrypoint("register_goals")
# TODO: Might v2 plugins need to register global subsystems? Hopefully not.
subsystems = invoke_entrypoint("global_subsystems")
if subsystems:
build_configuration.register_optionables(subsystems)
# The v2 target API is still TBD, so we keep build_file_aliases as a v1-only thing.
# Having thus no overlap between v1 and v2 backend entrypoints makes things much simpler.
# TODO: Revisit, ideally with a v2-only entry point, once the v2 target API is a thing.
build_file_aliases = invoke_entrypoint("build_file_aliases")
if build_file_aliases:
build_configuration.register_aliases(build_file_aliases)
else:
rules = invoke_entrypoint("rules")
if rules:
build_configuration.register_rules(rules)
build_file_aliases2 = invoke_entrypoint("build_file_aliases2")
if build_file_aliases2:
build_configuration.register_aliases(build_file_aliases2)
def load_backend(build_configuration: BuildConfiguration, backend_package: str,
is_v1_backend: bool) -> None:
"""Installs the given backend package into the build configuration.
:param build_configuration: the BuildConfiguration to install the backend plugin into.
:param backend_package: the package name containing the backend plugin register module that
provides the plugin entrypoints.
:param is_v1_backend: Is this a v1 or v2 backend.
:raises: :class:``pants.base.exceptions.BuildConfigurationError`` if there is a problem loading
the build configuration.
"""
backend_module = backend_package + ".register"
try:
module = importlib.import_module(backend_module)
except ImportError as ex:
traceback.print_exc()
raise BackendConfigurationError(
f"Failed to load the {backend_module} backend: {ex!r}")
def invoke_entrypoint(name):
entrypoint = getattr(module, name, lambda: None)
try:
return entrypoint()
except TypeError as e:
traceback.print_exc()
raise BackendConfigurationError(
f"Entrypoint {name} in {backend_module} must be a zero-arg callable: {e!r}"
)
# See the comment in `load_plugins` for why we load both `target_types` and
# `build_file_aliases` in both V1 and V2.
target_types = invoke_entrypoint("target_types")
if target_types:
build_configuration.register_target_types(target_types)
build_file_aliases = invoke_entrypoint("build_file_aliases")
if build_file_aliases:
build_configuration.register_aliases(build_file_aliases)
if is_v1_backend:
invoke_entrypoint("register_goals")
subsystems = invoke_entrypoint("global_subsystems")
if subsystems:
build_configuration.register_optionables(subsystems)
else:
rules = invoke_entrypoint("rules")
if rules:
build_configuration.register_rules(rules)
def load_backends_and_plugins(
plugins: List[str],
working_set: WorkingSet,
backends: List[str],
bc_builder: Optional[BuildConfiguration.Builder] = None,
) -> BuildConfiguration:
"""Load named plugins and source backends.
:param plugins: v2 plugins to load.
:param working_set: A pkg_resources.WorkingSet to load plugins from.
:param backends: v2 backends to load.
:param bc_builder: The BuildConfiguration (for adding aliases).
"""
bc_builder = bc_builder or BuildConfiguration.Builder()
load_build_configuration_from_source(bc_builder, backends)
load_plugins(bc_builder, plugins, working_set)
return bc_builder.create()
def create_bootstrap_scheduler(
options_bootstrapper: OptionsBootstrapper,
executor: PyExecutor | None = None) -> BootstrapScheduler:
bc_builder = BuildConfiguration.Builder()
# To load plugins, we only need access to the Python/PEX rules.
load_build_configuration_from_source(bc_builder, ["pants.backend.python"])
# And to plugin-loading-specific rules.
bc_builder.register_rules("_dummy_for_bootstrapping_",
plugin_resolver_rules())
# We allow unrecognized options to defer any option error handling until post-bootstrap.
bc_builder.allow_unknown_options()
return BootstrapScheduler(
EngineInitializer.setup_graph(
options_bootstrapper.bootstrap_options.for_global_scope(),
bc_builder.create(),
DynamicRemoteOptions.disabled(),
executor,
).scheduler)
def create_bootstrap_scheduler(
options_bootstrapper: OptionsBootstrapper,
executor: Optional[PyExecutor] = None,
) -> BootstrapScheduler:
bc_builder = BuildConfiguration.Builder()
# To load plugins, we only need access to the Python/PEX rules.
load_build_configuration_from_source(bc_builder, ["pants.backend.python"])
# And to plugin-loading-specific rules.
bc_builder.register_rules(plugin_resolver_rules())
# We allow unrecognized options to defer any option error handling until post-bootstrap.
bc_builder.allow_unknown_options()
return BootstrapScheduler(
EngineInitializer.setup_graph(
options_bootstrapper,
bc_builder.create(),
executor=executor,
# TODO: We use the default execution options to avoid invoking remote execution auth
# plugins. They should be loaded via rules using the bootstrap Scheduler in the future.
execution_options=DEFAULT_EXECUTION_OPTIONS,
).scheduler)
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.subprocess_dir = os.path.join(self.build_root, '.pids')
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'pants_subprocessdir': self.subprocess_dir,
'cache_key_gen_version': '0-test',
}
self.options['cache'] = {
'read_from': [],
'write_to': [],
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree,
build_ignore_patterns=self.build_ignore_patterns)
self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
# Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup.
clean_global_runtime_state(reset_subsystem=True)
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.subprocess_dir = os.path.join(self.build_root, '.pids')
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'pants_subprocessdir': self.subprocess_dir,
'cache_key_gen_version': '0-test',
}
self.options['cache'] = {
'read_from': [],
'write_to': [],
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.reset_build_graph()
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
# Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup.
clean_global_runtime_state(reset_runtracker=False, reset_subsystem=True)
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT"))
self.subprocess_dir = os.path.join(self.build_root, ".pids")
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, ".pants.d")
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[""] = {
"pants_workdir": self.pants_workdir,
"pants_supportdir": os.path.join(self.build_root, "build-support"),
"pants_distdir": os.path.join(self.build_root, "dist"),
"pants_configdir": os.path.join(self.build_root, "config"),
"pants_subprocessdir": self.subprocess_dir,
"cache_key_gen_version": "0-test",
}
self.options["cache"] = {"read_from": [], "write_to": []}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.reset_build_graph()
def load_backend(build_configuration: BuildConfiguration, backend_package: str) -> None:
"""Installs the given backend package into the build configuration.
:param build_configuration: the BuildConfiguration to install the backend plugin into.
:param backend_package: the package name containing the backend plugin register module that
provides the plugin entrypoints.
:raises: :class:``pants.base.exceptions.BuildConfigurationError`` if there is a problem loading
the build configuration."""
backend_module = backend_package + '.register'
try:
module = importlib.import_module(backend_module)
except ImportError as e:
traceback.print_exc()
raise BackendConfigurationError(f'Failed to load the {backend_module} backend: {e!r}')
def invoke_entrypoint(name):
entrypoint = getattr(module, name, lambda: None)
try:
return entrypoint()
except TypeError as e:
traceback.print_exc()
raise BackendConfigurationError(
f'Entrypoint {name} in {backend_module} must be a zero-arg callable: {e!r}'
)
build_file_aliases = invoke_entrypoint('build_file_aliases')
if build_file_aliases:
build_configuration.register_aliases(build_file_aliases)
subsystems = invoke_entrypoint('global_subsystems')
if subsystems:
build_configuration.register_optionables(subsystems)
rules = invoke_entrypoint('rules')
if rules:
build_configuration.register_rules(rules)
invoke_entrypoint('register_goals')
def build_config(cls): build_config = BuildConfiguration() build_config.register_aliases(cls.alias_groups()) build_config.register_rules(cls.rules()) return build_config
def setUp(self) -> None:
self.bc_builder = BuildConfiguration.Builder()
def build_config(cls):
build_config = BuildConfiguration.Builder()
build_config.register_aliases(cls.alias_groups())
build_config.register_rules(cls.rules())
build_config.register_target_types(cls.target_types())
return build_config.create()
def setUp(self):
self.build_configuration = BuildConfiguration()
class BuildConfigurationTest(unittest.TestCase):
def setUp(self):
self.build_configuration = BuildConfiguration()
def _register_aliases(self, **kwargs):
self.build_configuration.register_aliases(BuildFileAliases(**kwargs))
def test_register_bad(self):
with self.assertRaises(TypeError):
self.build_configuration.register_aliases(42)
def test_register_target_alias(self):
class Fred(Target):
pass
self._register_aliases(targets={'fred': Fred})
aliases = self.build_configuration.registered_aliases()
self.assertEqual({}, aliases.target_macro_factories)
self.assertEqual({}, aliases.objects)
self.assertEqual({}, aliases.context_aware_object_factories)
self.assertEqual(dict(fred=Fred), aliases.target_types)
with self._create_mock_build_file('fred') as build_file:
parse_state = self.build_configuration.initialize_parse_state(
build_file)
self.assertEqual(0, len(parse_state.objects))
self.assertEqual(1, len(parse_state.parse_globals))
target_call_proxy = parse_state.parse_globals['fred']
target_call_proxy(name='jake')
self.assertEqual(1, len(parse_state.objects))
target_proxy = parse_state.objects[0]
self.assertEqual('jake', target_proxy.addressed_name)
self.assertEqual(Fred, target_proxy.addressed_type)
def test_register_target_macro_facory(self):
class Fred(Target):
pass
class FredMacro(TargetMacro):
def __init__(self, parse_context):
self._parse_context = parse_context
def expand(self, *args, **kwargs):
return self._parse_context.create_object(
Fred, name='frog', dependencies=[kwargs['name']])
class FredFactory(TargetMacro.Factory):
@property
def target_types(self):
return {Fred}
def macro(self, parse_context):
return FredMacro(parse_context)
factory = FredFactory()
self._register_aliases(targets={'fred': factory})
aliases = self.build_configuration.registered_aliases()
self.assertEqual({}, aliases.target_types)
self.assertEqual({}, aliases.objects)
self.assertEqual({}, aliases.context_aware_object_factories)
self.assertEqual(dict(fred=factory), aliases.target_macro_factories)
with self._create_mock_build_file('fred') as build_file:
parse_state = self.build_configuration.initialize_parse_state(
build_file)
self.assertEqual(0, len(parse_state.objects))
self.assertEqual(1, len(parse_state.parse_globals))
target_call_proxy = parse_state.parse_globals['fred']
target_call_proxy(name='jake')
self.assertEqual(1, len(parse_state.objects))
target_proxy = parse_state.objects[0]
self.assertEqual('frog', target_proxy.addressed_name)
self.assertEqual(Fred, target_proxy.addressed_type)
self.assertEqual(['jake'], target_proxy.dependency_specs)
def test_register_exposed_object(self):
self._register_aliases(objects={'jane': 42})
aliases = self.build_configuration.registered_aliases()
self.assertEqual({}, aliases.target_types)
self.assertEqual({}, aliases.target_macro_factories)
self.assertEqual({}, aliases.context_aware_object_factories)
self.assertEqual(dict(jane=42), aliases.objects)
with self._create_mock_build_file('jane') as build_file:
parse_state = self.build_configuration.initialize_parse_state(
build_file)
self.assertEqual(0, len(parse_state.objects))
self.assertEqual(1, len(parse_state.parse_globals))
self.assertEqual(42, parse_state.parse_globals['jane'])
def test_register_exposed_context_aware_function(self):
self.do_test_exposed_context_aware_function(
lambda context: lambda: context.rel_path)
self.do_test_exposed_context_aware_function(
lambda context=None: lambda: context.rel_path)
def test_register_union_rules(self):
# Two calls to register_rules should merge relevant unions.
@union
class Base:
pass
class A:
pass
class B:
pass
self.build_configuration.register_rules([UnionRule(Base, A)])
self.build_configuration.register_rules([UnionRule(Base, B)])
self.assertEqual(set(self.build_configuration.union_rules()[Base]),
{A, B})
def george_method(self, parse_context):
return lambda: parse_context.rel_path
def test_register_exposed_context_aware_method(self):
self.do_test_exposed_context_aware_function(self.george_method)
@classmethod
def george_classmethod(cls, parse_context):
return lambda: parse_context.rel_path
def test_register_exposed_context_aware_classmethod(self):
self.do_test_exposed_context_aware_function(self.george_classmethod)
@staticmethod
def george_staticmethod(parse_context):
return lambda: parse_context.rel_path
def test_register_exposed_context_aware_staticmethod(self):
self.do_test_exposed_context_aware_function(self.george_staticmethod)
def do_test_exposed_context_aware_function(self, func, *args, **kwargs):
with self.do_test_exposed_context_aware_object(
func) as context_aware_object:
self.assertEqual('george', context_aware_object(*args, **kwargs))
def test_register_exposed_context_aware_class(self):
class George:
def __init__(self, parse_context):
self._parse_context = parse_context
def honorific(self):
return len(self._parse_context.rel_path)
with self.do_test_exposed_context_aware_object(
George) as context_aware_object:
self.assertEqual(6, context_aware_object.honorific())
@contextmanager
def do_test_exposed_context_aware_object(self,
context_aware_object_factory):
self._register_aliases(context_aware_object_factories={
'george': context_aware_object_factory
})
aliases = self.build_configuration.registered_aliases()
self.assertEqual({}, aliases.target_types)
self.assertEqual({}, aliases.target_macro_factories)
self.assertEqual({}, aliases.objects)
self.assertEqual(dict(george=context_aware_object_factory),
aliases.context_aware_object_factories)
with temporary_dir() as root:
build_file_path = os.path.join(root, 'george', 'BUILD')
touch(build_file_path)
build_file = BuildFile(FileSystemProjectTree(root), 'george/BUILD')
parse_state = self.build_configuration.initialize_parse_state(
build_file)
self.assertEqual(0, len(parse_state.objects))
self.assertEqual(1, len(parse_state.parse_globals))
yield parse_state.parse_globals['george']
@contextmanager
def _create_mock_build_file(self, dirname):
with temporary_dir() as root:
os.mkdir(os.path.join(root, dirname))
touch(os.path.join(root, dirname, 'BUILD'))
yield BuildFile(FileSystemProjectTree(root),
os.path.join(dirname, 'BUILD'))
def setup_legacy_graph_extended(
pants_ignore_patterns: List[str],
use_gitignore: bool,
local_store_dir: str,
local_execution_root_dir: str,
named_caches_dir: str,
build_file_prelude_globs: Tuple[str, ...],
options_bootstrapper: OptionsBootstrapper,
build_configuration: BuildConfiguration,
execution_options: ExecutionOptions,
build_root: Optional[str] = None,
native: Optional[Native] = None,
glob_match_error_behavior:
GlobMatchErrorBehavior = GlobMatchErrorBehavior.warn,
build_ignore_patterns=None,
exclude_target_regexps=None,
subproject_roots=None,
include_trace_on_error: bool = True,
) -> LegacyGraphScheduler:
"""Construct and return the components necessary for LegacyBuildGraph construction.
:param local_store_dir: The directory to use for storing the engine's LMDB store in.
:param local_execution_root_dir: The directory to use for local execution sandboxes.
:param named_caches_dir: The base directory for named cache storage.
:param build_file_prelude_globs: Globs to match files to be prepended to all BUILD files.
:param build_root: A path to be used as the build root. If None, then default is used.
:param native: An instance of the native-engine subsystem.
:param options_bootstrapper: A `OptionsBootstrapper` object containing bootstrap options.
:param build_configuration: The `BuildConfiguration` object to get build file aliases from.
:param glob_match_error_behavior: How to behave if a glob specified for a target's sources or
bundles does not expand to anything.
:param list build_ignore_patterns: A list of paths ignore patterns used when searching for BUILD
files, usually taken from the '--build-ignore' global option.
:param list exclude_target_regexps: A list of regular expressions for excluding targets.
:param list subproject_roots: Paths that correspond with embedded build roots
under the current build root.
:param include_trace_on_error: If True, when an error occurs, the error message will include
the graph trace.
:param execution_options: Option values for (remote) process execution.
"""
build_root = build_root or get_buildroot()
build_configuration = build_configuration or BuildConfigInitializer.get(
options_bootstrapper)
bootstrap_options = options_bootstrapper.bootstrap_options.for_global_scope(
)
build_file_aliases = build_configuration.registered_aliases()
rules = build_configuration.rules()
registered_target_types = RegisteredTargetTypes.create(
build_configuration.target_types())
symbol_table = _legacy_symbol_table(build_file_aliases,
registered_target_types)
execution_options = execution_options or DEFAULT_EXECUTION_OPTIONS
# Register "literal" subjects required for these rules.
parser = LegacyPythonCallbacksParser(symbol_table, build_file_aliases)
address_mapper = AddressMapper(
parser=parser,
prelude_glob_patterns=build_file_prelude_globs,
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_roots,
)
@rule
def glob_match_error_behavior_singleton() -> GlobMatchErrorBehavior:
return glob_match_error_behavior
@rule
def build_configuration_singleton() -> BuildConfiguration:
return build_configuration
@rule
def symbol_table_singleton() -> SymbolTable:
return symbol_table
@rule
def registered_target_types_singleton() -> RegisteredTargetTypes:
return registered_target_types
@rule
def union_membership_singleton() -> UnionMembership:
return UnionMembership(build_configuration.union_rules())
@rule
def build_root_singleton() -> BuildRoot:
return cast(BuildRoot, BuildRoot.instance)
# Create a Scheduler containing graph and filesystem rules, with no installed goals. The
# LegacyBuildGraph will explicitly request the products it needs.
rules = (
RootRule(Console),
glob_match_error_behavior_singleton,
build_configuration_singleton,
symbol_table_singleton,
registered_target_types_singleton,
union_membership_singleton,
build_root_singleton,
*interactive_runner.rules(),
*graph.rules(),
*options_parsing.rules(),
*process.rules(),
*target.rules(),
*create_legacy_graph_tasks(),
*create_fs_rules(),
*create_platform_rules(),
*create_graph_rules(address_mapper),
*structs_rules(),
*changed_rules(),
*binary_tool_rules(),
*binary_util_rules(),
*rules,
)
goal_map = EngineInitializer._make_goal_map_from_rules(rules)
union_rules = build_configuration.union_rules()
scheduler = Scheduler(
native=native,
ignore_patterns=pants_ignore_patterns,
use_gitignore=use_gitignore,
build_root=build_root,
local_store_dir=local_store_dir,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
rules=rules,
union_rules=union_rules,
execution_options=execution_options,
include_trace_on_error=include_trace_on_error,
visualize_to_dir=bootstrap_options.native_engine_visualize_to,
)
return LegacyGraphScheduler(scheduler, build_file_aliases, goal_map)
def setUp(self):
self.bc_builder = BuildConfiguration.Builder()
self.working_set = WorkingSet()
for entry in working_set.entries:
self.working_set.add_entry(entry)
def build_config(cls): build_config = BuildConfiguration() build_config.register_aliases(cls.alias_groups()) return build_config
def setUp(self):
super(BaseTestWithParser, self).setUp()
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration, self.build_root)
class LoaderTest(unittest.TestCase):
def setUp(self):
self.build_configuration = BuildConfiguration()
self.working_set = WorkingSet()
for entry in working_set.entries:
self.working_set.add_entry(entry)
def tearDown(self):
Goal.clear()
@contextmanager
def create_register(self, build_file_aliases=None, register_goals=None, global_subsystems=None,
module_name='register'):
package_name = b'__test_package_{0}'.format(uuid.uuid4().hex)
self.assertFalse(package_name in sys.modules)
package_module = types.ModuleType(package_name)
sys.modules[package_name] = package_module
try:
register_module_fqn = b'{0}.{1}'.format(package_name, module_name)
register_module = types.ModuleType(register_module_fqn)
setattr(package_module, module_name, register_module)
sys.modules[register_module_fqn] = register_module
def register_entrypoint(function_name, function):
if function:
setattr(register_module, function_name, function)
register_entrypoint('build_file_aliases', build_file_aliases)
register_entrypoint('global_subsystems', global_subsystems)
register_entrypoint('register_goals', register_goals)
yield package_name
finally:
del sys.modules[package_name]
def assert_empty_aliases(self):
registered_aliases = self.build_configuration.registered_aliases()
self.assertEqual(0, len(registered_aliases.target_types))
self.assertEqual(0, len(registered_aliases.target_macro_factories))
self.assertEqual(0, len(registered_aliases.objects))
self.assertEqual(0, len(registered_aliases.context_aware_object_factories))
self.assertEqual(self.build_configuration.subsystems(), set())
def test_load_valid_empty(self):
with self.create_register() as backend_package:
load_backend(self.build_configuration, backend_package)
self.assert_empty_aliases()
def test_load_valid_partial_aliases(self):
aliases = BuildFileAliases(targets={'bob': DummyTarget},
objects={'obj1': DummyObject1,
'obj2': DummyObject2})
with self.create_register(build_file_aliases=lambda: aliases) as backend_package:
load_backend(self.build_configuration, backend_package)
registered_aliases = self.build_configuration.registered_aliases()
self.assertEqual(DummyTarget, registered_aliases.target_types['bob'])
self.assertEqual(DummyObject1, registered_aliases.objects['obj1'])
self.assertEqual(DummyObject2, registered_aliases.objects['obj2'])
self.assertEqual(self.build_configuration.subsystems(), {DummySubsystem1, DummySubsystem2})
def test_load_valid_partial_goals(self):
def register_goals():
Goal.by_name('jack').install(TaskRegistrar('jill', DummyTask))
with self.create_register(register_goals=register_goals) as backend_package:
Goal.clear()
self.assertEqual(0, len(Goal.all()))
load_backend(self.build_configuration, backend_package)
self.assert_empty_aliases()
self.assertEqual(1, len(Goal.all()))
task_names = Goal.by_name('jack').ordered_task_names()
self.assertEqual(1, len(task_names))
task_name = task_names[0]
self.assertEqual('jill', task_name)
def test_load_invalid_entrypoint(self):
def build_file_aliases(bad_arg):
return BuildFileAliases()
with self.create_register(build_file_aliases=build_file_aliases) as backend_package:
with self.assertRaises(BuildConfigurationError):
load_backend(self.build_configuration, backend_package)
def test_load_invalid_module(self):
with self.create_register(module_name='register2') as backend_package:
with self.assertRaises(BuildConfigurationError):
load_backend(self.build_configuration, backend_package)
def test_load_missing_plugin(self):
with self.assertRaises(PluginNotFound):
self.load_plugins(['Foobar'])
def get_mock_plugin(self, name, version, reg=None, alias=None, after=None):
"""Make a fake Distribution (optionally with entry points)
Note the entry points do not actually point to code in the returned distribution --
the distribution does not even have a location and does not contain any code, just metadata.
A module is synthesized on the fly and installed into sys.modules under a random name.
If optional entry point callables are provided, those are added as methods to the module and
their name (foo/bar/baz in fake module) is added as the requested entry point to the mocked
metadata added to the returned dist.
:param string name: project_name for distribution (see pkg_resources)
:param string version: version for distribution (see pkg_resources)
:param callable reg: Optional callable for goal registration entry point
:param callable alias: Optional callable for build_file_aliases entry point
:param callable after: Optional callable for load_after list entry point
"""
plugin_pkg = b'demoplugin{0}'.format(uuid.uuid4().hex)
pkg = types.ModuleType(plugin_pkg)
sys.modules[plugin_pkg] = pkg
module_name = b'{0}.{1}'.format(plugin_pkg, 'demo')
plugin = types.ModuleType(module_name)
setattr(pkg, 'demo', plugin)
sys.modules[module_name] = plugin
metadata = {}
entry_lines = []
if reg is not None:
setattr(plugin, 'foo', reg)
entry_lines.append('register_goals = {}:foo\n'.format(module_name))
if alias is not None:
setattr(plugin, 'bar', alias)
entry_lines.append('build_file_aliases = {}:bar\n'.format(module_name))
if after is not None:
setattr(plugin, 'baz', after)
entry_lines.append('load_after = {}:baz\n'.format(module_name))
if entry_lines:
entry_data = '[pantsbuild.plugin]\n{}\n'.format('\n'.join(entry_lines))
metadata = {'entry_points.txt': entry_data}
return Distribution(project_name=name, version=version, metadata=MockMetadata(metadata))
def load_plugins(self, plugins):
load_plugins(self.build_configuration, plugins, self.working_set)
def test_plugin_load_and_order(self):
d1 = self.get_mock_plugin('demo1', '0.0.1', after=lambda: ['demo2'])
d2 = self.get_mock_plugin('demo2', '0.0.3')
self.working_set.add(d1)
# Attempting to load 'demo1' then 'demo2' should fail as 'demo1' requires 'after'=['demo2'].
with self.assertRaises(PluginLoadOrderError):
self.load_plugins(['demo1', 'demo2'])
# Attempting to load 'demo2' first should fail as it is not (yet) installed.
with self.assertRaises(PluginNotFound):
self.load_plugins(['demo2', 'demo1'])
# Installing demo2 and then loading in correct order should work though.
self.working_set.add(d2)
self.load_plugins(['demo2>=0.0.2', 'demo1'])
# But asking for a bad (not installed) version fails.
with self.assertRaises(VersionConflict):
self.load_plugins(['demo2>=0.0.5'])
def test_plugin_installs_goal(self):
def reg_goal():
Goal.by_name('plugindemo').install(TaskRegistrar('foo', DummyTask))
self.working_set.add(self.get_mock_plugin('regdemo', '0.0.1', reg=reg_goal))
# Start without the custom goal.
self.assertEqual(0, len(Goal.by_name('plugindemo').ordered_task_names()))
# Load plugin which registers custom goal.
self.load_plugins(['regdemo'])
# Now the custom goal exists.
self.assertEqual(1, len(Goal.by_name('plugindemo').ordered_task_names()))
self.assertEqual('foo', Goal.by_name('plugindemo').ordered_task_names()[0])
def test_plugin_installs_alias(self):
def reg_alias():
return BuildFileAliases(targets={'pluginalias': DummyTarget},
objects={'FROMPLUGIN1': DummyObject1,
'FROMPLUGIN2': DummyObject2})
self.working_set.add(self.get_mock_plugin('aliasdemo', '0.0.1', alias=reg_alias))
# Start with no aliases.
self.assert_empty_aliases()
# Now load the plugin which defines aliases.
self.load_plugins(['aliasdemo'])
# Aliases now exist.
registered_aliases = self.build_configuration.registered_aliases()
self.assertEqual(DummyTarget, registered_aliases.target_types['pluginalias'])
self.assertEqual(DummyObject1, registered_aliases.objects['FROMPLUGIN1'])
self.assertEqual(DummyObject2, registered_aliases.objects['FROMPLUGIN2'])
self.assertEqual(self.build_configuration.subsystems(), {DummySubsystem1, DummySubsystem2})
def test_subsystems(self):
def global_subsystems():
return {DummySubsystem1, DummySubsystem2}
with self.create_register(global_subsystems=global_subsystems) as backend_package:
load_backend(self.build_configuration, backend_package)
self.assertEqual(self.build_configuration.subsystems(),
{DummySubsystem1, DummySubsystem2})
def __init__(
self,
*,
rules: Iterable | None = None,
target_types: Iterable[type[Target]] | None = None,
objects: dict[str, Any] | None = None,
context_aware_object_factories: dict[str, Any] | None = None,
isolated_local_store: bool = False,
preserve_tmpdirs: bool = False,
ca_certs_path: str | None = None,
bootstrap_args: Iterable[str] = (),
) -> None:
bootstrap_args = [*bootstrap_args]
root_dir: Path | None = None
if preserve_tmpdirs:
root_dir = Path(mkdtemp(prefix="RuleRunner."))
print(
f"Preserving rule runner temporary directories at {root_dir}.",
file=sys.stderr)
bootstrap_args.extend([
"--no-process-execution-local-cleanup",
f"--local-execution-root-dir={root_dir}",
])
build_root = (root_dir / "BUILD_ROOT").resolve()
build_root.mkdir()
self.build_root = str(build_root)
else:
self.build_root = os.path.realpath(
safe_mkdtemp(prefix="_BUILD_ROOT"))
safe_mkdir(self.pants_workdir)
BuildRoot().path = self.build_root
# TODO: Redesign rule registration for tests to be more ergonomic and to make this less
# special-cased.
all_rules = (
*(rules or ()),
*source_root.rules(),
QueryRule(WrappedTarget, [Address]),
QueryRule(UnionMembership, []),
)
build_config_builder = BuildConfiguration.Builder()
build_config_builder.register_aliases(
BuildFileAliases(
objects=objects,
context_aware_object_factories=context_aware_object_factories))
build_config_builder.register_rules("_dummy_for_test_", all_rules)
build_config_builder.register_target_types("_dummy_for_test_",
target_types or ())
self.build_config = build_config_builder.create()
self.environment = CompleteEnvironment({})
self.options_bootstrapper = create_options_bootstrapper(
args=bootstrap_args)
options = self.options_bootstrapper.full_options(self.build_config)
global_options = self.options_bootstrapper.bootstrap_options.for_global_scope(
)
dynamic_remote_options, _ = DynamicRemoteOptions.from_options(
options, self.environment)
local_store_options = LocalStoreOptions.from_options(global_options)
if isolated_local_store:
if root_dir:
lmdb_store_dir = root_dir / "lmdb_store"
lmdb_store_dir.mkdir()
store_dir = str(lmdb_store_dir)
else:
store_dir = safe_mkdtemp(prefix="lmdb_store.")
local_store_options = dataclasses.replace(local_store_options,
store_dir=store_dir)
local_execution_root_dir = global_options.local_execution_root_dir
named_caches_dir = global_options.named_caches_dir
graph_session = EngineInitializer.setup_graph_extended(
pants_ignore_patterns=GlobalOptions.compute_pants_ignore(
self.build_root, global_options),
use_gitignore=False,
local_store_options=local_store_options,
local_execution_root_dir=local_execution_root_dir,
named_caches_dir=named_caches_dir,
build_root=self.build_root,
build_configuration=self.build_config,
executor=_EXECUTOR,
execution_options=ExecutionOptions.from_options(
global_options, dynamic_remote_options),
ca_certs_path=ca_certs_path,
engine_visualize_to=None,
).new_session(
build_id="buildid_for_test",
session_values=SessionValues({
OptionsBootstrapper: self.options_bootstrapper,
CompleteEnvironment: self.environment,
}),
)
self.scheduler = graph_session.scheduler_session
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot.
:API: public
"""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path.
:API: public
"""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
:API: public
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_workdir_dir(self, relpath):
"""Creates a directory under the work directory.
:API: public
relpath: The relative path to the directory from the work directory.
"""
path = os.path.join(self.pants_workdir, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the buildroot.
:API: public
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def create_workdir_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the work directory.
:API: public
relpath: The relative path to the file from the work directory.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.pants_workdir, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
:API: public
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
return BuildFile(self.address_mapper._project_tree, relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
derived_from=None,
synthetic=False,
**kwargs):
"""Creates a target and injects it into the test's build graph.
:API: public
:param string spec: The target address spec that locates this target.
:param type target_type: The concrete target subclass to create this new target from.
:param list dependencies: A list of target instances this new target depends on.
:param derived_from: The target this new target was derived from.
:type derived_from: :class:`pants.build_graph.target.Target`
"""
address = Address.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.apply_injectables([target])
self.build_graph.inject_target(target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from,
synthetic=synthetic)
# TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
# Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
traversables = [target.compute_dependency_specs(payload=target.payload)]
for dependency_spec in itertools.chain(*traversables):
dependency_address = Address.parse(dependency_spec, relative_to=address.spec_path)
dependency_target = self.build_graph.get_target(dependency_address)
if not dependency_target:
raise ValueError('Tests must make targets for dependency specs ahead of them '
'being traversed, {} tried to traverse {} which does not exist.'
.format(target, dependency_address))
if dependency_target not in target.dependencies:
self.build_graph.inject_dependency(dependent=target.address,
dependency=dependency_address)
target.mark_transitive_invalidation_hash_dirty()
return target
@property
def alias_groups(self):
"""
:API: public
"""
return BuildFileAliases(targets={'target': Target})
@property
def build_ignore_patterns(self):
"""
:API: public
"""
return None
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
# Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup.
clean_global_runtime_state(reset_subsystem=True)
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.subprocess_dir = os.path.join(self.build_root, '.pids')
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'pants_subprocessdir': self.subprocess_dir,
'cache_key_gen_version': '0-test',
}
self.options['cache'] = {
'read_from': [],
'write_to': [],
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.reset_build_graph()
def buildroot_files(self, relpath=None):
"""Returns the set of all files under the test build root.
:API: public
:param string relpath: If supplied, only collect files from this subtree.
:returns: All file paths found.
:rtype: set
"""
def scan():
for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or '')):
for f in files:
yield os.path.relpath(os.path.join(root, f), self.build_root)
return set(scan())
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree,
build_ignore_patterns=self.build_ignore_patterns)
self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
def set_options_for_scope(self, scope, **kwargs):
self.options[scope].update(kwargs)
def context(self, for_task_types=None, for_subsystems=None, options=None,
target_roots=None, console_outstream=None, workspace=None,
**kwargs):
"""
:API: public
:param dict **kwargs: keyword arguments passed in to `create_options_for_optionables`.
"""
# Many tests use source root functionality via the SourceRootConfig.global_instance().
# (typically accessed via Target.target_base), so we always set it up, for convenience.
optionables = {SourceRootConfig}
extra_scopes = set()
for_subsystems = for_subsystems or ()
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError('You must set a scope on your subsystem type before using it in tests.')
optionables.add(subsystem)
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError('You must set a scope on your task type before using it in tests.')
optionables.add(task_type)
# If task is expected to inherit goal-level options, register those directly on the task,
# by subclassing the goal options registrar and settings its scope to the task scope.
if issubclass(task_type, GoalOptionsMixin):
subclass_name = b'test_{}_{}_{}'.format(
task_type.__name__, task_type.goal_options_registrar_cls.options_scope,
task_type.options_scope)
optionables.add(type(subclass_name, (task_type.goal_options_registrar_cls, ),
{b'options_scope': task_type.options_scope}))
extra_scopes.update([si.scope for si in task_type.known_scope_infos()])
optionables.update(Subsystem.closure(
set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) |
self._build_configuration.subsystems()))
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
fake_options = create_options_for_optionables(
optionables, extra_scopes=extra_scopes, options=options, **kwargs)
Subsystem.reset(reset_options=True)
Subsystem.set_options(fake_options)
context = create_context_from_options(fake_options,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
return context
def tearDown(self):
"""
:API: public
"""
super(BaseTest, self).tearDown()
BuildFile.clear_cache()
Subsystem.reset()
def target(self, spec):
"""Resolves the given target address to a Target object.
:API: public
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
address = Address.parse(spec)
self.build_graph.inject_address_closure(address)
return self.build_graph.get_target(address)
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
:API: public
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec = CmdLineSpecParser(self.build_root).parse_spec(spec)
addresses = list(self.address_mapper.scan_specs([spec]))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [self.build_graph.get_target(address) for address in addresses]
return targets
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
:API: public
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
:API: public
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(path, dedent('''
%(target_type)s(name='%(name)s',
%(sources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
''' % dict(target_type=target_type,
name=name,
sources=('sources=%s,' % repr(sources)
if sources else ''),
java_sources=('java_sources=[%s],'
% ','.join(map(lambda str_target: '"%s"' % str_target,
kwargs.get('java_sources')))
if 'java_sources' in kwargs else ''),
provides=('provides=%s,' % kwargs.get('provides')
if 'provides' in kwargs else ''),
dependencies=('dependencies=%s,' % kwargs.get('dependencies')
if 'dependencies' in kwargs else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
"""
:API: public
"""
return self.create_library(path, 'resources', name, sources)
def assertUnorderedPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, unordered.
:API: public
"""
actual = list(itertools.islice(actual_iter, len(expected)))
self.assertEqual(sorted(expected), sorted(actual))
def assertPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, in order.
:API: public
"""
self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected))))
def assertInFile(self, string, file_path):
"""Verifies that a string appears in a file
:API: public
"""
with open(file_path) as f:
content = f.read()
self.assertIn(string, content, '"{}" is not in the file {}:\n{}'.format(string, f.name, content))
def get_bootstrap_options(self, cli_options=()):
"""Retrieves bootstrap options.
:param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`.
"""
# Can't parse any options without a pants.ini.
self.create_file('pants.ini')
return OptionsBootstrapper(args=cli_options).get_bootstrap_options().for_global_scope()
class LoggingRecorder(object):
"""Simple logging handler to record warnings."""
def __init__(self):
self._records = []
self.level = logging.DEBUG
def handle(self, record):
self._records.append(record)
def _messages_for_level(self, levelname):
return ['{}: {}'.format(record.name, record.getMessage())
for record in self._records if record.levelname == levelname]
def infos(self):
return self._messages_for_level('INFO')
def warnings(self):
return self._messages_for_level('WARNING')
@contextmanager
def captured_logging(self, level=None):
root_logger = logging.getLogger()
old_level = root_logger.level
root_logger.setLevel(level or logging.NOTSET)
handler = self.LoggingRecorder()
root_logger.addHandler(handler)
try:
yield handler
finally:
root_logger.setLevel(old_level)
root_logger.removeHandler(handler)
def setUp(self):
self.build_configuration = BuildConfiguration()
self.working_set = WorkingSet()
for entry in working_set.entries:
self.working_set.add_entry(entry)
class BuildConfigurationTest(unittest.TestCase):
def setUp(self):
self.build_configuration = BuildConfiguration()
def _register_aliases(self, **kwargs):
self.build_configuration.register_aliases(BuildFileAliases(**kwargs))
def test_register_bad(self):
with self.assertRaises(TypeError):
self.build_configuration.register_aliases(42)
def test_register_target_alias(self):
class Fred(Target):
pass
self._register_aliases(targets={'fred': Fred})
aliases = self.build_configuration.registered_aliases()
self.assertEqual({}, aliases.target_macro_factories)
self.assertEqual({}, aliases.objects)
self.assertEqual({}, aliases.context_aware_object_factories)
self.assertEqual(dict(fred=Fred), aliases.target_types)
with self._create_mock_build_file('fred') as build_file:
parse_state = self.build_configuration.initialize_parse_state(build_file)
self.assertEqual(0, len(parse_state.objects))
self.assertEqual(1, len(parse_state.parse_globals))
target_call_proxy = parse_state.parse_globals['fred']
target_call_proxy(name='jake')
self.assertEqual(1, len(parse_state.objects))
target_proxy = parse_state.objects[0]
self.assertEqual('jake', target_proxy.addressed_name)
self.assertEqual(Fred, target_proxy.addressed_type)
def test_register_target_macro_facory(self):
class Fred(Target):
pass
class FredMacro(TargetMacro):
def __init__(self, parse_context):
self._parse_context = parse_context
def expand(self, *args, **kwargs):
return self._parse_context.create_object(Fred, name='frog', dependencies=[kwargs['name']])
class FredFactory(TargetMacro.Factory):
@property
def target_types(self):
return {Fred}
def macro(self, parse_context):
return FredMacro(parse_context)
factory = FredFactory()
self._register_aliases(targets={'fred': factory})
aliases = self.build_configuration.registered_aliases()
self.assertEqual({}, aliases.target_types)
self.assertEqual({}, aliases.objects)
self.assertEqual({}, aliases.context_aware_object_factories)
self.assertEqual(dict(fred=factory), aliases.target_macro_factories)
with self._create_mock_build_file('fred') as build_file:
parse_state = self.build_configuration.initialize_parse_state(build_file)
self.assertEqual(0, len(parse_state.objects))
self.assertEqual(1, len(parse_state.parse_globals))
target_call_proxy = parse_state.parse_globals['fred']
target_call_proxy(name='jake')
self.assertEqual(1, len(parse_state.objects))
target_proxy = parse_state.objects[0]
self.assertEqual('frog', target_proxy.addressed_name)
self.assertEqual(Fred, target_proxy.addressed_type)
self.assertEqual(['jake'], target_proxy.dependency_specs)
def test_register_exposed_object(self):
self._register_aliases(objects={'jane': 42})
aliases = self.build_configuration.registered_aliases()
self.assertEqual({}, aliases.target_types)
self.assertEqual({}, aliases.target_macro_factories)
self.assertEqual({}, aliases.context_aware_object_factories)
self.assertEqual(dict(jane=42), aliases.objects)
with self._create_mock_build_file('jane') as build_file:
parse_state = self.build_configuration.initialize_parse_state(build_file)
self.assertEqual(0, len(parse_state.objects))
self.assertEqual(1, len(parse_state.parse_globals))
self.assertEqual(42, parse_state.parse_globals['jane'])
def test_register_exposed_context_aware_function(self):
self.do_test_exposed_context_aware_function(lambda context: lambda: context.rel_path)
self.do_test_exposed_context_aware_function(lambda context=None: lambda: context.rel_path)
def george_method(self, parse_context):
return lambda: parse_context.rel_path
def test_register_exposed_context_aware_method(self):
self.do_test_exposed_context_aware_function(self.george_method)
@classmethod
def george_classmethod(cls, parse_context):
return lambda: parse_context.rel_path
def test_register_exposed_context_aware_classmethod(self):
self.do_test_exposed_context_aware_function(self.george_classmethod)
@staticmethod
def george_staticmethod(parse_context):
return lambda: parse_context.rel_path
def test_register_exposed_context_aware_staticmethod(self):
self.do_test_exposed_context_aware_function(self.george_staticmethod)
def do_test_exposed_context_aware_function(self, func, *args, **kwargs):
with self.do_test_exposed_context_aware_object(func) as context_aware_object:
self.assertEqual('george', context_aware_object(*args, **kwargs))
def test_register_exposed_context_aware_class(self):
class George(object):
def __init__(self, parse_context):
self._parse_context = parse_context
def honorific(self):
return len(self._parse_context.rel_path)
with self.do_test_exposed_context_aware_object(George) as context_aware_object:
self.assertEqual(6, context_aware_object.honorific())
@contextmanager
def do_test_exposed_context_aware_object(self, context_aware_object_factory):
self._register_aliases(context_aware_object_factories={'george': context_aware_object_factory})
aliases = self.build_configuration.registered_aliases()
self.assertEqual({}, aliases.target_types)
self.assertEqual({}, aliases.target_macro_factories)
self.assertEqual({}, aliases.objects)
self.assertEqual(dict(george=context_aware_object_factory),
aliases.context_aware_object_factories)
with temporary_dir() as root:
build_file_path = os.path.join(root, 'george', 'BUILD')
touch(build_file_path)
build_file = BuildFile(FileSystemProjectTree(root), 'george/BUILD')
parse_state = self.build_configuration.initialize_parse_state(build_file)
self.assertEqual(0, len(parse_state.objects))
self.assertEqual(1, len(parse_state.parse_globals))
yield parse_state.parse_globals['george']
@contextmanager
def _create_mock_build_file(self, dirname):
with temporary_dir() as root:
os.mkdir(os.path.join(root, dirname))
touch(os.path.join(root, dirname, 'BUILD'))
yield BuildFile(FileSystemProjectTree(root), os.path.join(dirname, 'BUILD'))
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot.
:API: public
"""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path.
:API: public
"""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
:API: public
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_workdir_dir(self, relpath):
"""Creates a directory under the work directory.
:API: public
relpath: The relative path to the directory from the work directory.
"""
path = os.path.join(self.pants_workdir, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the buildroot.
:API: public
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def create_workdir_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the work directory.
:API: public
relpath: The relative path to the file from the work directory.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.pants_workdir, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
:API: public
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
return BuildFile(self.address_mapper._project_tree,
relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
derived_from=None,
synthetic=False,
**kwargs):
"""Creates a target and injects it into the test's build graph.
:API: public
:param string spec: The target address spec that locates this target.
:param type target_type: The concrete target subclass to create this new target from.
:param list dependencies: A list of target instances this new target depends on.
:param derived_from: The target this new target was derived from.
:type derived_from: :class:`pants.build_graph.target.Target`
"""
address = Address.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.apply_injectables([target])
self.build_graph.inject_target(
target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from,
synthetic=synthetic)
# TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
# Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
traversables = [
target.compute_dependency_specs(payload=target.payload)
]
for dependency_spec in itertools.chain(*traversables):
dependency_address = Address.parse(dependency_spec,
relative_to=address.spec_path)
dependency_target = self.build_graph.get_target(dependency_address)
if not dependency_target:
raise ValueError(
'Tests must make targets for dependency specs ahead of them '
'being traversed, {} tried to traverse {} which does not exist.'
.format(target, dependency_address))
if dependency_target not in target.dependencies:
self.build_graph.inject_dependency(
dependent=target.address, dependency=dependency_address)
target.mark_transitive_invalidation_hash_dirty()
return target
@property
def alias_groups(self):
"""
:API: public
"""
return BuildFileAliases(targets={'target': Target})
@property
def build_ignore_patterns(self):
"""
:API: public
"""
return None
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
# Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup.
clean_global_runtime_state(reset_subsystem=True)
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.subprocess_dir = os.path.join(self.build_root, '.pids')
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[GLOBAL_SCOPE] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'pants_subprocessdir': self.subprocess_dir,
'cache_key_gen_version': '0-test',
}
self.options['cache'] = {
'read_from': [],
'write_to': [],
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration,
self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.reset_build_graph()
def buildroot_files(self, relpath=None):
"""Returns the set of all files under the test build root.
:API: public
:param string relpath: If supplied, only collect files from this subtree.
:returns: All file paths found.
:rtype: set
"""
def scan():
for root, dirs, files in os.walk(
os.path.join(self.build_root, relpath or '')):
for f in files:
yield os.path.relpath(os.path.join(root, f),
self.build_root)
return set(scan())
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(
self.build_file_parser,
self.project_tree,
build_ignore_patterns=self.build_ignore_patterns)
self.build_graph = MutableBuildGraph(
address_mapper=self.address_mapper)
def set_options_for_scope(self, scope, **kwargs):
self.options[scope].update(kwargs)
def context(self,
for_task_types=None,
for_subsystems=None,
options=None,
target_roots=None,
console_outstream=None,
workspace=None,
scheduler=None,
**kwargs):
"""
:API: public
:param dict **kwargs: keyword arguments passed in to `create_options_for_optionables`.
"""
# Many tests use source root functionality via the SourceRootConfig.global_instance().
# (typically accessed via Target.target_base), so we always set it up, for convenience.
for_subsystems = set(for_subsystems or ())
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError(
'You must set a scope on your subsystem type before using it in tests.'
)
optionables = {
SourceRootConfig
} | self._build_configuration.subsystems() | for_subsystems
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError(
'You must set a scope on your task type before using it in tests.'
)
optionables.add(task_type)
# If task is expected to inherit goal-level options, register those directly on the task,
# by subclassing the goal options registrar and settings its scope to the task scope.
if issubclass(task_type, GoalOptionsMixin):
subclass_name = b'test_{}_{}_{}'.format(
task_type.__name__,
task_type.goal_options_registrar_cls.options_scope,
task_type.options_scope)
optionables.add(
type(subclass_name,
(task_type.goal_options_registrar_cls, ),
{b'options_scope': task_type.options_scope}))
# Now expand to all deps.
all_optionables = set()
for optionable in optionables:
all_optionables.update(si.optionable_cls
for si in optionable.known_scope_infos())
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
fake_options = create_options_for_optionables(all_optionables,
options=options,
**kwargs)
Subsystem.reset(reset_options=True)
Subsystem.set_options(fake_options)
context = create_context_from_options(
fake_options,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace,
scheduler=scheduler)
return context
def tearDown(self):
"""
:API: public
"""
super(BaseTest, self).tearDown()
BuildFile.clear_cache()
Subsystem.reset()
def target(self, spec):
"""Resolves the given target address to a Target object.
:API: public
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
address = Address.parse(spec)
self.build_graph.inject_address_closure(address)
return self.build_graph.get_target(address)
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
:API: public
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec = CmdLineSpecParser(self.build_root).parse_spec(spec)
addresses = list(self.address_mapper.scan_specs([spec]))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [
self.build_graph.get_target(address) for address in addresses
]
return targets
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
:API: public
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
:API: public
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(
path,
dedent('''
%(target_type)s(name='%(name)s',
%(sources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
''' % dict(
target_type=target_type,
name=name,
sources=('sources=%s,' % repr(sources) if sources else ''),
java_sources=('java_sources=[%s],' % ','.join(
map(lambda str_target: '"%s"' % str_target,
kwargs.get('java_sources')))
if 'java_sources' in kwargs else ''),
provides=('provides=%s,' % kwargs.get('provides')
if 'provides' in kwargs else ''),
dependencies=('dependencies=%s,' % kwargs.get('dependencies')
if 'dependencies' in kwargs else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
"""
:API: public
"""
return self.create_library(path, 'resources', name, sources)
def assertUnorderedPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, unordered.
:API: public
"""
actual = list(itertools.islice(actual_iter, len(expected)))
self.assertEqual(sorted(expected), sorted(actual))
def assertPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, in order.
:API: public
"""
self.assertEqual(expected,
list(itertools.islice(actual_iter, len(expected))))
def assertInFile(self, string, file_path):
"""Verifies that a string appears in a file
:API: public
"""
with open(file_path) as f:
content = f.read()
self.assertIn(
string, content, '"{}" is not in the file {}:\n{}'.format(
string, f.name, content))
def get_bootstrap_options(self, cli_options=()):
"""Retrieves bootstrap options.
:param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`.
"""
# Can't parse any options without a pants.ini.
self.create_file('pants.ini')
return OptionsBootstrapper(
args=cli_options).get_bootstrap_options().for_global_scope()
class LoggingRecorder(object):
"""Simple logging handler to record warnings."""
def __init__(self):
self._records = []
self.level = logging.DEBUG
def handle(self, record):
self._records.append(record)
def _messages_for_level(self, levelname):
return [
'{}: {}'.format(record.name, record.getMessage())
for record in self._records if record.levelname == levelname
]
def infos(self):
return self._messages_for_level('INFO')
def warnings(self):
return self._messages_for_level('WARNING')
@contextmanager
def captured_logging(self, level=None):
root_logger = logging.getLogger()
old_level = root_logger.level
root_logger.setLevel(level or logging.NOTSET)
handler = self.LoggingRecorder()
root_logger.addHandler(handler)
try:
yield handler
finally:
root_logger.setLevel(old_level)
root_logger.removeHandler(handler)
def setUp(self): self.build_configuration = BuildConfiguration()
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot."""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path."""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_workdir_dir(self, relpath):
"""Creates a directory under the work directory.
relpath: The relative path to the directory from the work directory.
"""
path = os.path.join(self.pants_workdir, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def create_workdir_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the work directory.
relpath: The relative path to the file from the work directory.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.pants_workdir, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
cls = self.address_mapper._build_file_type
return cls(root_dir=self.build_root, relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
derived_from=None,
**kwargs):
"""Creates a target and injects it into the test's build graph.
:param string spec: The target address spec that locates this target.
:param type target_type: The concrete target subclass to create this new target from.
:param list dependencies: A list of target instances this new target depends on.
:param derived_from: The target this new target was derived from.
:type derived_from: :class:`pants.build_graph.target.Target`
"""
address = Address.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.inject_target(target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from)
# TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
# Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
for traversable_dependency_spec in target.traversable_dependency_specs:
traversable_dependency_address = Address.parse(traversable_dependency_spec,
relative_to=address.spec_path)
traversable_dependency_target = self.build_graph.get_target(traversable_dependency_address)
if not traversable_dependency_target:
raise ValueError('Tests must make targets for traversable dependency specs ahead of them '
'being traversed, {} tried to traverse {} which does not exist.'
.format(target, traversable_dependency_address))
if traversable_dependency_target not in target.dependencies:
self.build_graph.inject_dependency(dependent=target.address,
dependency=traversable_dependency_address)
target.mark_transitive_invalidation_hash_dirty()
return target
@property
def alias_groups(self):
return BuildFileAliases(targets={'target': Target})
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'cache_key_gen_version': '0-test',
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
# We need a pants.ini, even if empty. get_buildroot() uses its presence.
self.create_file('pants.ini')
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def buildroot_files(self, relpath=None):
"""Returns the set of all files under the test build root.
:param string relpath: If supplied, only collect files from this subtree.
:returns: All file paths found.
:rtype: set
"""
def scan():
for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or '')):
for f in files:
yield os.path.relpath(os.path.join(root, f), self.build_root)
return set(scan())
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def set_options_for_scope(self, scope, **kwargs):
self.options[scope].update(kwargs)
def context(self, for_task_types=None, options=None, passthru_args=None, target_roots=None,
console_outstream=None, workspace=None, for_subsystems=None):
# Many tests use source root functionality via the SourceRootConfig.global_instance()
# (typically accessed via Target.target_base), so we always set it up, for convenience.
optionables = {SourceRootConfig}
extra_scopes = set()
for_subsystems = for_subsystems or ()
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError('You must set a scope on your subsystem type before using it in tests.')
optionables.add(subsystem)
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError('You must set a scope on your task type before using it in tests.')
optionables.add(task_type)
extra_scopes.update([si.scope for si in task_type.known_scope_infos()])
optionables.update(Subsystem.closure(
set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) |
self._build_configuration.subsystems()))
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
options = create_options_for_optionables(optionables,
extra_scopes=extra_scopes,
options=options)
Subsystem._options = options
context = create_context(options=options,
passthru_args=passthru_args,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
return context
def tearDown(self):
super(BaseTest, self).tearDown()
FilesystemBuildFile.clear_cache()
Subsystem.reset()
def target(self, spec):
"""Resolves the given target address to a Target object.
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
address = Address.parse(spec)
self.build_graph.inject_address_closure(address)
return self.build_graph.get_target(address)
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
addresses = list(spec_parser.parse_addresses(spec))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [self.build_graph.get_target(address) for address in addresses]
return targets
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(path, dedent('''
%(target_type)s(name='%(name)s',
%(sources)s
%(resources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
''' % dict(target_type=target_type,
name=name,
sources=('sources=%s,' % repr(sources)
if sources else ''),
resources=('resources=["%s"],' % kwargs.get('resources')
if 'resources' in kwargs else ''),
java_sources=('java_sources=[%s],'
% ','.join(map(lambda str_target: '"%s"' % str_target,
kwargs.get('java_sources')))
if 'java_sources' in kwargs else ''),
provides=('provides=%s,' % kwargs.get('provides')
if 'provides' in kwargs else ''),
dependencies=('dependencies=%s,' % kwargs.get('dependencies')
if 'dependencies' in kwargs else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
return self.create_library(path, 'resources', name, sources)
def assertUnorderedPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, unordered."""
actual = list(itertools.islice(actual_iter, len(expected)))
self.assertEqual(sorted(expected), sorted(actual))
def assertPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, in order."""
self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected))))
class BuildConfigurationTest(unittest.TestCase):
def setUp(self):
self.build_configuration = BuildConfiguration()
def _register_aliases(self, **kwargs):
self.build_configuration.register_aliases(BuildFileAliases(**kwargs))
def test_register_bad(self):
with self.assertRaises(TypeError):
self.build_configuration.register_aliases(42)
def test_register_target_alias(self):
class Fred(Target):
pass
self._register_aliases(targets={'fred': Fred})
aliases = self.build_configuration.registered_aliases()
self.assertEqual({}, aliases.target_macro_factories)
self.assertEqual({}, aliases.objects)
self.assertEqual({}, aliases.context_aware_object_factories)
self.assertEqual(dict(fred=Fred), aliases.target_types)
build_file = FilesystemBuildFile('/tmp', 'fred', must_exist=False)
parse_state = self.build_configuration.initialize_parse_state(
build_file)
self.assertEqual(0, len(parse_state.registered_addressable_instances))
self.assertEqual(1, len(parse_state.parse_globals))
target_call_proxy = parse_state.parse_globals['fred']
target_call_proxy(name='jake')
self.assertEqual(1, len(parse_state.registered_addressable_instances))
name, target_proxy = parse_state.registered_addressable_instances.pop()
self.assertEqual('jake', target_proxy.addressed_name)
self.assertEqual(Fred, target_proxy.addressed_type)
def test_register_target_macro_facory(self):
class Fred(Target):
pass
class FredMacro(TargetMacro):
def __init__(self, parse_context):
self._parse_context = parse_context
def expand(self, *args, **kwargs):
return self._parse_context.create_object(
Fred, name='frog', dependencies=[kwargs['name']])
class FredFactory(TargetMacro.Factory):
@property
def target_types(self):
return {Fred}
def macro(self, parse_context):
return FredMacro(parse_context)
factory = FredFactory()
self._register_aliases(targets={'fred': factory})
aliases = self.build_configuration.registered_aliases()
self.assertEqual({}, aliases.target_types)
self.assertEqual({}, aliases.objects)
self.assertEqual({}, aliases.context_aware_object_factories)
self.assertEqual(dict(fred=factory), aliases.target_macro_factories)
build_file = FilesystemBuildFile('/tmp', 'fred', must_exist=False)
parse_state = self.build_configuration.initialize_parse_state(
build_file)
self.assertEqual(0, len(parse_state.registered_addressable_instances))
self.assertEqual(1, len(parse_state.parse_globals))
target_call_proxy = parse_state.parse_globals['fred']
target_call_proxy(name='jake')
self.assertEqual(1, len(parse_state.registered_addressable_instances))
name, target_proxy = parse_state.registered_addressable_instances.pop()
self.assertEqual('frog', target_proxy.addressed_name)
self.assertEqual(Fred, target_proxy.addressed_type)
self.assertEqual(['jake'], target_proxy.dependency_specs)
def test_register_exposed_object(self):
self._register_aliases(objects={'jane': 42})
aliases = self.build_configuration.registered_aliases()
self.assertEqual({}, aliases.target_types)
self.assertEqual({}, aliases.target_macro_factories)
self.assertEqual({}, aliases.context_aware_object_factories)
self.assertEqual(dict(jane=42), aliases.objects)
build_file = FilesystemBuildFile('/tmp', 'jane', must_exist=False)
parse_state = self.build_configuration.initialize_parse_state(
build_file)
self.assertEqual(0, len(parse_state.registered_addressable_instances))
self.assertEqual(1, len(parse_state.parse_globals))
self.assertEqual(42, parse_state.parse_globals['jane'])
def test_register_exposed_context_aware_function(self):
self.do_test_exposed_context_aware_function(
lambda context: lambda: context.rel_path)
self.do_test_exposed_context_aware_function(
lambda context=None: lambda: context.rel_path)
def george_method(self, parse_context):
return lambda: parse_context.rel_path
def test_register_exposed_context_aware_method(self):
self.do_test_exposed_context_aware_function(self.george_method)
@classmethod
def george_classmethod(cls, parse_context):
return lambda: parse_context.rel_path
def test_register_exposed_context_aware_classmethod(self):
self.do_test_exposed_context_aware_function(self.george_classmethod)
@staticmethod
def george_staticmethod(parse_context):
return lambda: parse_context.rel_path
def test_register_exposed_context_aware_staticmethod(self):
self.do_test_exposed_context_aware_function(self.george_staticmethod)
def do_test_exposed_context_aware_function(self, func, *args, **kwargs):
with self.do_test_exposed_context_aware_object(
func) as context_aware_object:
self.assertEqual('george', context_aware_object(*args, **kwargs))
def test_register_exposed_context_aware_class(self):
class George(object):
def __init__(self, parse_context):
self._parse_context = parse_context
def honorific(self):
return len(self._parse_context.rel_path)
with self.do_test_exposed_context_aware_object(
George) as context_aware_object:
self.assertEqual(6, context_aware_object.honorific())
@contextmanager
def do_test_exposed_context_aware_object(self,
context_aware_object_factory):
self._register_aliases(context_aware_object_factories={
'george': context_aware_object_factory
})
aliases = self.build_configuration.registered_aliases()
self.assertEqual({}, aliases.target_types)
self.assertEqual({}, aliases.target_macro_factories)
self.assertEqual({}, aliases.objects)
self.assertEqual(dict(george=context_aware_object_factory),
aliases.context_aware_object_factories)
with temporary_dir() as root:
build_file_path = os.path.join(root, 'george', 'BUILD')
touch(build_file_path)
build_file = FilesystemBuildFile(root, 'george')
parse_state = self.build_configuration.initialize_parse_state(
build_file)
self.assertEqual(0,
len(parse_state.registered_addressable_instances))
self.assertEqual(1, len(parse_state.parse_globals))
yield parse_state.parse_globals['george']
