def create_python_subsystems(self, setup_options=None, repos_options=None):
Subsystem.reset(reset_options=True)
def create_subsystem(subsystem_type, options=None):
return global_subsystem_instance(subsystem_type,
options={subsystem_type.options_scope: options or {}})
return (create_subsystem(PythonSetup, setup_options),
create_subsystem(PythonRepos, repos_options))
def create(cls, options_bootstrapper, build_configuration, init_subsystems=True):
global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'
.format(global_bootstrap_options.pants_version, pants_version())
)
pants_runtime_python_version = global_bootstrap_options.pants_runtime_python_version
current_python_version = '.'.join(map(str, sys.version_info[0:2]))
if pants_runtime_python_version and pants_runtime_python_version != current_python_version:
raise BuildConfigurationError(
'Running Pants with a different Python interpreter version than requested. '
'You requested {}, but are running with {}.\n\n'
'Note that Pants cannot use the value you give for `--pants-runtime-python-version` to '
'dynamically change the interpreter it uses, as it is too late for it to change once the program '
'is already running. Instead, your setup script (e.g. `./pants`) must configure which Python '
'interpreter and virtualenv to use. For example, the setup script we distribute '
'at https://www.pantsbuild.org/install.html#recommended-installation will read the '
'`pants_runtime_python_version` defined in your pants.ini to determine which Python '
'version to run with.'.format(pants_runtime_python_version, current_python_version)
)
# Parse and register options.
options = cls._construct_options(options_bootstrapper, build_configuration)
GlobalOptionsRegistrar.validate_instance(options.for_global_scope())
if init_subsystems:
Subsystem.set_options(options)
return options
def test_closure_cycle(self):
class SubsystemC(Subsystem):
options_scope = 'c'
@classmethod
def subsystem_dependencies(cls):
return (SubsystemA, )
class SubsystemB(Subsystem):
options_scope = 'b'
@classmethod
def subsystem_dependencies(cls):
return (SubsystemC, )
class SubsystemA(Subsystem):
options_scope = 'a'
@classmethod
def subsystem_dependencies(cls):
return (SubsystemB, )
for root in SubsystemA, SubsystemB, SubsystemC:
with self.assertRaises(Subsystem.CycleException):
Subsystem.closure((root, ))
def select(argv):
# Parse positional arguments to the script.
args = _create_bootstrap_binary_arg_parser().parse_args(argv[1:])
# Resolve bootstrap options with a fake empty command line.
options_bootstrapper = OptionsBootstrapper.create(args=[argv[0]])
subsystems = (GlobalOptionsRegistrar, BinaryUtil.Factory)
known_scope_infos = reduce(set.union, (ss.known_scope_infos() for ss in subsystems), set())
options = options_bootstrapper.get_full_options(known_scope_infos)
# Initialize Subsystems.
Subsystem.set_options(options)
# If the filename provided ends in a known archive extension (such as ".tar.gz"), then we get the
# appropriate Archiver to pass to BinaryUtil.
archiver_for_current_binary = None
filename = args.filename or args.util_name
try:
archiver_for_current_binary = archiver_for_path(filename)
# BinaryRequest requires the `name` field to be provided without an extension, as it appends the
# archiver's extension if one is provided, so we have to remove it here.
filename = filename[:-(len(archiver_for_current_binary.extension) + 1)]
except ValueError:
pass
binary_util = BinaryUtil.Factory.create()
binary_request = BinaryRequest(
supportdir='bin/{}'.format(args.util_name),
version=args.version,
name=filename,
platform_dependent=True,
external_url_generator=None,
archiver=archiver_for_current_binary)
return binary_util.select(binary_request)
def select(argv):
# Parse positional arguments to the script.
args = _create_bootstrap_binary_arg_parser().parse_args(argv[1:])
# Resolve bootstrap options with a fake empty command line.
options_bootstrapper = OptionsBootstrapper.create(args=[argv[0]])
subsystems = (GlobalOptionsRegistrar, BinaryUtil.Factory)
known_scope_infos = reduce(set.union,
(ss.known_scope_infos() for ss in subsystems),
set())
options = options_bootstrapper.get_full_options(known_scope_infos)
# Initialize Subsystems.
Subsystem.set_options(options)
# If the filename provided ends in a known archive extension (such as ".tar.gz"), then we get the
# appropriate Archiver to pass to BinaryUtil.
archiver_for_current_binary = None
filename = args.filename or args.util_name
try:
archiver_for_current_binary = archiver_for_path(filename)
# BinaryRequest requires the `name` field to be provided without an extension, as it appends the
# archiver's extension if one is provided, so we have to remove it here.
filename = filename[:-(len(archiver_for_current_binary.extension) + 1)]
except ValueError:
pass
binary_util = BinaryUtil.Factory.create()
binary_request = BinaryRequest(supportdir='bin/{}'.format(args.util_name),
version=args.version,
name=filename,
platform_dependent=True,
external_url_generator=None,
archiver=archiver_for_current_binary)
return binary_util.select(binary_request)
def tearDown(self):
"""
:API: public
"""
super(BaseTest, self).tearDown()
BuildFile.clear_cache()
Subsystem.reset()
def _create_interpreter_cache(self, setup_options=None):
Subsystem.reset(reset_options=True)
self.context(for_subsystems=[PythonInterpreterCache],
options={
'python-setup': setup_options,
})
return PythonInterpreterCache.global_instance()
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'cache_key_gen_version': '0-test',
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
# We need a pants.ini, even if empty. get_buildroot() uses its presence.
self.create_file('pants.ini')
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def create(cls, options_bootstrapper, build_configuration, init_subsystems=True):
global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'
.format(global_bootstrap_options.pants_version, pants_version())
)
pants_runtime_python_version = global_bootstrap_options.pants_runtime_python_version
current_python_version = '.'.join(map(str, sys.version_info[0:2]))
if pants_runtime_python_version and pants_runtime_python_version != current_python_version:
raise BuildConfigurationError(
'Running Pants with a different Python interpreter version than requested. '
'You requested {}, but are running with {}.\n\n'
'Note that Pants cannot use the value you give for `--pants-runtime-python-version` to '
'dynamically change the interpreter it uses, as it is too late for it to change once the program '
'is already running. Instead, your setup script (e.g. `./pants`) must configure which Python '
'interpreter and virtualenv to use. For example, the setup script we distribute '
'at https://www.pantsbuild.org/install.html#recommended-installation will read the '
'`pants_runtime_python_version` defined in your pants.ini to determine which Python '
'version to run with.'.format(pants_runtime_python_version, current_python_version)
)
# Parse and register options.
options = cls._construct_options(options_bootstrapper, build_configuration)
GlobalOptionsRegistrar.validate_instance(options.for_global_scope())
if init_subsystems:
Subsystem.set_options(options)
return options
def _install_options(self, options_bootstrapper, build_configuration):
"""Parse and register options.
:returns: An Options object representing the full set of runtime options.
"""
# TODO: This inline import is currently necessary to resolve a ~legitimate cycle between
# `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`.
from pants.bin.goal_runner import GoalRunner
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(
GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems()
)
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options
def _jar_dependency_management(self, **flags):
Subsystem.reset()
options = {
JarDependencyManagement.options_scope: flags,
}
return global_subsystem_instance(JarDependencyManagement,
options=options)
def _install_options(self, options_bootstrapper, build_configuration):
"""Parse and register options.
:returns: An Options object representing the full set of runtime options.
"""
# TODO: This inline import is currently necessary to resolve a ~legitimate cycle between
# `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`.
from pants.bin.goal_runner import GoalRunner
# Now that plugins and backends are loaded, we can gather the known scopes.
# Gather the optionables that are not scoped to any other. All known scopes are reachable
# via these optionables' known_scope_infos() methods.
top_level_optionables = ({GlobalOptionsRegistrar} |
GoalRunner.subsystems() |
build_configuration.subsystems() |
set(Goal.get_optionables()))
known_scope_infos = sorted({
si for optionable in top_level_optionables for si in optionable.known_scope_infos()
})
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
distinct_optionable_classes = sorted({si.optionable_cls for si in known_scope_infos},
key=lambda o: o.options_scope)
for optionable_cls in distinct_optionable_classes:
optionable_cls.register_options_on_scope(options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options
def subsystem_instance(subsystem_type, scope=None, **options):
"""Creates a Subsystem instance for test.
:API: public
:param type subsystem_type: The subclass of :class:`pants.subsystem.subsystem.Subsystem`
to create.
:param string scope: An optional scope to create the subsystem in; defaults to global.
:param **options: Keyword args representing option values explicitly set via the command line.
"""
if not issubclass(subsystem_type, Subsystem):
raise TypeError(
'The given `subsystem_type` was not a subclass of `Subsystem`: {}'.
format(subsystem_type))
optionables = Subsystem.closure([subsystem_type])
updated_options = dict(
Subsystem._options.items()) if Subsystem._options else {}
if options:
updated_options.update(options)
Subsystem._options = create_options_for_optionables(
optionables, options=updated_options)
try:
if scope is None:
yield subsystem_type.global_instance()
else:
class ScopedOptionable(Optionable):
options_scope = scope
options_scope_category = ScopeInfo.SUBSYSTEM
yield subsystem_type.scoped_instance(ScopedOptionable)
finally:
Subsystem.reset()
def context(self, for_task_types=None, for_subsystems=None, options=None,
target_roots=None, console_outstream=None, workspace=None,
**kwargs):
"""
:API: public
:param dict **kwargs: keyword arguments passed in to `create_options_for_optionables`.
"""
# Many tests use source root functionality via the SourceRootConfig.global_instance().
# (typically accessed via Target.target_base), so we always set it up, for convenience.
optionables = {SourceRootConfig}
extra_scopes = set()
for_subsystems = for_subsystems or ()
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError('You must set a scope on your subsystem type before using it in tests.')
optionables.add(subsystem)
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError('You must set a scope on your task type before using it in tests.')
optionables.add(task_type)
# If task is expected to inherit goal-level options, register those directly on the task,
# by subclassing the goal options registrar and settings its scope to the task scope.
if issubclass(task_type, GoalOptionsMixin):
subclass_name = b'test_{}_{}_{}'.format(
task_type.__name__, task_type.goal_options_registrar_cls.options_scope,
task_type.options_scope)
optionables.add(type(subclass_name, (task_type.goal_options_registrar_cls, ),
{b'options_scope': task_type.options_scope}))
extra_scopes.update([si.scope for si in task_type.known_scope_infos()])
optionables.update(Subsystem.closure(
set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) |
self._build_configuration.subsystems()))
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
fake_options = create_options_for_optionables(
optionables, extra_scopes=extra_scopes, options=options, **kwargs)
Subsystem.reset(reset_options=True)
Subsystem.set_options(fake_options)
context = create_context_from_options(fake_options,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
return context
def subsystem_instance(subsystem_type, scope=None, **options):
"""Creates a Subsystem instance for test.
:API: public
:param type subsystem_type: The subclass of :class:`pants.subsystem.subsystem.Subsystem`
to create.
:param string scope: An optional scope to create the subsystem in; defaults to global.
:param **options: Keyword args representing option values explicitly set via the command line.
"""
if not issubclass(subsystem_type, Subsystem):
raise TypeError('The given `subsystem_type` was not a subclass of `Subsystem`: {}'
.format(subsystem_type))
optionables = Subsystem.closure([subsystem_type])
updated_options = dict(Subsystem._options.items()) if Subsystem._options else {}
if options:
updated_options.update(options)
Subsystem._options = create_options_for_optionables(optionables, options=updated_options)
try:
if scope is None:
yield subsystem_type.global_instance()
else:
class ScopedOptionable(Optionable):
options_scope = scope
options_scope_category = ScopeInfo.SUBSYSTEM
yield subsystem_type.scoped_instance(ScopedOptionable)
finally:
Subsystem.reset()
def tearDown(self): """ :API: public """ super(BaseTest, self).tearDown() BuildFile.clear_cache() Subsystem.reset()
def test_closure_cycle(self):
class SubsystemC(Subsystem):
options_scope = 'c'
@classmethod
def subsystem_dependencies(cls):
return (SubsystemA,)
class SubsystemB(Subsystem):
options_scope = 'b'
@classmethod
def subsystem_dependencies(cls):
return (SubsystemC,)
class SubsystemA(Subsystem):
options_scope = 'a'
@classmethod
def subsystem_dependencies(cls):
return (SubsystemB,)
for root in SubsystemA, SubsystemB, SubsystemC:
with self.assertRaises(Subsystem.CycleException):
Subsystem.closure((root,))
def mk_cache(spec):
Subsystem.reset()
self.set_options_for_scope(CacheSetup.subscope(DummyTask.options_scope),
read_from=spec, compression=1)
self.context(for_task_types=[DummyTask]) # Force option initialization.
cache_factory = CacheSetup.create_cache_factory_for_task(DummyTask)
return cache_factory.get_read_cache()
def test_closure_graph(self):
class SubsystemB(Subsystem):
options_scope = 'b'
@classmethod
def subsystem_dependencies(cls):
return (DummySubsystem, )
class SubsystemA(Subsystem):
options_scope = 'a'
@classmethod
def subsystem_dependencies(cls):
return (DummySubsystem, SubsystemB)
self.assertEqual({DummySubsystem, SubsystemB},
Subsystem.closure((SubsystemB, )))
self.assertEqual({DummySubsystem, SubsystemA, SubsystemB},
Subsystem.closure((SubsystemA, )))
self.assertEqual({DummySubsystem, SubsystemA, SubsystemB},
Subsystem.closure((SubsystemA, SubsystemB)))
self.assertEqual({DummySubsystem, SubsystemA, SubsystemB},
Subsystem.closure(
(DummySubsystem, SubsystemA, SubsystemB)))
def create_graph_from_specs(self, graph_helper, specs):
Subsystem.reset()
session = graph_helper.new_session()
target_roots = self.create_target_roots(specs, session,
session.symbol_table)
graph = session.create_build_graph(target_roots)[0]
return graph, target_roots
def init_subsystems(subsystem_types, options=None):
"""Initialize subsystems for use in tests.
Does not create an instance. This function is for setting up subsystems that the code
under test creates.
Note that there is some redundancy between this function and BaseTest.context(for_subsystems=...).
TODO: Fix that.
:API: public
:param list subsystem_types: The subclasses of :class:`pants.subsystem.subsystem.Subsystem`
to create.
:param options: dict of scope -> (dict of option name -> value).
The scopes may be those of the global instances of the subsystems (i.e.,
subsystem_type.options_scope) and/or the scopes of instances of the
subsystems they transitively depend on.
"""
for s in subsystem_types:
if not Subsystem.is_subsystem_type(s):
raise TypeError('{} is not a subclass of `Subsystem`'.format(s))
optionables = Subsystem.closure(subsystem_types)
if options:
allowed_scopes = {o.options_scope for o in optionables}
for scope in options.keys():
if scope != '' and scope not in allowed_scopes:
raise ValueError('`{}` is not the scope of any of these subsystems: {}'.format(
scope, optionables))
# Don't trample existing subsystem options, in case a test has set up some
# other subsystems in some other way.
updated_options = dict(Subsystem._options.items()) if Subsystem._options else {}
if options:
updated_options.update(options)
Subsystem.set_options(create_options_for_optionables(optionables, options=updated_options))
def context(self, for_task_types=None, for_subsystems=None, options=None,
target_roots=None, console_outstream=None, workspace=None,
scheduler=None, **kwargs):
"""
:API: public
:param dict **kwargs: keyword arguments passed in to `create_options_for_optionables`.
"""
# Many tests use source root functionality via the SourceRootConfig.global_instance().
# (typically accessed via Target.target_base), so we always set it up, for convenience.
for_subsystems = set(for_subsystems or ())
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError('You must set a scope on your subsystem type before using it in tests.')
optionables = {SourceRootConfig} | self._build_configuration.subsystems() | for_subsystems
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError('You must set a scope on your task type before using it in tests.')
optionables.add(task_type)
# If task is expected to inherit goal-level options, register those directly on the task,
# by subclassing the goal options registrar and settings its scope to the task scope.
if issubclass(task_type, GoalOptionsMixin):
subclass_name = b'test_{}_{}_{}'.format(
task_type.__name__, task_type.goal_options_registrar_cls.options_scope,
task_type.options_scope)
optionables.add(type(subclass_name, (task_type.goal_options_registrar_cls, ),
{b'options_scope': task_type.options_scope}))
# Now expand to all deps.
all_optionables = set()
for optionable in optionables:
all_optionables.update(si.optionable_cls for si in optionable.known_scope_infos())
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
fake_options = create_options_for_optionables(
all_optionables, options=options, **kwargs)
Subsystem.reset(reset_options=True)
Subsystem.set_options(fake_options)
context = create_context_from_options(fake_options,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace,
scheduler=scheduler)
return context
def mk_cache(spec, resolver=None):
Subsystem.reset()
self.set_options_for_scope(CacheSetup.subscope(DummyTask.options_scope),
read_from=spec, compression=1)
self.context(for_task_types=[DummyTask]) # Force option initialization.
cache_factory = CacheSetup.create_cache_factory_for_task(DummyTask,
pinger=self.pinger,
resolver=resolver)
return cache_factory.get_read_cache()
def test_uninitialized_scoped_instance(self):
Subsystem.reset()
class UninitializedOptional(Optionable):
options_scope = 'optional'
optional = UninitializedOptional()
with self.assertRaisesRegexp(Subsystem.UninitializedSubsystemError,
r'UninitializedSubsystem.*uninitialized-scope'):
UninitializedSubsystem.scoped_instance(optional)
def test_uninitialized_scoped_instance(self):
Subsystem.reset()
class UninitializedOptional(Optionable):
options_scope = 'optional'
optional = UninitializedOptional()
with self.assertRaisesRegexp(Subsystem.UninitializedSubsystemError,
r'UninitializedSubsystem.*uninitialized-scope'):
UninitializedSubsystem.scoped_instance(optional)
def __init__(self, *args, **kwargs):
Subsystem.__init__(self, *args, **kwargs)
options = self.get_options()
self._workdir = options.pants_workdir
self._watchman_path = options.path
# N.B. watchman has 3 log levels: 0 == no logging, 1 == standard logging, 2 == verbose logging.
self._watchman_log_level = '2' if options.level == 'debug' else '1'
self._logger = logging.getLogger(__name__)
self._watchman = None
def test_uninitialized_scoped_instance(self) -> None:
Subsystem.reset()
class UninitializedOptional(Optionable):
options_scope = "optional"
optional = UninitializedOptional()
with self.assertRaisesRegex(
Subsystem.UninitializedSubsystemError, r"UninitializedSubsystem.*uninitialized-scope"
):
UninitializedSubsystem.scoped_instance(optional)
def test_is_python2(constraints, compatibilities):
Subsystem.reset()
init_subsystem(
PythonSetup,
{
PythonSetup.options_scope: {
"interpreter_constraints": RankedValue(Rank.CONFIG,
constraints)
}
},
)
assert is_python2(compatibilities, PythonSetup.global_instance())
def _clean_runtime_state(self):
"""Resets the runtime state from running ./pants -> running in the fork()'d daemon context."""
# TODO(kwlzn): Make this logic available to PantsRunner et al for inline state reset before
# pants runs to improve testability and avoid potential bitrot.
# Reset RunTracker state.
RunTracker.global_instance().reset(reset_options=False)
# Reset Subsystem options.
Subsystem.set_options(None)
# Reset Goals and Tasks.
Goal.clear()
def _clean_runtime_state(self):
"""Resets the runtime state from running ./pants -> running in the fork()'d daemon context."""
# TODO(kwlzn): Make this logic available to PantsRunner et al for inline state reset before
# pants runs to improve testability and avoid potential bitrot.
# Reset RunTracker state.
RunTracker.global_instance().reset(reset_options=False)
# Reset Subsystem options.
Subsystem.set_options(None)
# Reset Goals and Tasks.
Goal.clear()
def _setup_options(self, options_bootstrapper, working_set):
# TODO: This inline import is currently necessary to resolve a ~legitimate cycle between
# `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`.
from pants.bin.goal_runner import GoalRunner
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'.format(
global_bootstrap_options.pants_version, pants_version()
)
)
# Get logging setup prior to loading backends so that they can log as needed.
if self._init_logging:
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(
GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems()
)
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def _setup_options(self, options_bootstrapper, working_set):
# TODO: This inline import is currently necessary to resolve a ~legitimate cycle between
# `GoalRunner`->`EngineInitializer`->`OptionsInitializer`->`GoalRunner`.
from pants.bin.goal_runner import GoalRunner
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'
.format(global_bootstrap_options.pants_version,
pants_version()))
# Get logging setup prior to loading backends so that they can log as needed.
if self._init_logging:
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_backends_and_plugins(
plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(GoalRunner.subsystems()
| Goal.subsystems()
| build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def clean_global_runtime_state(reset_subsystem=False):
"""Resets the global runtime state of a pants runtime for cleaner forking.
:param bool reset_subsystem: Whether or not to clean Subsystem global state.
"""
if reset_subsystem:
# Reset subsystem state.
Subsystem.reset()
# Reset Goals and Tasks.
Goal.clear()
# Reset global plugin state.
BuildConfigInitializer.reset()
def clean_global_runtime_state(reset_subsystem=False):
"""Resets the global runtime state of a pants runtime for cleaner forking.
:param bool reset_subsystem: Whether or not to clean Subsystem global state.
"""
if reset_subsystem:
# Reset subsystem state.
Subsystem.reset()
# Reset Goals and Tasks.
Goal.clear()
# Reset backend/plugins state.
OptionsInitializer.reset()
def context(self, for_task_types=None, options=None, passthru_args=None, target_roots=None,
console_outstream=None, workspace=None, for_subsystems=None):
"""
:API: public
"""
# Many tests use source root functionality via the SourceRootConfig.global_instance()
# (typically accessed via Target.target_base), so we always set it up, for convenience.
optionables = {SourceRootConfig}
extra_scopes = set()
for_subsystems = for_subsystems or ()
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError('You must set a scope on your subsystem type before using it in tests.')
optionables.add(subsystem)
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError('You must set a scope on your task type before using it in tests.')
optionables.add(task_type)
extra_scopes.update([si.scope for si in task_type.known_scope_infos()])
optionables.update(Subsystem.closure(
set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) |
self._build_configuration.subsystems()))
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
options = create_options_for_optionables(optionables,
extra_scopes=extra_scopes,
options=options)
Subsystem.reset(reset_options=True)
Subsystem.set_options(options)
context = create_context(options=options,
passthru_args=passthru_args,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
return context
def context(self, for_task_types=None, options=None, passthru_args=None, target_roots=None,
console_outstream=None, workspace=None, for_subsystems=None):
"""
:API: public
"""
# Many tests use source root functionality via the SourceRootConfig.global_instance()
# (typically accessed via Target.target_base), so we always set it up, for convenience.
optionables = {SourceRootConfig}
extra_scopes = set()
for_subsystems = for_subsystems or ()
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError('You must set a scope on your subsystem type before using it in tests.')
optionables.add(subsystem)
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError('You must set a scope on your task type before using it in tests.')
optionables.add(task_type)
extra_scopes.update([si.scope for si in task_type.known_scope_infos()])
optionables.update(Subsystem.closure(
set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) |
self._build_configuration.subsystems()))
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
options = create_options_for_optionables(optionables,
extra_scopes=extra_scopes,
options=options)
Subsystem.reset(reset_options=True)
Subsystem.set_options(options)
context = create_context(options=options,
passthru_args=passthru_args,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
return context
def create(cls, options_bootstrapper, build_configuration, init_subsystems=True):
global_bootstrap_options = options_bootstrapper.get_bootstrap_options().for_global_scope()
if global_bootstrap_options.pants_version != pants_version():
raise BuildConfigurationError(
'Version mismatch: Requested version was {}, our version is {}.'
.format(global_bootstrap_options.pants_version, pants_version())
)
# Parse and register options.
options = cls._construct_options(options_bootstrapper, build_configuration)
if init_subsystems:
Subsystem.set_options(options)
return options
def _run(self):
global_options = self._options.for_global_scope()
streaming_handlers = global_options.streaming_workunits_handlers
report_interval = global_options.streaming_workunits_report_interval
callbacks = Subsystem.get_streaming_workunit_callbacks(
streaming_handlers)
streaming_reporter = StreamingWorkunitHandler(
self._scheduler_session,
callbacks=callbacks,
report_interval_seconds=report_interval)
help_output = self._maybe_handle_help()
if help_output is not None:
self._exiter.exit(help_output)
with streaming_reporter.session():
try:
engine_result = self._maybe_run_v2()
goal_runner_result = self._maybe_run_v1()
finally:
run_tracker_result = self._finish_run()
final_exit_code = self._compute_final_exit_code(
engine_result, goal_runner_result, run_tracker_result)
self._exiter.exit(final_exit_code)
def _run(self):
engine_result = PANTS_FAILED_EXIT_CODE
goal_runner_result = PANTS_FAILED_EXIT_CODE
try:
self._maybe_handle_help()
streaming_handlers = self._options.for_global_scope().streaming_workunits_handlers
callbacks = Subsystem.get_streaming_workunit_callbacks(streaming_handlers)
streaming_reporter = StreamingWorkunitHandler(self._scheduler_session, callbacks=callbacks)
with streaming_reporter.session():
engine_result = self._maybe_run_v2()
goal_runner_result = self._maybe_run_v1()
finally:
try:
self._update_stats()
run_tracker_result = self._run_tracker.end()
except ValueError as e:
# Calling .end() sometimes writes to a closed file, so we return a dummy result here.
logger.exception(e)
run_tracker_result = PANTS_SUCCEEDED_EXIT_CODE
final_exit_code = self._compute_final_exit_code(
engine_result,
goal_runner_result,
run_tracker_result
)
self._exiter.exit(final_exit_code)
def mk_cache(spec):
Subsystem.reset()
self.set_options_for_scope(CacheSetup.subscope(
DummyTask.options_scope),
read_from=spec,
compression=1)
self.context(for_task_types=[DummyTask
]) # Force option initialization.
cache_factory = CacheSetup.create_cache_factory_for_task(
DummyTask,
pinger=MockPinger({
'host1': 5,
'host2:666': 3,
'host3': 7
}))
return cache_factory.get_read_cache()
def run(self, start_time: float) -> ExitCode:
self._set_start_time(start_time)
with maybe_profiled(self.profile_path):
global_options = self.options.for_global_scope()
streaming_handlers = global_options.streaming_workunits_handlers
report_interval = global_options.streaming_workunits_report_interval
callbacks = Subsystem.get_streaming_workunit_callbacks(
streaming_handlers)
streaming_reporter = StreamingWorkunitHandler(
self.graph_session.scheduler_session,
callbacks=callbacks,
report_interval_seconds=report_interval,
)
if self.options.help_request:
help_printer = HelpPrinter(
options=self.options,
union_membership=self.union_membership)
return help_printer.print_help()
v1 = global_options.v1
v2 = global_options.v2
with streaming_reporter.session():
engine_result, goal_runner_result = PANTS_FAILED_EXIT_CODE, PANTS_FAILED_EXIT_CODE
try:
engine_result = self._maybe_run_v2(v2)
goal_runner_result = self._maybe_run_v1(v1)
except Exception as e:
ExceptionSink.log_exception(e)
run_tracker_result = self._finish_run(
self._merge_exit_codes(engine_result, goal_runner_result))
return self._merge_exit_codes(engine_result, goal_runner_result,
run_tracker_result)
def test_closure_tree(self):
class SubsystemB(Subsystem):
options_scope = 'b'
class SubsystemA(Subsystem):
options_scope = 'a'
@classmethod
def subsystem_dependencies(cls):
return (DummySubsystem, SubsystemB)
self.assertEqual({DummySubsystem, SubsystemA, SubsystemB}, Subsystem.closure((SubsystemA,)))
self.assertEqual({DummySubsystem, SubsystemA, SubsystemB},
Subsystem.closure((SubsystemA, SubsystemB)))
self.assertEqual({DummySubsystem, SubsystemA, SubsystemB},
Subsystem.closure((DummySubsystem, SubsystemA, SubsystemB)))
def run(self):
global_options = self.options.for_global_scope()
exiter = LocalExiter.wrap_global_exiter(self._run_tracker, self._repro)
profiled = maybe_profiled(self.profile_path)
with exiter, profiled:
streaming_handlers = global_options.streaming_workunits_handlers
report_interval = global_options.streaming_workunits_report_interval
callbacks = Subsystem.get_streaming_workunit_callbacks(
streaming_handlers)
streaming_reporter = StreamingWorkunitHandler(
self.graph_session.scheduler_session,
callbacks=callbacks,
report_interval_seconds=report_interval,
)
if self.options.help_request:
help_printer = HelpPrinter(
options=self.options,
union_membership=self.union_membership)
help_output = help_printer.print_help()
self._exiter.exit(help_output)
v1 = global_options.v1
v2 = global_options.v2
with streaming_reporter.session():
try:
engine_result = self._maybe_run_v2(v2)
goal_runner_result = self._maybe_run_v1(v1)
finally:
run_tracker_result = self._finish_run()
final_exit_code = self._compute_final_exit_code(
engine_result, goal_runner_result, run_tracker_result)
self._exiter.exit(final_exit_code)
def fake_distribution_locator(*versions):
"""Sets up a fake distribution locator with fake distributions.
Creates one distribution for each java version passed as an argument, and yields a list
of paths to the java homes for each distribution.
"""
with fake_distributions(versions) as paths:
path_options = {
DistributionLocator.options_scope: {
"paths": {
os_name: paths
}
}
}
Subsystem.reset()
init_subsystem(DistributionLocator, options=path_options)
yield paths
def fake_distribution_locator(*versions):
"""Sets up a fake distribution locator with fake distributions.
Creates one distribution for each java version passed as an argument, and yields a list of
paths to the java homes for each distribution.
"""
with fake_distributions(versions) as paths:
path_options = {
DistributionLocator.options_scope: {
'paths': {
os_name: paths,
}
}
}
Subsystem.reset()
init_subsystem(DistributionLocator, options=path_options)
yield paths
def _setup_options(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(
plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(GoalRunner.subsystems()
| Goal.subsystems()
| build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def _setup_options(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems via a union of all known subsystem sets.
subsystems = Subsystem.closure(
GoalRunner.subsystems() | Goal.subsystems() | build_configuration.subsystems()
)
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
options = options_bootstrapper.get_full_options(known_scope_infos)
self._register_options(subsystems, options)
# Make the options values available to all subsystems.
Subsystem.set_options(options)
return options, build_configuration
def context(
self,
for_task_types=None,
options=None,
passthru_args=None,
target_roots=None,
console_outstream=None,
workspace=None,
for_subsystems=None,
):
optionables = set()
extra_scopes = set()
for_subsystems = for_subsystems or ()
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError("You must set a scope on your subsystem type before using it in tests.")
optionables.add(subsystem)
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError("You must set a scope on your task type before using it in tests.")
optionables.add(task_type)
extra_scopes.update([si.scope for si in task_type.known_scope_infos()])
optionables.update(
Subsystem.closure(
set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()])
| self._build_configuration.subsystems()
)
)
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
option_values = create_options_for_optionables(optionables, extra_scopes=extra_scopes, options=options)
context = create_context(
options=option_values,
passthru_args=passthru_args,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace,
)
Subsystem._options = context.options
return context
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.subprocess_dir = os.path.join(self.build_root, '.pids')
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'pants_subprocessdir': self.subprocess_dir,
'cache_key_gen_version': '0-test',
}
self.options['cache'] = {
'read_from': [],
'write_to': [],
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree,
build_ignore_patterns=self.build_ignore_patterns)
self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
def register_subsystems(self, subsystems):
"""Registers the given subsystem types.
:param subsystems: The subsystem types to register.
:type subsystems: :class:`collections.Iterable` containing
:class:`pants.subsystem.subsystem.Subsystem` subclasses.
"""
if not isinstance(subsystems, Iterable):
raise TypeError('The subsystems must be an iterable, given {}'.format(subsystems))
invalid_subsystems = [s for s in subsystems if not Subsystem.is_subsystem_type(s)]
if invalid_subsystems:
raise TypeError('The following items from the given subsystems are not Subsystem '
'subclasses:\n\t{}'.format('\n\t'.join(str(i) for i in invalid_subsystems)))
self._subsystems.update(subsystems)
def _options(self):
# NB: The PluginResolver runs very early in the pants startup sequence before the standard
# Subsystem facility is wired up. As a result PluginResolver is not itself a Subsystem with
# PythonRepos as a dependency. Instead it does the minimum possible work to hand-roll
# bootstrapping of the Subsystem it needs.
subsystems = Subsystem.closure([PythonRepos])
known_scope_infos = [subsystem.get_scope_info() for subsystem in subsystems]
options = self._options_bootstrapper.get_full_options(known_scope_infos)
# Ignore command line flags since we'd blow up on any we don't understand (most of them).
# If someone wants to bootstrap plugins in a one-off custom way they'll need to use env vars
# or a --pants-config-files pointing to a custom pants.ini snippet.
defaulted_only_options = options.drop_flag_values()
GlobalOptionsRegistrar.register_options_on_scope(defaulted_only_options)
for subsystem in subsystems:
subsystem.register_options_on_scope(defaulted_only_options)
return defaulted_only_options
def tearDown(self): super(BaseTest, self).tearDown() FilesystemBuildFile.clear_cache() Subsystem.reset()
def test_uninitialized_global(self):
Subsystem.reset()
with self.assertRaisesRegexp(Subsystem.UninitializedSubsystemError,
r'UninitializedSubsystem.*uninitialized-scope'):
UninitializedSubsystem.global_instance()
def test_closure_simple(self):
self.assertEqual({DummySubsystem}, Subsystem.closure((DummySubsystem,)))
def setUp(self): super(PantsRunIntegrationTest, self).setUp() # Some integration tests rely on clean subsystem state (e.g., to set up a DistributionLocator). Subsystem.reset()
def tearDown(self): """ :API: public """ super(TestBase, self).tearDown() Subsystem.reset()
def create_graph_from_specs(self, graph_helper, specs): Subsystem.reset() target_roots = self.create_target_roots(specs) graph = graph_helper.create_build_graph(target_roots)[0] return graph, target_roots
