def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(
self.build_file_parser,
self.project_tree,
build_ignore_patterns=self.build_ignore_patterns)
self.build_graph = MutableBuildGraph(
address_mapper=self.address_mapper)
def test_address_lookup_error_hierarchy(self):
self.assertIsInstance(BuildFileAddressMapper.AddressNotInBuildFile(),
AddressLookupError)
self.assertIsInstance(BuildFileAddressMapper.EmptyBuildFileError(),
AddressLookupError)
self.assertIsInstance(
BuildFileAddressMapper.InvalidBuildFileReference(),
AddressLookupError)
self.assertIsInstance(BuildFileAddressMapper.InvalidAddressError(),
AddressLookupError)
self.assertIsInstance(BuildFileAddressMapper.BuildFileScanError(),
AddressLookupError)
def __init__(self,
root_dir,
options,
build_config,
run_tracker,
reporting,
exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
# Will be provided through context.address_mapper.build_ignore_patterns.
self._spec_excludes = None
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._project_tree = self._get_project_tree(
self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config,
self._root_dir)
build_ignore_patterns = self._global_options.ignore_patterns or []
build_ignore_patterns.extend(
BuildFile._spec_excludes_to_gitignore_syntax(
self._root_dir, self._global_options.spec_excludes))
self._address_mapper = BuildFileAddressMapper(self._build_file_parser,
self._project_tree,
build_ignore_patterns)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp)
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'cache_key_gen_version': '0-test',
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
# We need a pants.ini, even if empty. get_buildroot() uses its presence.
self.create_file('pants.ini')
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def test_exclude_target_regexps(self):
address_mapper_with_exclude = BuildFileAddressMapper(
self.build_file_parser,
self.project_tree,
exclude_target_regexps=[r'.*:b.*'])
self.assert_scanned(['::'],
expected=[':root', 'a', 'a/b:c'],
address_mapper=address_mapper_with_exclude)
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.subprocess_dir = os.path.join(self.build_root, '.pids')
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'pants_subprocessdir': self.subprocess_dir,
'cache_key_gen_version': '0-test',
}
self.options['cache'] = {
'read_from': [],
'write_to': [],
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration,
self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.address_mapper = BuildFileAddressMapper(
self.build_file_parser,
self.project_tree,
build_ignore_patterns=self.build_ignore_patterns)
self.build_graph = MutableBuildGraph(
address_mapper=self.address_mapper)
def _init_graph(self,
use_engine,
pants_ignore_patterns,
build_ignore_patterns,
exclude_target_regexps,
target_specs,
workdir,
graph_helper=None,
subproject_build_roots=None):
"""Determine the BuildGraph, AddressMapper and spec_roots for a given run.
:param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph.
:param list pants_ignore_patterns: The pants ignore patterns from '--pants-ignore'.
:param list build_ignore_patterns: The build ignore patterns from '--build-ignore',
applied during BUILD file searching.
:param str workdir: The pants workdir.
:param list exclude_target_regexps: Regular expressions for targets to be excluded.
:param list target_specs: The original target specs.
:param LegacyGraphHelper graph_helper: A LegacyGraphHelper to use for graph construction,
if available. This would usually come from the daemon.
:returns: A tuple of (BuildGraph, AddressMapper, opt Scheduler, spec_roots).
"""
# N.B. Use of the daemon implies use of the v2 engine.
if graph_helper or use_engine:
# The daemon may provide a `graph_helper`. If that's present, use it for graph construction.
if not graph_helper:
native = Native.create(self._global_options)
native.set_panic_handler()
graph_helper = EngineInitializer.setup_legacy_graph(
pants_ignore_patterns,
workdir,
self._global_options.build_file_imports,
native=native,
build_file_aliases=self._build_config.registered_aliases(),
build_ignore_patterns=build_ignore_patterns,
exclude_target_regexps=exclude_target_regexps,
subproject_roots=subproject_build_roots,
include_trace_on_error=self._options.for_global_scope(
).print_exception_stacktrace)
target_roots = TargetRoots.create(
options=self._options,
build_root=self._root_dir,
change_calculator=graph_helper.change_calculator)
graph, address_mapper = graph_helper.create_build_graph(
target_roots, self._root_dir)
return graph, address_mapper, graph_helper.scheduler, target_roots.as_specs(
)
else:
spec_roots = TargetRoots.parse_specs(target_specs, self._root_dir)
address_mapper = BuildFileAddressMapper(
self._build_file_parser,
get_project_tree(self._global_options), build_ignore_patterns,
exclude_target_regexps, subproject_build_roots)
return MutableBuildGraph(
address_mapper), address_mapper, None, spec_roots
def test_build_ignore_patterns(self):
expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c']
# This bogus BUILD file gets in the way of parsing.
self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n')
with self.assertRaises(AddressLookupError):
self.assert_scanned(['::'], expected=expected_specs)
address_mapper_with_ignore = BuildFileAddressMapper(self.build_file_parser,
self.project_tree,
build_ignore_patterns=['some'])
self.assert_scanned(['::'], expected=expected_specs, address_mapper=address_mapper_with_ignore)
def test_build_ignore_patterns(self):
expected_specs = [':root', 'a', 'a:b', 'a/b', 'a/b:c']
# This bogus BUILD file gets in the way of parsing.
self.add_to_build_file('some/dir', 'COMPLETELY BOGUS BUILDFILE)\n')
with self.assertRaises(CmdLineSpecParser.BadSpecError):
self.assert_parsed_list(cmdline_spec_list=['::'],
expected=expected_specs)
address_mapper_with_ignore = BuildFileAddressMapper(
self.build_file_parser,
self.project_tree,
build_ignore_patterns=['some'])
self.spec_parser = CmdLineSpecParser(self.build_root,
address_mapper_with_ignore)
self.assert_parsed_list(cmdline_spec_list=['::'],
expected=expected_specs)
def __init__(self, root_dir, options, build_config, run_tracker, reporting, build_graph=None,
exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param BuildGraph build_graph: A BuildGraph instance (for graph reuse, optional).
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
# Will be provided through context.address_mapper.build_ignore_patterns.
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
pants_ignore = self._global_options.pants_ignore or []
self._project_tree = self._get_project_tree(self._global_options.build_file_rev, pants_ignore)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
build_ignore_patterns = self._global_options.ignore_patterns or []
self._address_mapper = BuildFileAddressMapper(
self._build_file_parser,
self._project_tree,
build_ignore_patterns,
exclude_target_regexps=self._global_options.exclude_target_regexp
)
self._build_graph = self._select_buildgraph(self._global_options.enable_v2_engine,
self._global_options.pants_ignore,
build_graph)
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.subprocess_dir = os.path.join(self.build_root, '.pids')
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'pants_subprocessdir': self.subprocess_dir,
'cache_key_gen_version': '0-test',
}
self.options['cache'] = {
'read_from': [],
'write_to': [],
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree,
build_ignore_patterns=self.build_ignore_patterns)
self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._project_tree = self._get_project_tree(self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._project_tree)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp,
)
class GoalRunnerFactory(object):
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._project_tree = self._get_project_tree(self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._project_tree)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp,
)
def _get_project_tree(self, build_file_rev):
"""Creates the project tree for build files for use in a given pants run."""
if build_file_rev:
return ScmProjectTree(self._root_dir, get_scm(), build_file_rev)
else:
return FileSystemProjectTree(self._root_dir)
def _expand_goals(self, goals):
"""Check and populate the requested goals for a given run."""
for goal in goals:
try:
self._address_mapper.resolve_spec(goal)
logger.warning(
"Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)
)
except AddressLookupError:
pass
if self._help_request:
help_printer = HelpPrinter(self._options)
result = help_printer.print_help()
self._exiter(result)
self._goals.extend([Goal.by_name(goal) for goal in goals])
def _expand_specs(self, specs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name="parse", labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag))
for spec in specs:
for address in self._spec_parser.parse_addresses(spec, fail_fast):
self._build_graph.inject_address_closure(address)
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def _maybe_launch_pantsd(self):
"""Launches pantsd if configured to do so."""
if self._global_options.enable_pantsd:
# Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on.
with self._run_tracker.new_workunit(name="pantsd", labels=[WorkUnitLabel.SETUP]):
PantsDaemonLauncher.global_instance().maybe_launch()
def _is_quiet(self):
return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain
def _setup_context(self):
self._maybe_launch_pantsd()
with self._run_tracker.new_workunit(name="setup", labels=[WorkUnitLabel.SETUP]):
self._expand_goals(self._requested_goals)
self._expand_specs(self._target_specs, self._fail_fast)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(
self._global_options, self._is_quiet(), self._run_tracker
)
context = Context(
options=self._options,
run_tracker=self._run_tracker,
target_roots=self._targets,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
spec_excludes=self._spec_excludes,
invalidation_report=invalidation_report,
)
return context, invalidation_report
def setup(self):
context, invalidation_report = self._setup_context()
return GoalRunner(
context=context,
goals=self._goals,
kill_nailguns=self._kill_nailguns,
run_tracker=self._run_tracker,
invalidation_report=invalidation_report,
exiter=self._exiter,
)
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot.
:API: public
"""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path.
:API: public
"""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
:API: public
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_workdir_dir(self, relpath):
"""Creates a directory under the work directory.
:API: public
relpath: The relative path to the directory from the work directory.
"""
path = os.path.join(self.pants_workdir, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the buildroot.
:API: public
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def create_workdir_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the work directory.
:API: public
relpath: The relative path to the file from the work directory.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.pants_workdir, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
:API: public
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
return BuildFile(self.address_mapper._project_tree, relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
derived_from=None,
synthetic=False,
**kwargs):
"""Creates a target and injects it into the test's build graph.
:API: public
:param string spec: The target address spec that locates this target.
:param type target_type: The concrete target subclass to create this new target from.
:param list dependencies: A list of target instances this new target depends on.
:param derived_from: The target this new target was derived from.
:type derived_from: :class:`pants.build_graph.target.Target`
"""
address = Address.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.apply_injectables([target])
self.build_graph.inject_target(target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from,
synthetic=synthetic)
# TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
# Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
traversables = [target.compute_dependency_specs(payload=target.payload)]
for dependency_spec in itertools.chain(*traversables):
dependency_address = Address.parse(dependency_spec, relative_to=address.spec_path)
dependency_target = self.build_graph.get_target(dependency_address)
if not dependency_target:
raise ValueError('Tests must make targets for dependency specs ahead of them '
'being traversed, {} tried to traverse {} which does not exist.'
.format(target, dependency_address))
if dependency_target not in target.dependencies:
self.build_graph.inject_dependency(dependent=target.address,
dependency=dependency_address)
target.mark_transitive_invalidation_hash_dirty()
return target
@property
def alias_groups(self):
"""
:API: public
"""
return BuildFileAliases(targets={'target': Target})
@property
def build_ignore_patterns(self):
"""
:API: public
"""
return None
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
# Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup.
clean_global_runtime_state(reset_subsystem=True)
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.subprocess_dir = os.path.join(self.build_root, '.pids')
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'pants_subprocessdir': self.subprocess_dir,
'cache_key_gen_version': '0-test',
}
self.options['cache'] = {
'read_from': [],
'write_to': [],
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.reset_build_graph()
def buildroot_files(self, relpath=None):
"""Returns the set of all files under the test build root.
:API: public
:param string relpath: If supplied, only collect files from this subtree.
:returns: All file paths found.
:rtype: set
"""
def scan():
for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or '')):
for f in files:
yield os.path.relpath(os.path.join(root, f), self.build_root)
return set(scan())
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree,
build_ignore_patterns=self.build_ignore_patterns)
self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
def set_options_for_scope(self, scope, **kwargs):
self.options[scope].update(kwargs)
def context(self, for_task_types=None, for_subsystems=None, options=None,
target_roots=None, console_outstream=None, workspace=None,
**kwargs):
"""
:API: public
:param dict **kwargs: keyword arguments passed in to `create_options_for_optionables`.
"""
# Many tests use source root functionality via the SourceRootConfig.global_instance().
# (typically accessed via Target.target_base), so we always set it up, for convenience.
optionables = {SourceRootConfig}
extra_scopes = set()
for_subsystems = for_subsystems or ()
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError('You must set a scope on your subsystem type before using it in tests.')
optionables.add(subsystem)
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError('You must set a scope on your task type before using it in tests.')
optionables.add(task_type)
# If task is expected to inherit goal-level options, register those directly on the task,
# by subclassing the goal options registrar and settings its scope to the task scope.
if issubclass(task_type, GoalOptionsMixin):
subclass_name = b'test_{}_{}_{}'.format(
task_type.__name__, task_type.goal_options_registrar_cls.options_scope,
task_type.options_scope)
optionables.add(type(subclass_name, (task_type.goal_options_registrar_cls, ),
{b'options_scope': task_type.options_scope}))
extra_scopes.update([si.scope for si in task_type.known_scope_infos()])
optionables.update(Subsystem.closure(
set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) |
self._build_configuration.subsystems()))
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
fake_options = create_options_for_optionables(
optionables, extra_scopes=extra_scopes, options=options, **kwargs)
Subsystem.reset(reset_options=True)
Subsystem.set_options(fake_options)
context = create_context_from_options(fake_options,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
return context
def tearDown(self):
"""
:API: public
"""
super(BaseTest, self).tearDown()
BuildFile.clear_cache()
Subsystem.reset()
def target(self, spec):
"""Resolves the given target address to a Target object.
:API: public
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
address = Address.parse(spec)
self.build_graph.inject_address_closure(address)
return self.build_graph.get_target(address)
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
:API: public
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec = CmdLineSpecParser(self.build_root).parse_spec(spec)
addresses = list(self.address_mapper.scan_specs([spec]))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [self.build_graph.get_target(address) for address in addresses]
return targets
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
:API: public
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
:API: public
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(path, dedent('''
%(target_type)s(name='%(name)s',
%(sources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
''' % dict(target_type=target_type,
name=name,
sources=('sources=%s,' % repr(sources)
if sources else ''),
java_sources=('java_sources=[%s],'
% ','.join(map(lambda str_target: '"%s"' % str_target,
kwargs.get('java_sources')))
if 'java_sources' in kwargs else ''),
provides=('provides=%s,' % kwargs.get('provides')
if 'provides' in kwargs else ''),
dependencies=('dependencies=%s,' % kwargs.get('dependencies')
if 'dependencies' in kwargs else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
"""
:API: public
"""
return self.create_library(path, 'resources', name, sources)
def assertUnorderedPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, unordered.
:API: public
"""
actual = list(itertools.islice(actual_iter, len(expected)))
self.assertEqual(sorted(expected), sorted(actual))
def assertPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, in order.
:API: public
"""
self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected))))
def assertInFile(self, string, file_path):
"""Verifies that a string appears in a file
:API: public
"""
with open(file_path) as f:
content = f.read()
self.assertIn(string, content, '"{}" is not in the file {}:\n{}'.format(string, f.name, content))
def get_bootstrap_options(self, cli_options=()):
"""Retrieves bootstrap options.
:param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`.
"""
# Can't parse any options without a pants.ini.
self.create_file('pants.ini')
return OptionsBootstrapper(args=cli_options).get_bootstrap_options().for_global_scope()
class LoggingRecorder(object):
"""Simple logging handler to record warnings."""
def __init__(self):
self._records = []
self.level = logging.DEBUG
def handle(self, record):
self._records.append(record)
def _messages_for_level(self, levelname):
return ['{}: {}'.format(record.name, record.getMessage())
for record in self._records if record.levelname == levelname]
def infos(self):
return self._messages_for_level('INFO')
def warnings(self):
return self._messages_for_level('WARNING')
@contextmanager
def captured_logging(self, level=None):
root_logger = logging.getLogger()
old_level = root_logger.level
root_logger.setLevel(level or logging.NOTSET)
handler = self.LoggingRecorder()
root_logger.addHandler(handler)
try:
yield handler
finally:
root_logger.setLevel(old_level)
root_logger.removeHandler(handler)
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot.
:API: public
"""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path.
:API: public
"""
if os.path.basename(relpath).startswith("BUILD"):
return relpath
else:
return os.path.join(relpath, "BUILD")
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
:API: public
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_workdir_dir(self, relpath):
"""Creates a directory under the work directory.
:API: public
relpath: The relative path to the directory from the work directory.
"""
path = os.path.join(self.pants_workdir, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents="", mode="wb"):
"""Writes to a file under the buildroot.
:API: public
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def create_workdir_file(self, relpath, contents="", mode="wb"):
"""Writes to a file under the work directory.
:API: public
relpath: The relative path to the file from the work directory.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.pants_workdir, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
:API: public
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode="a")
return BuildFile(self.address_mapper._project_tree, relpath=self.build_path(relpath))
def make_target(self, spec="", target_type=Target, dependencies=None, derived_from=None, synthetic=False, **kwargs):
"""Creates a target and injects it into the test's build graph.
:API: public
:param string spec: The target address spec that locates this target.
:param type target_type: The concrete target subclass to create this new target from.
:param list dependencies: A list of target instances this new target depends on.
:param derived_from: The target this new target was derived from.
:type derived_from: :class:`pants.build_graph.target.Target`
"""
address = Address.parse(spec)
target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs)
dependencies = dependencies or []
self.build_graph.inject_target(
target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from, synthetic=synthetic
)
# TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
# Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
for traversable_dependency_spec in target.traversable_dependency_specs:
traversable_dependency_address = Address.parse(traversable_dependency_spec, relative_to=address.spec_path)
traversable_dependency_target = self.build_graph.get_target(traversable_dependency_address)
if not traversable_dependency_target:
raise ValueError(
"Tests must make targets for traversable dependency specs ahead of them "
"being traversed, {} tried to traverse {} which does not exist.".format(
target, traversable_dependency_address
)
)
if traversable_dependency_target not in target.dependencies:
self.build_graph.inject_dependency(dependent=target.address, dependency=traversable_dependency_address)
target.mark_transitive_invalidation_hash_dirty()
return target
@property
def alias_groups(self):
"""
:API: public
"""
return BuildFileAliases(targets={"target": Target})
@property
def build_ignore_patterns(self):
"""
:API: public
"""
return None
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
# Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup.
clean_global_runtime_state(reset_runtracker=False, reset_subsystem=True)
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT"))
self.subprocess_dir = os.path.join(self.build_root, ".pids")
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, ".pants.d")
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[""] = {
"pants_workdir": self.pants_workdir,
"pants_supportdir": os.path.join(self.build_root, "build-support"),
"pants_distdir": os.path.join(self.build_root, "dist"),
"pants_configdir": os.path.join(self.build_root, "config"),
"pants_subprocessdir": self.subprocess_dir,
"cache_key_gen_version": "0-test",
}
self.options["cache"] = {"read_from": [], "write_to": []}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.reset_build_graph()
def buildroot_files(self, relpath=None):
"""Returns the set of all files under the test build root.
:API: public
:param string relpath: If supplied, only collect files from this subtree.
:returns: All file paths found.
:rtype: set
"""
def scan():
for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or "")):
for f in files:
yield os.path.relpath(os.path.join(root, f), self.build_root)
return set(scan())
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(
self.build_file_parser, self.project_tree, build_ignore_patterns=self.build_ignore_patterns
)
self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
def set_options_for_scope(self, scope, **kwargs):
self.options[scope].update(kwargs)
def context(
self,
for_task_types=None,
options=None,
passthru_args=None,
target_roots=None,
console_outstream=None,
workspace=None,
for_subsystems=None,
):
"""
:API: public
"""
# Many tests use source root functionality via the SourceRootConfig.global_instance()
# (typically accessed via Target.target_base), so we always set it up, for convenience.
optionables = {SourceRootConfig}
extra_scopes = set()
for_subsystems = for_subsystems or ()
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError("You must set a scope on your subsystem type before using it in tests.")
optionables.add(subsystem)
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError("You must set a scope on your task type before using it in tests.")
optionables.add(task_type)
extra_scopes.update([si.scope for si in task_type.known_scope_infos()])
optionables.update(
Subsystem.closure(
set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()])
| self._build_configuration.subsystems()
)
)
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
options = create_options_for_optionables(optionables, extra_scopes=extra_scopes, options=options)
Subsystem.reset(reset_options=True)
Subsystem.set_options(options)
context = create_context(
options=options,
passthru_args=passthru_args,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace,
)
return context
def tearDown(self):
"""
:API: public
"""
super(BaseTest, self).tearDown()
BuildFile.clear_cache()
Subsystem.reset()
def target(self, spec):
"""Resolves the given target address to a Target object.
:API: public
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
address = Address.parse(spec)
self.build_graph.inject_address_closure(address)
return self.build_graph.get_target(address)
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
:API: public
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec = CmdLineSpecParser(self.build_root).parse_spec(spec)
addresses = list(self.address_mapper.scan_specs([spec]))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [self.build_graph.get_target(address) for address in addresses]
return targets
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
:API: public
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
:API: public
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(
path,
dedent(
"""
%(target_type)s(name='%(name)s',
%(sources)s
%(resources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
"""
% dict(
target_type=target_type,
name=name,
sources=("sources=%s," % repr(sources) if sources else ""),
resources=('resources=["%s"],' % kwargs.get("resources") if "resources" in kwargs else ""),
java_sources=(
"java_sources=[%s],"
% ",".join(map(lambda str_target: '"%s"' % str_target, kwargs.get("java_sources")))
if "java_sources" in kwargs
else ""
),
provides=("provides=%s," % kwargs.get("provides") if "provides" in kwargs else ""),
dependencies=("dependencies=%s," % kwargs.get("dependencies") if "dependencies" in kwargs else ""),
)
),
)
return self.target("%s:%s" % (path, name))
def create_resources(self, path, name, *sources):
"""
:API: public
"""
return self.create_library(path, "resources", name, sources)
def assertUnorderedPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, unordered.
:API: public
"""
actual = list(itertools.islice(actual_iter, len(expected)))
self.assertEqual(sorted(expected), sorted(actual))
def assertPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, in order.
:API: public
"""
self.assertEqual(expected, list(itertools.islice(actual_iter, len(expected))))
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot.
:API: public
"""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path.
:API: public
"""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
:API: public
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_workdir_dir(self, relpath):
"""Creates a directory under the work directory.
:API: public
relpath: The relative path to the directory from the work directory.
"""
path = os.path.join(self.pants_workdir, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the buildroot.
:API: public
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def create_workdir_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the work directory.
:API: public
relpath: The relative path to the file from the work directory.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.pants_workdir, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
:API: public
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
return BuildFile(self.address_mapper._project_tree,
relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
derived_from=None,
synthetic=False,
**kwargs):
"""Creates a target and injects it into the test's build graph.
:API: public
:param string spec: The target address spec that locates this target.
:param type target_type: The concrete target subclass to create this new target from.
:param list dependencies: A list of target instances this new target depends on.
:param derived_from: The target this new target was derived from.
:type derived_from: :class:`pants.build_graph.target.Target`
"""
address = Address.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.apply_injectables([target])
self.build_graph.inject_target(
target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from,
synthetic=synthetic)
# TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
# Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
traversables = [
target.compute_dependency_specs(payload=target.payload)
]
for dependency_spec in itertools.chain(*traversables):
dependency_address = Address.parse(dependency_spec,
relative_to=address.spec_path)
dependency_target = self.build_graph.get_target(dependency_address)
if not dependency_target:
raise ValueError(
'Tests must make targets for dependency specs ahead of them '
'being traversed, {} tried to traverse {} which does not exist.'
.format(target, dependency_address))
if dependency_target not in target.dependencies:
self.build_graph.inject_dependency(
dependent=target.address, dependency=dependency_address)
target.mark_transitive_invalidation_hash_dirty()
return target
@property
def alias_groups(self):
"""
:API: public
"""
return BuildFileAliases(targets={'target': Target})
@property
def build_ignore_patterns(self):
"""
:API: public
"""
return None
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
# Avoid resetting the Runtracker here, as that is specific to fork'd process cleanup.
clean_global_runtime_state(reset_subsystem=True)
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.subprocess_dir = os.path.join(self.build_root, '.pids')
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[GLOBAL_SCOPE] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'pants_subprocessdir': self.subprocess_dir,
'cache_key_gen_version': '0-test',
}
self.options['cache'] = {
'read_from': [],
'write_to': [],
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration,
self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.reset_build_graph()
def buildroot_files(self, relpath=None):
"""Returns the set of all files under the test build root.
:API: public
:param string relpath: If supplied, only collect files from this subtree.
:returns: All file paths found.
:rtype: set
"""
def scan():
for root, dirs, files in os.walk(
os.path.join(self.build_root, relpath or '')):
for f in files:
yield os.path.relpath(os.path.join(root, f),
self.build_root)
return set(scan())
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(
self.build_file_parser,
self.project_tree,
build_ignore_patterns=self.build_ignore_patterns)
self.build_graph = MutableBuildGraph(
address_mapper=self.address_mapper)
def set_options_for_scope(self, scope, **kwargs):
self.options[scope].update(kwargs)
def context(self,
for_task_types=None,
for_subsystems=None,
options=None,
target_roots=None,
console_outstream=None,
workspace=None,
scheduler=None,
**kwargs):
"""
:API: public
:param dict **kwargs: keyword arguments passed in to `create_options_for_optionables`.
"""
# Many tests use source root functionality via the SourceRootConfig.global_instance().
# (typically accessed via Target.target_base), so we always set it up, for convenience.
for_subsystems = set(for_subsystems or ())
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError(
'You must set a scope on your subsystem type before using it in tests.'
)
optionables = {
SourceRootConfig
} | self._build_configuration.subsystems() | for_subsystems
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError(
'You must set a scope on your task type before using it in tests.'
)
optionables.add(task_type)
# If task is expected to inherit goal-level options, register those directly on the task,
# by subclassing the goal options registrar and settings its scope to the task scope.
if issubclass(task_type, GoalOptionsMixin):
subclass_name = b'test_{}_{}_{}'.format(
task_type.__name__,
task_type.goal_options_registrar_cls.options_scope,
task_type.options_scope)
optionables.add(
type(subclass_name,
(task_type.goal_options_registrar_cls, ),
{b'options_scope': task_type.options_scope}))
# Now expand to all deps.
all_optionables = set()
for optionable in optionables:
all_optionables.update(si.optionable_cls
for si in optionable.known_scope_infos())
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
fake_options = create_options_for_optionables(all_optionables,
options=options,
**kwargs)
Subsystem.reset(reset_options=True)
Subsystem.set_options(fake_options)
context = create_context_from_options(
fake_options,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace,
scheduler=scheduler)
return context
def tearDown(self):
"""
:API: public
"""
super(BaseTest, self).tearDown()
BuildFile.clear_cache()
Subsystem.reset()
def target(self, spec):
"""Resolves the given target address to a Target object.
:API: public
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
address = Address.parse(spec)
self.build_graph.inject_address_closure(address)
return self.build_graph.get_target(address)
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
:API: public
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec = CmdLineSpecParser(self.build_root).parse_spec(spec)
addresses = list(self.address_mapper.scan_specs([spec]))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [
self.build_graph.get_target(address) for address in addresses
]
return targets
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
:API: public
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
:API: public
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(
path,
dedent('''
%(target_type)s(name='%(name)s',
%(sources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
''' % dict(
target_type=target_type,
name=name,
sources=('sources=%s,' % repr(sources) if sources else ''),
java_sources=('java_sources=[%s],' % ','.join(
map(lambda str_target: '"%s"' % str_target,
kwargs.get('java_sources')))
if 'java_sources' in kwargs else ''),
provides=('provides=%s,' % kwargs.get('provides')
if 'provides' in kwargs else ''),
dependencies=('dependencies=%s,' % kwargs.get('dependencies')
if 'dependencies' in kwargs else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
"""
:API: public
"""
return self.create_library(path, 'resources', name, sources)
def assertUnorderedPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, unordered.
:API: public
"""
actual = list(itertools.islice(actual_iter, len(expected)))
self.assertEqual(sorted(expected), sorted(actual))
def assertPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, in order.
:API: public
"""
self.assertEqual(expected,
list(itertools.islice(actual_iter, len(expected))))
def assertInFile(self, string, file_path):
"""Verifies that a string appears in a file
:API: public
"""
with open(file_path) as f:
content = f.read()
self.assertIn(
string, content, '"{}" is not in the file {}:\n{}'.format(
string, f.name, content))
def get_bootstrap_options(self, cli_options=()):
"""Retrieves bootstrap options.
:param cli_options: An iterable of CLI flags to pass as arguments to `OptionsBootstrapper`.
"""
# Can't parse any options without a pants.ini.
self.create_file('pants.ini')
return OptionsBootstrapper(
args=cli_options).get_bootstrap_options().for_global_scope()
class LoggingRecorder(object):
"""Simple logging handler to record warnings."""
def __init__(self):
self._records = []
self.level = logging.DEBUG
def handle(self, record):
self._records.append(record)
def _messages_for_level(self, levelname):
return [
'{}: {}'.format(record.name, record.getMessage())
for record in self._records if record.levelname == levelname
]
def infos(self):
return self._messages_for_level('INFO')
def warnings(self):
return self._messages_for_level('WARNING')
@contextmanager
def captured_logging(self, level=None):
root_logger = logging.getLogger()
old_level = root_logger.level
root_logger.setLevel(level or logging.NOTSET)
handler = self.LoggingRecorder()
root_logger.addHandler(handler)
try:
yield handler
finally:
root_logger.setLevel(old_level)
root_logger.removeHandler(handler)
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot.
:API: public
"""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path.
:API: public
"""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
:API: public
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_workdir_dir(self, relpath):
"""Creates a directory under the work directory.
:API: public
relpath: The relative path to the directory from the work directory.
"""
path = os.path.join(self.pants_workdir, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the buildroot.
:API: public
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def create_workdir_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the work directory.
:API: public
relpath: The relative path to the file from the work directory.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.pants_workdir, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
:API: public
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
return BuildFile(self.address_mapper._project_tree,
relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
derived_from=None,
synthetic=False,
**kwargs):
"""Creates a target and injects it into the test's build graph.
:API: public
:param string spec: The target address spec that locates this target.
:param type target_type: The concrete target subclass to create this new target from.
:param list dependencies: A list of target instances this new target depends on.
:param derived_from: The target this new target was derived from.
:type derived_from: :class:`pants.build_graph.target.Target`
"""
address = Address.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.inject_target(
target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from,
synthetic=synthetic)
# TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
# Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
for traversable_dependency_spec in target.traversable_dependency_specs:
traversable_dependency_address = Address.parse(
traversable_dependency_spec, relative_to=address.spec_path)
traversable_dependency_target = self.build_graph.get_target(
traversable_dependency_address)
if not traversable_dependency_target:
raise ValueError(
'Tests must make targets for traversable dependency specs ahead of them '
'being traversed, {} tried to traverse {} which does not exist.'
.format(target, traversable_dependency_address))
if traversable_dependency_target not in target.dependencies:
self.build_graph.inject_dependency(
dependent=target.address,
dependency=traversable_dependency_address)
target.mark_transitive_invalidation_hash_dirty()
return target
@property
def alias_groups(self):
"""
:API: public
"""
return BuildFileAliases(targets={'target': Target})
@property
def build_ignore_patterns(self):
"""
:API: public
"""
return None
def setUp(self):
"""
:API: public
"""
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.subprocess_dir = os.path.join(self.build_root, '.pids')
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'pants_subprocessdir': self.subprocess_dir,
'cache_key_gen_version': '0-test',
}
self.options['cache'] = {
'read_from': [],
'write_to': [],
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration,
self.build_root)
self.project_tree = FileSystemProjectTree(self.build_root)
self.address_mapper = BuildFileAddressMapper(
self.build_file_parser,
self.project_tree,
build_ignore_patterns=self.build_ignore_patterns)
self.build_graph = MutableBuildGraph(
address_mapper=self.address_mapper)
def buildroot_files(self, relpath=None):
"""Returns the set of all files under the test build root.
:API: public
:param string relpath: If supplied, only collect files from this subtree.
:returns: All file paths found.
:rtype: set
"""
def scan():
for root, dirs, files in os.walk(
os.path.join(self.build_root, relpath or '')):
for f in files:
yield os.path.relpath(os.path.join(root, f),
self.build_root)
return set(scan())
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(
self.build_file_parser, FileSystemProjectTree(self.build_root))
self.build_graph = MutableBuildGraph(
address_mapper=self.address_mapper)
def set_options_for_scope(self, scope, **kwargs):
self.options[scope].update(kwargs)
def context(self,
for_task_types=None,
options=None,
passthru_args=None,
target_roots=None,
console_outstream=None,
workspace=None,
for_subsystems=None):
"""
:API: public
"""
# Many tests use source root functionality via the SourceRootConfig.global_instance()
# (typically accessed via Target.target_base), so we always set it up, for convenience.
optionables = {SourceRootConfig}
extra_scopes = set()
for_subsystems = for_subsystems or ()
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError(
'You must set a scope on your subsystem type before using it in tests.'
)
optionables.add(subsystem)
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError(
'You must set a scope on your task type before using it in tests.'
)
optionables.add(task_type)
extra_scopes.update(
[si.scope for si in task_type.known_scope_infos()])
optionables.update(
Subsystem.closure(
set([
dep.subsystem_cls
for dep in task_type.subsystem_dependencies_iter()
]) | self._build_configuration.subsystems()))
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
options = create_options_for_optionables(optionables,
extra_scopes=extra_scopes,
options=options)
Subsystem.reset(reset_options=True)
Subsystem.set_options(options)
context = create_context(options=options,
passthru_args=passthru_args,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
return context
def tearDown(self):
"""
:API: public
"""
super(BaseTest, self).tearDown()
BuildFile.clear_cache()
Subsystem.reset()
def target(self, spec):
"""Resolves the given target address to a Target object.
:API: public
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
address = Address.parse(spec)
self.build_graph.inject_address_closure(address)
return self.build_graph.get_target(address)
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
:API: public
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec = CmdLineSpecParser(self.build_root).parse_spec(spec)
addresses = list(self.address_mapper.scan_specs([spec]))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [
self.build_graph.get_target(address) for address in addresses
]
return targets
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
:API: public
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
:API: public
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(
path,
dedent('''
%(target_type)s(name='%(name)s',
%(sources)s
%(resources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
''' % dict(
target_type=target_type,
name=name,
sources=('sources=%s,' % repr(sources) if sources else ''),
resources=('resources=["%s"],' % kwargs.get('resources')
if 'resources' in kwargs else ''),
java_sources=('java_sources=[%s],' % ','.join(
map(lambda str_target: '"%s"' % str_target,
kwargs.get('java_sources')))
if 'java_sources' in kwargs else ''),
provides=('provides=%s,' % kwargs.get('provides')
if 'provides' in kwargs else ''),
dependencies=('dependencies=%s,' % kwargs.get('dependencies')
if 'dependencies' in kwargs else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
"""
:API: public
"""
return self.create_library(path, 'resources', name, sources)
def assertUnorderedPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, unordered.
:API: public
"""
actual = list(itertools.islice(actual_iter, len(expected)))
self.assertEqual(sorted(expected), sorted(actual))
def assertPrefixEqual(self, expected, actual_iter):
"""Consumes len(expected) items from the given iter, and asserts that they match, in order.
:API: public
"""
self.assertEqual(expected,
list(itertools.islice(actual_iter, len(expected))))
class GoalRunnerFactory(object):
def __init__(self,
root_dir,
options,
build_config,
run_tracker,
reporting,
exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._build_file_type = self._get_buildfile_type(
self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config,
self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser,
self._build_file_type)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp)
def _get_buildfile_type(self, build_file_rev):
"""Selects the BuildFile type for use in a given pants run."""
if build_file_rev:
ScmBuildFile.set_rev(build_file_rev)
ScmBuildFile.set_scm(get_scm())
return ScmBuildFile
else:
return FilesystemBuildFile
def _expand_goals(self, goals):
"""Check and populate the requested goals for a given run."""
for goal in goals:
if self._address_mapper.from_cache(self._root_dir,
goal,
must_exist=False).file_exists():
logger.warning(
"Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
if self._help_request:
help_printer = HelpPrinter(self._options)
help_printer.print_help()
self._exiter(0)
self._goals.extend([Goal.by_name(goal) for goal in goals])
def _expand_specs(self, specs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name='parse',
labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag,
filter_for_tag))
for spec in specs:
for address in self._spec_parser.parse_addresses(
spec, fail_fast):
self._build_graph.inject_address_closure(address)
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def _is_quiet(self):
return any(
goal.has_task_of_type(QuietTaskMixin)
for goal in self._goals) or self._explain
def _setup_context(self):
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self._run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self._address_mapper,
self._build_file_parser)
with self._run_tracker.new_workunit(name='setup',
labels=[WorkUnitLabel.SETUP]):
self._expand_goals(self._requested_goals)
self._expand_specs(self._target_specs, self._fail_fast)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(
self._global_options, self._is_quiet(), self._run_tracker)
context = Context(options=self._options,
run_tracker=self._run_tracker,
target_roots=self._targets,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
spec_excludes=self._spec_excludes,
invalidation_report=invalidation_report)
return context, invalidation_report
def setup(self):
context, invalidation_report = self._setup_context()
return GoalRunner(context=context,
goals=self._goals,
kill_nailguns=self._kill_nailguns,
run_tracker=self._run_tracker,
invalidation_report=invalidation_report)
class GoalRunnerFactory(object):
def __init__(self, root_dir, options, build_config, run_tracker, reporting, build_graph=None,
exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param BuildGraph build_graph: A BuildGraph instance (for graph reuse, optional).
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
# Will be provided through context.address_mapper.build_ignore_patterns.
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
pants_ignore = self._global_options.pants_ignore or []
self._project_tree = self._get_project_tree(self._global_options.build_file_rev, pants_ignore)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
build_ignore_patterns = self._global_options.ignore_patterns or []
self._address_mapper = BuildFileAddressMapper(
self._build_file_parser,
self._project_tree,
build_ignore_patterns,
exclude_target_regexps=self._global_options.exclude_target_regexp
)
self._build_graph = self._select_buildgraph(self._global_options.enable_v2_engine,
self._global_options.pants_ignore,
build_graph)
def _select_buildgraph(self, use_engine, path_ignore_patterns, cached_buildgraph=None):
"""Selects a BuildGraph to use then constructs and returns it.
:param bool use_engine: Whether or not to use the v2 engine to construct the BuildGraph.
:param list path_ignore_patterns: The path ignore patterns from `--pants-ignore`.
:param LegacyBuildGraph cached_buildgraph: A cached graph to reuse, if available.
"""
if cached_buildgraph is not None:
return cached_buildgraph
elif use_engine:
root_specs = EngineInitializer.parse_commandline_to_spec_roots(options=self._options,
build_root=self._root_dir)
graph_helper = EngineInitializer.setup_legacy_graph(path_ignore_patterns)
return graph_helper.create_graph(root_specs)
else:
return MutableBuildGraph(self._address_mapper)
def _get_project_tree(self, build_file_rev, pants_ignore):
"""Creates the project tree for build files for use in a given pants run."""
if build_file_rev:
return ScmProjectTree(self._root_dir, get_scm(), build_file_rev, pants_ignore)
else:
return FileSystemProjectTree(self._root_dir, pants_ignore)
def _expand_goals(self, goals):
"""Check and populate the requested goals for a given run."""
for goal in goals:
try:
self._address_mapper.resolve_spec(goal)
logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
except AddressLookupError:
pass
if self._help_request:
help_printer = HelpPrinter(self._options)
result = help_printer.print_help()
self._exiter(result)
self._goals.extend([Goal.by_name(goal) for goal in goals])
def _expand_specs(self, spec_strs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag))
# Parse all specs into unique Spec objects.
spec_parser = CmdLineSpecParser(self._root_dir)
specs = OrderedSet()
for spec_str in spec_strs:
specs.add(spec_parser.parse_spec(spec_str))
# Then scan them to generate unique Addresses.
for address in self._build_graph.inject_specs_closure(specs, fail_fast):
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def _maybe_launch_pantsd(self):
"""Launches pantsd if configured to do so."""
if self._global_options.enable_pantsd:
# Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on.
with self._run_tracker.new_workunit(name='pantsd', labels=[WorkUnitLabel.SETUP]):
pantsd_launcher = PantsDaemonLauncher.Factory.global_instance().create(EngineInitializer)
pantsd_launcher.maybe_launch()
def _is_quiet(self):
return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain
def _setup_context(self):
self._maybe_launch_pantsd()
with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]):
self._expand_goals(self._requested_goals)
self._expand_specs(self._target_specs, self._fail_fast)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(self._global_options,
self._is_quiet(),
self._run_tracker)
context = Context(options=self._options,
run_tracker=self._run_tracker,
target_roots=self._targets,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
invalidation_report=invalidation_report)
return context, invalidation_report
def setup(self):
context, invalidation_report = self._setup_context()
return GoalRunner(context=context,
goals=self._goals,
kill_nailguns=self._kill_nailguns,
run_tracker=self._run_tracker,
invalidation_report=invalidation_report,
exiter=self._exiter)
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(
self.build_file_parser, FileSystemProjectTree(self.build_root))
self.build_graph = MutableBuildGraph(
address_mapper=self.address_mapper)
class GoalRunnerFactory(object):
def __init__(self,
root_dir,
options,
build_config,
run_tracker,
reporting,
exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
# Will be provided through context.address_mapper.build_ignore_patterns.
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._project_tree = self._get_project_tree(
self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config,
self._root_dir)
build_ignore_patterns = self._global_options.ignore_patterns or []
self._address_mapper = BuildFileAddressMapper(
self._build_file_parser,
self._project_tree,
build_ignore_patterns,
exclude_target_regexps=self._global_options.exclude_target_regexp)
self._build_graph = MutableBuildGraph(self._address_mapper)
def _get_project_tree(self, build_file_rev):
"""Creates the project tree for build files for use in a given pants run."""
if build_file_rev:
return ScmProjectTree(self._root_dir, get_scm(), build_file_rev)
else:
return FileSystemProjectTree(self._root_dir)
def _expand_goals(self, goals):
"""Check and populate the requested goals for a given run."""
for goal in goals:
try:
self._address_mapper.resolve_spec(goal)
logger.warning(
"Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
except AddressLookupError:
pass
if self._help_request:
help_printer = HelpPrinter(self._options)
result = help_printer.print_help()
self._exiter(result)
self._goals.extend([Goal.by_name(goal) for goal in goals])
def _expand_specs(self, spec_strs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name='parse',
labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag,
filter_for_tag))
# Parse all specs into unique Spec objects.
spec_parser = CmdLineSpecParser(self._root_dir)
specs = OrderedSet()
for spec_str in spec_strs:
specs.add(spec_parser.parse_spec(spec_str))
# Then scan them to generate unique Addresses.
for address in self._build_graph.inject_specs_closure(
specs, fail_fast):
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def _maybe_launch_pantsd(self):
"""Launches pantsd if configured to do so."""
if self._global_options.enable_pantsd:
# Avoid runtracker output if pantsd is disabled. Otherwise, show up to inform the user its on.
with self._run_tracker.new_workunit(name='pantsd',
labels=[WorkUnitLabel.SETUP]):
PantsDaemonLauncher.global_instance().maybe_launch()
def _is_quiet(self):
return any(
goal.has_task_of_type(QuietTaskMixin)
for goal in self._goals) or self._explain
def _setup_context(self):
self._maybe_launch_pantsd()
with self._run_tracker.new_workunit(name='setup',
labels=[WorkUnitLabel.SETUP]):
self._expand_goals(self._requested_goals)
self._expand_specs(self._target_specs, self._fail_fast)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(
self._global_options, self._is_quiet(), self._run_tracker)
context = Context(options=self._options,
run_tracker=self._run_tracker,
target_roots=self._targets,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
invalidation_report=invalidation_report)
return context, invalidation_report
def setup(self):
context, invalidation_report = self._setup_context()
return GoalRunner(context=context,
goals=self._goals,
kill_nailguns=self._kill_nailguns,
run_tracker=self._run_tracker,
invalidation_report=invalidation_report,
exiter=self._exiter)
def build_file_address_mapper(self):
return BuildFileAddressMapper(self.build_file_parser,
FileSystemProjectTree(self.build_root),
build_ignore_patterns=['subdir'])
def build_file_address_mapper(self):
return BuildFileAddressMapper(self.build_file_parser,
FileSystemProjectTree(self.build_root))
def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FileSystemProjectTree(self.build_root)) self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, self.project_tree,
build_ignore_patterns=self.build_ignore_patterns)
self.build_graph = MutableBuildGraph(address_mapper=self.address_mapper)
def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile) self.build_graph = BuildGraph(address_mapper=self.address_mapper)
class GoalRunnerFactory(object):
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp
)
def _get_buildfile_type(self, build_file_rev):
"""Selects the BuildFile type for use in a given pants run."""
if build_file_rev:
ScmBuildFile.set_rev(build_file_rev)
ScmBuildFile.set_scm(get_scm())
return ScmBuildFile
else:
return FilesystemBuildFile
def _expand_goals(self, goals):
"""Check and populate the requested goals for a given run."""
for goal in goals:
if self._address_mapper.from_cache(self._root_dir, goal, must_exist=False).file_exists():
logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
if self._help_request:
help_printer = HelpPrinter(self._options)
help_printer.print_help()
self._exiter(0)
self._goals.extend([Goal.by_name(goal) for goal in goals])
def _expand_specs(self, specs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag))
for spec in specs:
for address in self._spec_parser.parse_addresses(spec, fail_fast):
self._build_graph.inject_address_closure(address)
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def _is_quiet(self):
return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain
def _setup_context(self):
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self._run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self._address_mapper, self._build_file_parser)
with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]):
self._expand_goals(self._requested_goals)
self._expand_specs(self._target_specs, self._fail_fast)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(self._global_options,
self._is_quiet(),
self._run_tracker)
context = Context(options=self._options,
run_tracker=self._run_tracker,
target_roots=self._targets,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
spec_excludes=self._spec_excludes,
invalidation_report=invalidation_report)
return context, invalidation_report
def setup(self):
context, invalidation_report = self._setup_context()
return GoalRunner(context=context,
goals=self._goals,
kill_nailguns=self._kill_nailguns,
run_tracker=self._run_tracker,
invalidation_report=invalidation_report)
