def test_file_have_coding_utf8(self):
"""
Look through all .py files and ensure they start with the line '# coding=utf8'
"""
build_configuration = load_build_configuration_from_source()
build_file_parser = BuildFileParser(root_dir=get_buildroot(),
build_configuration=build_configuration)
address_mapper = BuildFileAddressMapper(build_file_parser)
build_graph = BuildGraph(address_mapper=address_mapper)
for address in address_mapper.scan_addresses(get_buildroot()):
build_graph.inject_address_closure(address)
def has_hand_coded_python_files(tgt):
return (not tgt.is_synthetic) and tgt.is_original and tgt.has_sources('.py')
nonconforming_files = []
for target in build_graph.targets(has_hand_coded_python_files):
for src in target.sources_relative_to_buildroot():
with open(os.path.join(get_buildroot(), src), 'r') as python_file:
coding_line = python_file.readline()
if '' == coding_line and os.path.basename(src) == '__init__.py':
continue
if coding_line[0:2] == '#!':
# Executable file: look for the coding on the second line.
coding_line = python_file.readline()
if not coding_line.rstrip() == '# coding=utf-8':
nonconforming_files.append(src)
if len(nonconforming_files) > 0:
self.fail('Expected these files to contain first line "# coding=utf8": '
+ str(nonconforming_files))
def test_transitive_closure_spec(self):
class FakeTarget(Target):
def __init__(self, *args, **kwargs):
super(FakeTarget, self).__init__(*args, payload=None, **kwargs)
with self.workspace('./BUILD', 'a/BUILD', 'a/b/BUILD') as root_dir:
with open(os.path.join(root_dir, './BUILD'), 'w') as build:
build.write(dedent('''
fake(name="foo",
dependencies=[
'a',
])
'''))
with open(os.path.join(root_dir, 'a/BUILD'), 'w') as build:
build.write(dedent('''
fake(name="a",
dependencies=[
'a/b:bat',
])
'''))
with open(os.path.join(root_dir, 'a/b/BUILD'), 'w') as build:
build.write(dedent('''
fake(name="bat")
'''))
parser = BuildFileParser(root_dir=root_dir,
exposed_objects={},
path_relative_utils={},
target_alias_map={'fake': FakeTarget})
build_graph = BuildGraph()
parser.inject_spec_closure_into_build_graph(':foo', build_graph)
self.assertEqual(len(build_graph.dependencies_of(SyntheticAddress(':foo'))), 1)
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'cache_key_gen_version': '0-test',
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
# We need a pants.ini, even if empty. get_buildroot() uses its presence.
self.create_file('pants.ini')
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration,
self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser,
FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def test_transitive_closure_spec(self):
with self.workspace('./BUILD', 'a/BUILD', 'a/b/BUILD') as root_dir:
with open(os.path.join(root_dir, './BUILD'), 'w') as build:
build.write(
dedent('''
fake(name="foo",
dependencies=[
'a',
])
'''))
with open(os.path.join(root_dir, 'a/BUILD'), 'w') as build:
build.write(
dedent('''
fake(name="a",
dependencies=[
'a/b:bat',
])
'''))
with open(os.path.join(root_dir, 'a/b/BUILD'), 'w') as build:
build.write(dedent('''
fake(name="bat")
'''))
build_configuration = BuildConfiguration()
build_configuration.register_target_alias('fake', Target)
parser = BuildFileParser(build_configuration, root_dir=root_dir)
build_graph = BuildGraph(self.address_mapper)
parser.inject_spec_closure_into_build_graph(':foo', build_graph)
self.assertEqual(
len(build_graph.dependencies_of(
SyntheticAddress.parse(':foo'))), 1)
def test_transitive_closure_spec(self):
with self.workspace('./BUILD', 'a/BUILD', 'a/b/BUILD') as root_dir:
with open(os.path.join(root_dir, './BUILD'), 'w') as build:
build.write(dedent('''
fake(name="foo",
dependencies=[
'a',
])
'''))
with open(os.path.join(root_dir, 'a/BUILD'), 'w') as build:
build.write(dedent('''
fake(name="a",
dependencies=[
'a/b:bat',
])
'''))
with open(os.path.join(root_dir, 'a/b/BUILD'), 'w') as build:
build.write(dedent('''
fake(name="bat")
'''))
build_configuration = BuildConfiguration()
build_configuration.register_target_alias('fake', Target)
parser = BuildFileParser(build_configuration, root_dir=root_dir)
build_graph = BuildGraph(self.address_mapper)
parser.inject_spec_closure_into_build_graph(':foo', build_graph)
self.assertEqual(len(build_graph.dependencies_of(SyntheticAddress.parse(':foo'))), 1)
def setUp(self):
self.real_build_root = BuildRoot().path
self.build_root = mkdtemp(suffix='_BUILD_ROOT')
BuildRoot().path = self.build_root
self.create_file('pants.ini')
self.build_file_parser = BuildFileParser(self.build_root)
self.build_file_parser.register_alias_groups(self.alias_groups)
self.build_graph = BuildGraph()
def setUp(self):
self.build_root = mkdtemp(suffix='_BUILD_ROOT')
BuildRoot().path = self.build_root
self.create_file('pants.ini')
self.build_file_parser = make_default_build_file_parser(
self.build_root)
self.build_graph = BuildGraph()
self.config = Config.load()
def setUp(self):
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration,
self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def scan(self, root=None):
"""Scans and parses all BUILD files found under ``root``.
Only BUILD files found under ``root`` are parsed as roots in the graph, but any dependencies of
targets parsed in the root tree's BUILD files will be followed and this may lead to BUILD files
outside of ``root`` being parsed and included in the returned build graph.
:param string root: The path to scan; by default, the build root.
:returns: A new build graph encapsulating the targets found.
"""
build_graph = BuildGraph(self.address_mapper)
for address in self.address_mapper.scan_addresses(root, spec_excludes=self.spec_excludes):
build_graph.inject_address_closure(address)
return build_graph
def scan(self, root=None):
"""Scans and parses all BUILD files found under ``root``.
Only BUILD files found under ``root`` are parsed as roots in the graph, but any dependencies of
targets parsed in the root tree's BUILD files will be followed and this may lead to BUILD files
outside of ``root`` being parsed and included in the returned build graph.
:param string root: The path to scan; by default, the build root.
:returns: A new build graph encapsulating the targets found.
"""
build_graph = BuildGraph(self.address_mapper)
for address in self.address_mapper.scan_addresses(root):
build_graph.inject_address_closure(address)
return build_graph
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
extra_paths = self.config.getlist('backends', 'python-path', [])
if extra_paths:
sys.path.extend(extra_paths)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scopes = ['']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
self.register_options()
self.run_tracker = RunTracker.from_config(self.config)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'cache_key_gen_version': '0-test',
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
# We need a pants.ini, even if empty. get_buildroot() uses its presence.
self.create_file('pants.ini')
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def test_register_bad_target_alias(self):
with self.assertRaises(TypeError):
self.build_configuration.register_target_alias('fred', object())
target = Target('fred', SyntheticAddress.parse('a:b'), BuildGraph(address_mapper=None))
with self.assertRaises(TypeError):
self.build_configuration.register_target_alias('fred', target)
def test_lookup_exception(self):
"""
There is code that depends on the fact that TransitiveLookupError is a subclass
of AddressLookupError
"""
self.assertIsInstance(BuildGraph.TransitiveLookupError(),
AddressLookupError)
def setUp(self):
self.build_root = mkdtemp(suffix='_BUILD_ROOT')
BuildRoot().path = self.build_root
self.create_file('pants.ini')
self.build_file_parser = make_default_build_file_parser(self.build_root)
self.build_graph = BuildGraph()
self.config = Config.load()
def test_create_bad_targets(self):
with self.assertRaises(TypeError):
BuildFileAliases(targets={'fred': object()})
target = Target('fred', Address.parse('a:b'),
BuildGraph(address_mapper=None))
with self.assertRaises(TypeError):
BuildFileAliases(targets={'fred': target})
def test_file_have_coding_utf8(self):
"""
Look through all .py files and ensure they start with the line '# coding=utf8'
"""
config = Config.load()
backend_packages = config.getlist('backends', 'packages')
build_configuration = load_plugins_and_backends(
backends=backend_packages)
build_file_parser = BuildFileParser(
root_dir=get_buildroot(), build_configuration=build_configuration)
address_mapper = BuildFileAddressMapper(build_file_parser)
build_graph = BuildGraph(address_mapper=address_mapper)
for address in address_mapper.scan_addresses(
get_buildroot(),
spec_excludes=[config.getdefault('pants_workdir')]):
build_graph.inject_address_closure(address)
def has_hand_coded_python_files(tgt):
return (not tgt.is_synthetic
) and tgt.is_original and tgt.has_sources('.py')
nonconforming_files = []
for target in build_graph.targets(has_hand_coded_python_files):
for src in target.sources_relative_to_buildroot():
with open(os.path.join(get_buildroot(), src),
'r') as python_file:
coding_line = python_file.readline()
if '' == coding_line and os.path.basename(
src) == '__init__.py':
continue
if coding_line[0:2] == '#!':
# Executable file: look for the coding on the second line.
coding_line = python_file.readline()
if not coding_line.rstrip() == '# coding=utf-8':
nonconforming_files.append(src)
if len(nonconforming_files) > 0:
self.fail(
'Expected these files to contain first line "# coding=utf8": '
+ str(nonconforming_files))
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.new_options = defaultdict(dict) # scope -> key-value mapping.
self.new_options[''] = {
'pants_workdir': os.path.join(self.build_root, '.pants.d'),
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist')
}
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration,
self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def setUp(self):
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration, self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp
)
def test_transitive_closure_spec(self):
class FakeTarget(Target):
def __init__(self, *args, **kwargs):
super(FakeTarget, self).__init__(*args, payload=None, **kwargs)
with self.workspace('./BUILD', 'a/BUILD', 'a/b/BUILD') as root_dir:
with open(os.path.join(root_dir, './BUILD'), 'w') as build:
build.write(
dedent('''
fake(name="foo",
dependencies=[
'a',
])
'''))
with open(os.path.join(root_dir, 'a/BUILD'), 'w') as build:
build.write(
dedent('''
fake(name="a",
dependencies=[
'a/b:bat',
])
'''))
with open(os.path.join(root_dir, 'a/b/BUILD'), 'w') as build:
build.write(dedent('''
fake(name="bat")
'''))
parser = BuildFileParser(root_dir=root_dir,
exposed_objects={},
path_relative_utils={},
target_alias_map={'fake': FakeTarget})
build_graph = BuildGraph()
parser.inject_spec_closure_into_build_graph(':foo', build_graph)
self.assertEqual(
len(build_graph.dependencies_of(SyntheticAddress(':foo'))), 1)
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.new_options = defaultdict(dict) # scope -> key-value mapping.
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration, self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': os.path.join(self.build_root, '.pants.d'),
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'cache_key_gen_version': '0-test'
}
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration, self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp,
)
def reset_build_graph(self): """Start over with a fresh build graph with no targets in it.""" self.address_mapper = BuildFileAddressMapper(self.build_file_parser) self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(
plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
# TODO: Create a 'Subsystem' abstraction instead of special-casing run-tracker here
# and in register_options().
known_scopes = ['', 'run-tracker']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(
known_scopes=known_scopes)
self.register_options()
# TODO(Eric Ayers) We are missing log messages. Set the log level earlier
# Enable standard python logging for code with no handle to a context/work-unit.
self._setup_logging() # NB: self.options are needed for this call.
self.run_tracker = RunTracker.from_options(self.options)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(
Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(
build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir,
relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def setup(self):
options_bootstrapper = OptionsBootstrapper()
bootstrap_options = options_bootstrapper.get_bootstrap_options()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = bootstrap_options.for_global_scope().plugins
backend_packages = bootstrap_options.for_global_scope().backend_packages
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [ScopeInfo.for_global_scope()]
# Add scopes for all needed subsystems.
subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
def _run():
"""
To add additional paths to sys.path, add a block to the config similar to the following:
[main]
roots: ['src/python/pants_internal/test/',]
"""
version = get_version()
if len(sys.argv) == 2 and sys.argv[1] == _VERSION_OPTION:
_do_exit(version)
root_dir = get_buildroot()
if not os.path.exists(root_dir):
_exit_and_fail('PANTS_BUILD_ROOT does not point to a valid path: %s' % root_dir)
if len(sys.argv) < 2:
argv = ['goal']
else:
argv = sys.argv[1:]
# Hack to force ./pants -h etc. to redirect to goal.
if argv[0] != 'goal' and set(['-h', '--help', 'help']).intersection(argv):
argv = ['goal'] + argv
parser = optparse.OptionParser(add_help_option=False, version=version)
RcFile.install_disable_rc_option(parser)
parser.add_option(_LOG_EXIT_OPTION,
action='store_true',
default=False,
dest='log_exit',
help = 'Log an exit message on success or failure.')
config = Config.load()
# XXX(wickman) This should be in the command goal, not un pants_exe.py!
run_tracker = RunTracker.from_config(config)
report = initial_reporting(config, run_tracker)
run_tracker.start(report)
url = run_tracker.run_info.get_info('report_url')
if url:
run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
run_tracker.log(Report.INFO, '(To run a reporting server: ./pants goal server)')
build_file_parser = BuildFileParser(root_dir=root_dir, run_tracker=run_tracker)
build_graph = BuildGraph(run_tracker=run_tracker)
additional_backends = config.getlist('backends', 'packages')
load_backends_from_source(build_file_parser, additional_backends=additional_backends)
command_class, command_args = _parse_command(root_dir, argv)
command = command_class(run_tracker,
root_dir,
parser,
command_args,
build_file_parser,
build_graph)
try:
if command.serialized():
def onwait(pid):
process = psutil.Process(pid)
print('Waiting on pants process %d (%s) to complete' %
(pid, ' '.join(process.cmdline)), file=sys.stderr)
return True
runfile = os.path.join(root_dir, '.pants.run')
lock = Lock.acquire(runfile, onwait=onwait)
else:
lock = Lock.unlocked()
try:
result = command.run(lock)
if result:
run_tracker.set_root_outcome(WorkUnit.FAILURE)
_do_exit(result)
except KeyboardInterrupt:
command.cleanup()
raise
finally:
lock.release()
finally:
run_tracker.end()
# Must kill nailguns only after run_tracker.end() is called, because there may still
# be pending background work that needs a nailgun.
if (hasattr(command.options, 'cleanup_nailguns') and command.options.cleanup_nailguns) \
or config.get('nailgun', 'autokill', default=False):
NailgunTask.killall(None)
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot."""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path."""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
relpath: The relative path to the directory from the build root.
"""
safe_mkdir(os.path.join(self.build_root, relpath))
def create_file(self, relpath, contents='', mode='w'):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
with safe_open(os.path.join(self.build_root, relpath),
mode=mode) as fp:
fp.write(contents)
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
derived_from=None,
**kwargs):
address = SyntheticAddress(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.inject_target(
target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from)
return target
def setUp(self):
self.build_root = mkdtemp(suffix='_BUILD_ROOT')
BuildRoot().path = self.build_root
self.create_file('pants.ini')
self.build_file_parser = make_default_build_file_parser(
self.build_root)
self.build_graph = BuildGraph()
self.config = Config.load()
def tearDown(self):
BuildRoot().reset()
SourceRoot.reset()
safe_rmtree(self.build_root)
BuildFileCache.clear()
self.build_file_parser.clear_registered_context()
def target(self, spec):
"""Resolves the given target address to a Target object.
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
if self.build_graph.get_target_from_spec(spec) is None:
self.build_file_parser.inject_spec_closure_into_build_graph(
spec, self.build_graph)
return self.build_graph.get_target_from_spec(spec)
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for provides, resources, java_sources
"""
self.create_files(path, sources)
self.add_to_build_file(
path,
dedent('''
%(target_type)s(name='%(name)s',
sources=%(sources)s,
%(resources)s
%(provides)s
%(java_sources)s
)
''' % dict(
target_type=target_type,
name=name,
sources=repr(sources or []),
resources=('resources=[pants("%s")],' % kwargs.get('resources')
if kwargs.has_key('resources') else ''),
provides=(dedent('''provides=artifact(
org = 'com.twitter',
name = '%s',
repo = pants('build-support/ivy:ivy')
),
''' %
name if kwargs.has_key('provides') else '')),
java_sources=('java_sources=[%s]' % ','.join(
map(lambda str_target: 'pants("%s")' % str_target,
kwargs.get('java_sources')))
if kwargs.has_key('java_sources') else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
return self.create_library(path, 'resources', name, sources)
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scopes = ['']
# Add scopes for global subsystem instances.
for subsystem_type in set(self.subsystems) | Goal.global_subsystem_types():
known_scopes.append(subsystem_type.qualify_scope(Options.GLOBAL_SCOPE))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
self.register_options()
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
build_file = self.address_mapper.from_cache(root_dir=self.root_dir, relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
class GoalRunnerFactory(object):
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp,
)
def _get_buildfile_type(self, build_file_rev):
"""Selects the BuildFile type for use in a given pants run."""
if build_file_rev:
ScmBuildFile.set_rev(build_file_rev)
ScmBuildFile.set_scm(get_scm())
return ScmBuildFile
else:
return FilesystemBuildFile
def _expand_goals(self, goals):
"""Check and populate the requested goals for a given run."""
for goal in goals:
if self._address_mapper.from_cache(self._root_dir, goal, must_exist=False).file_exists():
logger.warning(
"Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal)
)
if self._help_request:
help_printer = HelpPrinter(self._options)
help_printer.print_help()
self._exiter(0)
self._goals.extend([Goal.by_name(goal) for goal in goals])
def _expand_specs(self, specs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name="parse", labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag))
for spec in specs:
for address in self._spec_parser.parse_addresses(spec, fail_fast):
self._build_graph.inject_address_closure(address)
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def _is_quiet(self):
return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain
def _setup_context(self):
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self._run_tracker.new_workunit(name="bootstrap", labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self._address_mapper, self._build_file_parser)
with self._run_tracker.new_workunit(name="setup", labels=[WorkUnitLabel.SETUP]):
self._expand_goals(self._requested_goals)
self._expand_specs(self._target_specs, self._fail_fast)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(
self._global_options, self._is_quiet(), self._run_tracker
)
context = Context(
options=self._options,
run_tracker=self._run_tracker,
target_roots=self._targets,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
spec_excludes=self._spec_excludes,
invalidation_report=invalidation_report,
)
return context, invalidation_report
def setup(self):
context, invalidation_report = self._setup_context()
return GoalRunner(
context=context,
goals=self._goals,
kill_nailguns=self._kill_nailguns,
run_tracker=self._run_tracker,
invalidation_report=invalidation_report,
)
def setup(self, options_bootstrapper, working_set):
bootstrap_options = options_bootstrapper.get_bootstrap_options()
global_bootstrap_options = bootstrap_options.for_global_scope()
# The pants_version may be set in pants.ini for bootstrapping, so we make sure the user actually
# requested the version on the command line before deciding to print the version and exit.
if global_bootstrap_options.is_flagged('pants_version'):
print(global_bootstrap_options.pants_version)
self._exiter(0)
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(global_bootstrap_options)
# Add any extra paths to python path (e.g., for loading extra source backends).
for path in global_bootstrap_options.pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = global_bootstrap_options.plugins
backend_packages = global_bootstrap_options.backend_packages
build_configuration = load_plugins_and_backends(plugins, working_set, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [GlobalOptionsRegistrar.get_scope_info()]
# Add scopes for all needed subsystems.
subsystems = Subsystem.closure(set(self.subsystems) |
Goal.subsystems() |
build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(subsystem.get_scope_info())
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot."""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path."""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
relpath: The relative path to the directory from the build root.
"""
safe_mkdir(os.path.join(self.build_root, relpath))
def create_file(self, relpath, contents='', mode='w'):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
with safe_open(os.path.join(self.build_root, relpath), mode=mode) as fp:
fp.write(contents)
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
derived_from=None,
**kwargs):
address = SyntheticAddress.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.inject_target(target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from)
return target
@property
def alias_groups(self):
return {'target_aliases': {'dependencies': Dependencies}}
def setUp(self):
self.real_build_root = BuildRoot().path
self.build_root = mkdtemp(suffix='_BUILD_ROOT')
BuildRoot().path = self.build_root
self.create_file('pants.ini')
self.build_file_parser = BuildFileParser(self.build_root)
self.build_file_parser.register_alias_groups(self.alias_groups)
self.build_graph = BuildGraph()
def config(self, overrides=''):
"""Returns a config valid for the test build root."""
if overrides:
with temporary_file() as fp:
fp.write(overrides)
fp.close()
with environment_as(PANTS_CONFIG_OVERRIDE=fp.name):
return Config.load()
else:
return Config.load()
def create_options(self, **kwargs):
return dict(**kwargs)
def context(self, config='', options=None, target_roots=None, **kwargs):
return create_context(config=self.config(overrides=config),
options=self.create_options(**(options or {})),
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
**kwargs)
def tearDown(self):
BuildRoot().reset()
SourceRoot.reset()
safe_rmtree(self.build_root)
BuildFileCache.clear()
self.build_file_parser.clear_registered_context()
def target(self, spec):
"""Resolves the given target address to a Target object.
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
if self.build_graph.get_target_from_spec(spec) is None:
self.build_file_parser.inject_spec_closure_into_build_graph(spec, self.build_graph)
return self.build_graph.get_target_from_spec(spec)
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources and java_sources
"""
self.create_files(path, sources)
self.add_to_build_file(path, dedent('''
%(target_type)s(name='%(name)s',
sources=%(sources)s,
%(resources)s
%(java_sources)s
)
''' % dict(target_type=target_type,
name=name,
sources=repr(sources or []),
resources=('resources=[pants("%s")],' % kwargs.get('resources')
if kwargs.has_key('resources') else ''),
java_sources=('java_sources=[%s]'
% ','.join(map(lambda str_target: 'pants("%s")' % str_target,
kwargs.get('java_sources')))
if kwargs.has_key('java_sources') else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
return self.create_library(path, 'resources', name, sources)
@contextmanager
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
def __init__(self, run_tracker, root_dir, parser, args):
"""run_tracker: The (already opened) RunTracker to track this run with
root_dir: The root directory of the pants workspace
parser: an OptionParser
args: the subcommand arguments to parse"""
self.run_tracker = run_tracker
self.root_dir = root_dir
# TODO(pl): Gross that we're doing a local import here, but this has dependendencies
# way down into specific Target subclasses, and I'd prefer to make it explicit that this
# import is in many ways similar to to third party plugin imports below.
from pants.base.build_file_aliases import (
target_aliases, object_aliases,
applicative_path_relative_util_aliases,
partial_path_relative_util_aliases)
for alias, target_type in target_aliases.items():
BuildFileParser.register_target_alias(alias, target_type)
for alias, obj in object_aliases.items():
BuildFileParser.register_exposed_object(alias, obj)
for alias, util in applicative_path_relative_util_aliases.items():
BuildFileParser.register_applicative_path_relative_util(
alias, util)
for alias, util in partial_path_relative_util_aliases.items():
BuildFileParser.register_partial_path_relative_util(alias, util)
config = Config.load()
# TODO(pl): This is awful but I need something quick and dirty to support
# injection of third party Targets and tools into BUILD file context
plugins = config.getlist('plugins', 'entry_points', default=[])
for entry_point_spec in plugins:
module, entry_point = entry_point_spec.split(':')
plugin_module = __import__(module, globals(), locals(),
[entry_point], 0)
getattr(plugin_module, entry_point)(config)
self.build_file_parser = BuildFileParser(root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.build_graph = BuildGraph(run_tracker=self.run_tracker)
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
# try:
build_file = BuildFile(root_dir=self.root_dir, relpath=path)
self.build_file_parser.parse_build_file_family(build_file)
# except (TypeError, ImportError):
# error(path, include_traceback=True)
# except (IOError, SyntaxError):
# error(path)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
# Override the OptionParser's error with more useful output
def error(message=None, show_help=True):
if message:
print(message + '\n')
if show_help:
parser.print_help()
parser.exit(status=1)
parser.error = error
self.error = error
self.setup_parser(parser, args)
self.options, self.args = parser.parse_args(args)
self.parser = parser
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot."""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path."""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
return BuildFile(root_dir=self.build_root, relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
resources = None,
derived_from=None,
**kwargs):
address = SyntheticAddress.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
dependencies.extend(resources or [])
self.build_graph.inject_target(target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from)
return target
@property
def alias_groups(self):
return BuildFileAliases.create(targets={'target': Dependencies})
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.new_options = defaultdict(dict) # scope -> key-value mapping.
self.new_options[''] = {
'pants_workdir': os.path.join(self.build_root, '.pants.d'),
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist')
}
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration, self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def config(self, overrides=''):
"""Returns a config valid for the test build root."""
if overrides:
with temporary_file() as fp:
fp.write(overrides)
fp.close()
with environment_as(PANTS_CONFIG_OVERRIDE=fp.name):
return Config.load()
else:
return Config.load()
def set_new_options_for_scope(self, scope, **kwargs):
self.new_options[scope].update(kwargs)
def context(self, for_task_types=None, config='', options=None, new_options=None,
target_roots=None, **kwargs):
for_task_types = for_task_types or []
new_options = new_options or {}
new_option_values = defaultdict(dict)
# Get values for all new-style options registered by the tasks in for_task_types.
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError('You must set a scope on your task type before using it in tests.')
# We provide our own test-only registration implementation, bypassing argparse.
# When testing we set option values directly, so we don't care about cmd-line flags, config,
# env vars etc. In fact, for test isolation we explicitly don't want to look at those.
def register(*rargs, **rkwargs):
scoped_options = new_option_values[scope]
default = rkwargs.get('default')
if default is None and rkwargs.get('action') == 'append':
default = []
for flag_name in rargs:
option_name = flag_name.lstrip('-').replace('-', '_')
scoped_options[option_name] = default
task_type.register_options(register)
# Now override with any caller-specified values.
# TODO(benjy): Get rid of the new_options arg, and require tests to call set_new_options.
for scope, opts in new_options.items():
for key, val in opts.items():
new_option_values[scope][key] = val
for scope, opts in self.new_options.items():
for key, val in opts.items():
new_option_values[scope][key] = val
return create_context(config=self.config(overrides=config),
new_options = new_option_values,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
**kwargs)
def tearDown(self):
BuildRoot().reset()
SourceRoot.reset()
safe_rmtree(self.build_root)
BuildFile.clear_cache()
def target(self, spec):
"""Resolves the given target address to a Target object.
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
if self.build_graph.get_target_from_spec(spec) is None:
self.build_graph.inject_spec_closure(spec)
return self.build_graph.get_target_from_spec(spec)
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(path, dedent('''
%(target_type)s(name='%(name)s',
%(sources)s
%(resources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
''' % dict(target_type=target_type,
name=name,
sources=('sources=%s,' % repr(sources)
if sources else ''),
resources=('resources=["%s"],' % kwargs.get('resources')
if 'resources' in kwargs else ''),
java_sources=('java_sources=[%s],'
% ','.join(map(lambda str_target: '"%s"' % str_target,
kwargs.get('java_sources')))
if 'java_sources' in kwargs else ''),
provides=('provides=%s,' % kwargs.get('provides')
if 'provides' in kwargs else ''),
dependencies=('dependencies=%s,' % kwargs.get('dependencies')
if 'dependencies' in kwargs else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
return self.create_library(path, 'resources', name, sources)
@contextmanager
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
def populate_exclusive_groups(self, context, key=None, classpaths=None, target_predicate=None):
"""
Helps actual test cases to populate the "exclusives_groups" products data mapping
in the context, which holds the classpath values for targets.
:param context: The execution context where the products data mapping lives.
:param key: key for list of classpaths in the "exclusives_groups" data mapping.
None is the default value for most common cases.
:param classpaths: a list of classpath strings. If not specified, ['none'] will be used.
:param target_predicate: filter predicate for the context.targets(). For most common test
cases, None value is good enough.
"""
exclusives_mapping = ExclusivesMapping(context)
exclusives_mapping.set_base_classpath_for_group(
key or '<none>', [('default', entry) for entry in classpaths or ['none']])
exclusives_mapping._populate_target_maps(context.targets(target_predicate))
context.products.safe_create_data('exclusives_groups', lambda: exclusives_mapping)
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot."""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path."""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
cls = self.address_mapper._build_file_type
return cls(root_dir=self.build_root, relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
resources=None,
derived_from=None,
**kwargs):
address = SyntheticAddress.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
dependencies.extend(resources or [])
self.build_graph.inject_target(
target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from)
return target
@property
def alias_groups(self):
return BuildFileAliases.create(targets={'target': Dependencies})
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'cache_key_gen_version': '0-test',
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
# We need a pants.ini, even if empty. get_buildroot() uses its presence.
self.create_file('pants.ini')
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration,
self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser,
FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(self.build_file_parser,
FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def set_options_for_scope(self, scope, **kwargs):
self.options[scope].update(kwargs)
def context(self,
for_task_types=None,
options=None,
target_roots=None,
console_outstream=None,
workspace=None,
for_subsystems=None):
optionables = set()
extra_scopes = set()
for_subsystems = for_subsystems or ()
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError(
'You must set a scope on your subsystem type before using it in tests.'
)
optionables.add(subsystem)
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError(
'You must set a scope on your task type before using it in tests.'
)
optionables.add(task_type)
extra_scopes.update(
[si.scope for si in task_type.known_scope_infos()])
optionables.update(
Subsystem.closure(
set([
dep.subsystem_cls
for dep in task_type.subsystem_dependencies_iter()
]) | self._build_configuration.subsystems()))
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
option_values = create_options_for_optionables(
optionables, extra_scopes=extra_scopes, options=options)
context = create_context(options=option_values,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
Subsystem._options = context.options
return context
def tearDown(self):
SourceRoot.reset()
FilesystemBuildFile.clear_cache()
Subsystem.reset()
def target(self, spec):
"""Resolves the given target address to a Target object.
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
address = SyntheticAddress.parse(spec)
self.build_graph.inject_address_closure(address)
return self.build_graph.get_target(address)
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
addresses = list(spec_parser.parse_addresses(spec))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [
self.build_graph.get_target(address) for address in addresses
]
return targets
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(
path,
dedent('''
%(target_type)s(name='%(name)s',
%(sources)s
%(resources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
''' % dict(
target_type=target_type,
name=name,
sources=('sources=%s,' % repr(sources) if sources else ''),
resources=('resources=["%s"],' % kwargs.get('resources')
if 'resources' in kwargs else ''),
java_sources=('java_sources=[%s],' % ','.join(
map(lambda str_target: '"%s"' % str_target,
kwargs.get('java_sources')))
if 'java_sources' in kwargs else ''),
provides=('provides=%s,' % kwargs.get('provides')
if 'provides' in kwargs else ''),
dependencies=('dependencies=%s,' % kwargs.get('dependencies')
if 'dependencies' in kwargs else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
return self.create_library(path, 'resources', name, sources)
@contextmanager
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
def populate_compile_classpath(self, context, classpath=None):
"""
Helps actual test cases to populate the 'compile_classpath' products data mapping
in the context, which holds the classpath value for targets.
:param context: The execution context where the products data mapping lives.
:param classpath: a list of classpath strings. If not specified,
[os.path.join(self.buildroot, 'none')] will be used.
"""
classpath = classpath or [os.path.join(self.build_root, 'none')]
compile_classpaths = context.products.get_data('compile_classpath',
lambda: UnionProducts())
compile_classpaths.add_for_targets(context.targets(),
[('default', entry)
for entry in classpath])
@contextmanager
def add_data(self, context_products, data_type, target, *products):
make_products = lambda: defaultdict(MultipleRootedProducts)
data_by_target = context_products.get_data(data_type, make_products)
with temporary_dir() as outdir:
def create_product(product):
abspath = os.path.join(outdir, product)
with safe_open(abspath, mode='w') as fp:
fp.write(product)
return abspath
data_by_target[target].add_abs_paths(outdir,
map(create_product, products))
yield temporary_dir
@contextmanager
def add_products(self, context_products, product_type, target, *products):
product_mapping = context_products.get(product_type)
with temporary_dir() as outdir:
def create_product(product):
with safe_open(os.path.join(outdir, product), mode='w') as fp:
fp.write(product)
return product
product_mapping.add(target, outdir, map(create_product, products))
yield temporary_dir
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot."""
@classmethod
def setUpClass(cls):
"""Ensure that all code has a config to read from the cache.
TODO: Yuck. Get rid of this after plumbing options through in the right places.
"""
super(BaseTest, cls).setUpClass()
Config.cache(Config.load())
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path."""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
return BuildFile(root_dir=self.build_root, relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
resources = None,
derived_from=None,
**kwargs):
address = SyntheticAddress.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
dependencies.extend(resources or [])
self.build_graph.inject_target(target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from)
return target
@property
def alias_groups(self):
return BuildFileAliases.create(targets={'target': Dependencies})
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'cache_key_gen_version': '0-test',
}
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration, self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def set_options_for_scope(self, scope, **kwargs):
self.options[scope].update(kwargs)
def context(self, for_task_types=None, options=None, target_roots=None,
console_outstream=None, workspace=None):
for_task_types = for_task_types or []
options = options or {}
option_values = defaultdict(dict)
registered_global_subsystems = set()
# Get default values for all options registered by the tasks in for_task_types.
# TODO: This is clunky and somewhat repetitive of the real registration code.
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError('You must set a scope on your task type before using it in tests.')
# We provide our own test-only registration implementation, bypassing argparse.
# When testing we set option values directly, so we don't care about cmd-line flags, config,
# env vars etc. In fact, for test isolation we explicitly don't want to look at those.
def register_func(on_scope):
def register(*rargs, **rkwargs):
scoped_options = option_values[on_scope]
default = rkwargs.get('default')
if default is None and rkwargs.get('action') == 'append':
default = []
for flag_name in rargs:
option_name = flag_name.lstrip('-').replace('-', '_')
scoped_options[option_name] = default
# TODO: Set register.bootstrap here, for good measure?
register.scope = on_scope
return register
task_type.register_options(register_func(scope))
for subsystem in task_type.global_subsystems():
if subsystem not in registered_global_subsystems:
subsystem.register_options(register_func(subsystem.qualify_scope(Options.GLOBAL_SCOPE)))
registered_global_subsystems.add(subsystem)
for subsystem in task_type.task_subsystems():
subsystem.register_options(register_func(subsystem.qualify_scope(scope)))
# Now override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
for scope, opts in options.items():
for key, val in opts.items():
option_values[scope][key] = val
for scope, opts in self.options.items():
for key, val in opts.items():
option_values[scope][key] = val
# Make inner scopes inherit option values from their enclosing scopes.
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for scope in sorted(option_values.keys()):
if scope != Options.GLOBAL_SCOPE:
enclosing_scope = scope.rpartition('.')[0]
opts = option_values[scope]
for key, val in option_values.get(enclosing_scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
context = create_context(options=option_values,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
Subsystem._options = context.options
return context
def tearDown(self):
BuildRoot().reset()
SourceRoot.reset()
safe_rmtree(self.build_root)
BuildFile.clear_cache()
def target(self, spec):
"""Resolves the given target address to a Target object.
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
address = SyntheticAddress.parse(spec)
self.build_graph.inject_address_closure(address)
return self.build_graph.get_target(address)
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
addresses = list(spec_parser.parse_addresses(spec))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [self.build_graph.get_target(address) for address in addresses]
return targets
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(path, dedent('''
%(target_type)s(name='%(name)s',
%(sources)s
%(resources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
''' % dict(target_type=target_type,
name=name,
sources=('sources=%s,' % repr(sources)
if sources else ''),
resources=('resources=["%s"],' % kwargs.get('resources')
if 'resources' in kwargs else ''),
java_sources=('java_sources=[%s],'
% ','.join(map(lambda str_target: '"%s"' % str_target,
kwargs.get('java_sources')))
if 'java_sources' in kwargs else ''),
provides=('provides=%s,' % kwargs.get('provides')
if 'provides' in kwargs else ''),
dependencies=('dependencies=%s,' % kwargs.get('dependencies')
if 'dependencies' in kwargs else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
return self.create_library(path, 'resources', name, sources)
@contextmanager
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
def populate_compile_classpath(self, context, classpath=None):
"""
Helps actual test cases to populate the 'compile_classpath' products data mapping
in the context, which holds the classpath value for targets.
:param context: The execution context where the products data mapping lives.
:param classpath: a list of classpath strings. If not specified, ['none'] will be used.
"""
compile_classpaths = context.products.get_data('compile_classpath', lambda: UnionProducts())
compile_classpaths.add_for_targets(context.targets(), [('default', entry) for entry in classpath or ['none']])
@contextmanager
def add_data(self, context_products, data_type, target, *products):
make_products = lambda: defaultdict(MultipleRootedProducts)
data_by_target = context_products.get_data(data_type, make_products)
with temporary_dir() as outdir:
def create_product(product):
abspath = os.path.join(outdir, product)
with safe_open(abspath, mode='w') as fp:
fp.write(product)
return abspath
data_by_target[target].add_abs_paths(outdir, map(create_product, products))
yield temporary_dir
@contextmanager
def add_products(self, context_products, product_type, target, *products):
product_mapping = context_products.get(product_type)
with temporary_dir() as outdir:
def create_product(product):
with safe_open(os.path.join(outdir, product), mode='w') as fp:
fp.write(product)
return product
product_mapping.add(target, outdir, map(create_product, products))
yield temporary_dir
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot."""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path."""
if os.path.basename(relpath).startswith("BUILD"):
return relpath
else:
return os.path.join(relpath, "BUILD")
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents="", mode="wb"):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode="a")
cls = self.address_mapper._build_file_type
return cls(root_dir=self.build_root, relpath=self.build_path(relpath))
def make_target(self, spec="", target_type=Target, dependencies=None, derived_from=None, **kwargs):
"""Creates a target and injects it into the test's build graph.
:param string spec: The target address spec that locates this target.
:param type target_type: The concrete target subclass to create this new target from.
:param list dependencies: A list of target instances this new target depends on.
:param derived_from: The target this new target was derived from.
:type derived_from: :class:`pants.base.target.Target`
"""
address = Address.parse(spec)
target = target_type(name=address.target_name, address=address, build_graph=self.build_graph, **kwargs)
dependencies = dependencies or []
self.build_graph.inject_target(
target, dependencies=[dep.address for dep in dependencies], derived_from=derived_from
)
# TODO(John Sirois): This re-creates a little bit too much work done by the BuildGraph.
# Fixup the BuildGraph to deal with non BuildFileAddresses better and just leverage it.
for traversable_dependency_spec in target.traversable_dependency_specs:
traversable_dependency_address = Address.parse(traversable_dependency_spec, relative_to=address.spec_path)
traversable_dependency_target = self.build_graph.get_target(traversable_dependency_address)
if not traversable_dependency_target:
raise ValueError(
"Tests must make targets for traversable dependency specs ahead of them "
"being traversed, {} tried to traverse {} which does not exist.".format(
target, traversable_dependency_address
)
)
if traversable_dependency_target not in target.dependencies:
self.build_graph.inject_dependency(dependent=target.address, dependency=traversable_dependency_address)
target.mark_transitive_invalidation_hash_dirty()
return target
@property
def alias_groups(self):
# NB: In a normal pants deployment, 'target' is an alias for
# `pants.backend.core.targets.dependencies.Dependencies`. We avoid that dependency on the core
# backend here since the `BaseTest` is used by lower level tests in base and since the
# `Dependencies` type itself is nothing more than an alias for Target that carries along a
# pydoc for the BUILD dictionary.
return BuildFileAliases(targets={"target": Target})
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix="_BUILD_ROOT"))
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, ".pants.d")
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[""] = {
"pants_workdir": self.pants_workdir,
"pants_supportdir": os.path.join(self.build_root, "build-support"),
"pants_distdir": os.path.join(self.build_root, "dist"),
"pants_configdir": os.path.join(self.build_root, "config"),
"cache_key_gen_version": "0-test",
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
# We need a pants.ini, even if empty. get_buildroot() uses its presence.
self.create_file("pants.ini")
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def buildroot_files(self, relpath=None):
"""Returns the set of all files under the test build root.
:param string relpath: If supplied, only collect files from this subtree.
:returns: All file paths found.
:rtype: set
"""
def scan():
for root, dirs, files in os.walk(os.path.join(self.build_root, relpath or "")):
for f in files:
yield os.path.relpath(os.path.join(root, f), self.build_root)
return set(scan())
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def set_options_for_scope(self, scope, **kwargs):
self.options[scope].update(kwargs)
def context(
self,
for_task_types=None,
options=None,
target_roots=None,
console_outstream=None,
workspace=None,
for_subsystems=None,
):
optionables = set()
extra_scopes = set()
for_subsystems = for_subsystems or ()
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError("You must set a scope on your subsystem type before using it in tests.")
optionables.add(subsystem)
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError("You must set a scope on your task type before using it in tests.")
optionables.add(task_type)
extra_scopes.update([si.scope for si in task_type.known_scope_infos()])
optionables.update(
Subsystem.closure(
set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()])
| self._build_configuration.subsystems()
)
)
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
option_values = create_options_for_optionables(optionables, extra_scopes=extra_scopes, options=options)
context = create_context(
options=option_values,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace,
)
Subsystem._options = context.options
return context
def tearDown(self):
SourceRoot.reset()
FilesystemBuildFile.clear_cache()
Subsystem.reset()
def target(self, spec):
"""Resolves the given target address to a Target object.
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
address = Address.parse(spec)
self.build_graph.inject_address_closure(address)
return self.build_graph.get_target(address)
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
addresses = list(spec_parser.parse_addresses(spec))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [self.build_graph.get_target(address) for address in addresses]
return targets
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(
path,
dedent(
"""
%(target_type)s(name='%(name)s',
%(sources)s
%(resources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
"""
% dict(
target_type=target_type,
name=name,
sources=("sources=%s," % repr(sources) if sources else ""),
resources=('resources=["%s"],' % kwargs.get("resources") if "resources" in kwargs else ""),
java_sources=(
"java_sources=[%s],"
% ",".join(map(lambda str_target: '"%s"' % str_target, kwargs.get("java_sources")))
if "java_sources" in kwargs
else ""
),
provides=("provides=%s," % kwargs.get("provides") if "provides" in kwargs else ""),
dependencies=("dependencies=%s," % kwargs.get("dependencies") if "dependencies" in kwargs else ""),
)
),
)
return self.target("%s:%s" % (path, name))
def create_resources(self, path, name, *sources):
return self.create_library(path, "resources", name, sources)
@contextmanager
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
def populate_compile_classpath(self, context, classpath=None):
"""
Helps actual test cases to populate the 'compile_classpath' products data mapping
in the context, which holds the classpath value for targets.
:param context: The execution context where the products data mapping lives.
:param classpath: a list of classpath strings. If not specified,
[os.path.join(self.buildroot, 'none')] will be used.
"""
classpath = classpath or [os.path.join(self.build_root, "none")]
compile_classpaths = context.products.get_data("compile_classpath", lambda: UnionProducts())
compile_classpaths.add_for_targets(context.targets(), [("default", entry) for entry in classpath])
@contextmanager
def add_data(self, context_products, data_type, target, *products):
make_products = lambda: defaultdict(MultipleRootedProducts)
data_by_target = context_products.get_data(data_type, make_products)
with temporary_dir() as outdir:
def create_product(product):
abspath = os.path.join(outdir, product)
with safe_open(abspath, mode="w") as fp:
fp.write(product)
return abspath
data_by_target[target].add_abs_paths(outdir, map(create_product, products))
yield temporary_dir
@contextmanager
def add_products(self, context_products, product_type, target, *products):
product_mapping = context_products.get(product_type)
with temporary_dir() as outdir:
def create_product(product):
with safe_open(os.path.join(outdir, product), mode="w") as fp:
fp.write(product)
return product
product_mapping.add(target, outdir, map(create_product, products))
yield temporary_dir
class GoalRunner(object):
"""Lists installed goals or else executes a named goal."""
def __init__(self, root_dir):
"""
:param root_dir: The root directory of the pants workspace.
"""
self.root_dir = root_dir
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
sys.path.extend(bootstrap_options.for_global_scope().pythonpath)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scopes = ['']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
self.register_options()
self.run_tracker = RunTracker.from_config(self.config)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def get_spec_excludes(self):
# Note: Only call after register_options() has been called.
return [os.path.join(self.root_dir, spec_exclude)
for spec_exclude in self.options.for_global_scope().spec_excludes]
@property
def global_options(self):
return self.options.for_global_scope()
def register_options(self):
# Add a 'bootstrap' attribute to the register function, so that register_global can
# access the bootstrap option values.
def register_global(*args, **kwargs):
return self.options.register_global(*args, **kwargs)
register_global.bootstrap = self.options.bootstrap_option_values()
register_global_options(register_global)
for goal in Goal.all():
goal.register_options(self.options)
def _expand_goals_and_specs(self):
logger = logging.getLogger(__name__)
goals = self.options.goals
specs = self.options.target_specs
fail_fast = self.options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal, must_exist=False).exists():
logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
if self.options.is_help:
self.options.print_help(goals=goals)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper,
spec_excludes=self.get_spec_excludes(),
exclude_target_regexps=self.global_options.exclude_target_regexp)
with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
def run(self):
def fail():
self.run_tracker.set_root_outcome(WorkUnit.FAILURE)
kill_nailguns = self.options.for_global_scope().kill_nailguns
try:
result = self._do_run()
if result:
fail()
except KeyboardInterrupt:
fail()
# On ctrl-c we always kill nailguns, otherwise they might keep running
# some heavyweight compilation and gum up the system during a subsequent run.
kill_nailguns = True
raise
except Exception:
fail()
raise
finally:
self.run_tracker.end()
# Must kill nailguns only after run_tracker.end() is called, otherwise there may still
# be pending background work that needs a nailgun.
if kill_nailguns:
# TODO: This is JVM-specific and really doesn't belong here.
# TODO: Make this more selective? Only kill nailguns that affect state?
# E.g., checkstyle may not need to be killed.
NailgunTask.killall(log.info)
return result
def _do_run(self):
# TODO(John Sirois): Consider moving to straight python logging. The divide between the
# context/work-unit logging and standard python logging doesn't buy us anything.
# TODO(Eric Ayers) We are missing log messages. Set the log level earlier
# Enable standard python logging for code with no handle to a context/work-unit.
if self.global_options.level:
LogOptions.set_stderr_log_level((self.global_options.level or 'info').upper())
logdir = self.global_options.logdir or self.config.get('goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
prev_log_level = None
# If quiet, temporarily change stderr log level to kill init's output.
if self.global_options.quiet:
prev_log_level = LogOptions.loglevel_name(LogOptions.stderr_log_level())
# loglevel_name can fail, so only change level if we were able to get the current one.
if prev_log_level is not None:
LogOptions.set_stderr_log_level(LogOptions._LOG_LEVEL_NONE_KEY)
log.init('goals')
if prev_log_level is not None:
LogOptions.set_stderr_log_level(prev_log_level)
else:
log.init()
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
is_explain = self.global_options.explain
update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker)
context = Context(
config=self.config,
options=self.options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.get_spec_excludes()
)
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown))
return 1
engine = RoundEngine()
return engine.execute(context, self.goals)
class GoalRunner(object):
"""Lists installed goals or else executes a named goal."""
def __init__(self, root_dir):
"""
:param root_dir: The root directory of the pants workspace.
"""
self.root_dir = root_dir
@property
def subsystems(self):
# Subsystems used outside of any task.
return SourceRootBootstrapper, Reporting, RunTracker
def setup(self):
options_bootstrapper = OptionsBootstrapper()
bootstrap_options = options_bootstrapper.get_bootstrap_options()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = bootstrap_options.for_global_scope().plugins
backend_packages = bootstrap_options.for_global_scope().backend_packages
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [ScopeInfo.for_global_scope()]
# Add scopes for all needed subsystems.
subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
@property
def spec_excludes(self):
# Note: Only call after register_options() has been called.
return self.options.for_global_scope().spec_excludes
@property
def global_options(self):
return self.options.for_global_scope()
def register_options(self, subsystems):
# Standalone global options.
GlobalOptionsRegistrar.register_options_on_scope(self.options)
# Options for subsystems.
for subsystem in subsystems:
subsystem.register_options_on_scope(self.options)
# TODO(benjy): Should Goals be subsystems? Or should the entire goal-running mechanism
# be a subsystem?
for goal in Goal.all():
# Register task options.
goal.register_options(self.options)
def _expand_goals_and_specs(self):
goals = self.options.goals
specs = self.options.target_specs
fail_fast = self.options.for_global_scope().fail_fast
for goal in goals:
if self.address_mapper.from_cache(get_buildroot(), goal, must_exist=False).file_exists():
logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
if self.options.print_help_if_requested():
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper,
spec_excludes=self.spec_excludes,
exclude_target_regexps=self.global_options.exclude_target_regexp)
with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self.global_options.tag, filter_for_tag))
for spec in specs:
for address in spec_parser.parse_addresses(spec, fail_fast):
self.build_graph.inject_address_closure(address)
tgt = self.build_graph.get_target(address)
if tag_filter(tgt):
self.targets.append(tgt)
self.goals = [Goal.by_name(goal) for goal in goals]
def run(self):
def fail():
self.run_tracker.set_root_outcome(WorkUnit.FAILURE)
kill_nailguns = self.options.for_global_scope().kill_nailguns
try:
result = self._do_run()
if result:
fail()
except KeyboardInterrupt:
fail()
# On ctrl-c we always kill nailguns, otherwise they might keep running
# some heavyweight compilation and gum up the system during a subsequent run.
kill_nailguns = True
raise
except Exception:
fail()
raise
finally:
self.run_tracker.end()
# Must kill nailguns only after run_tracker.end() is called, otherwise there may still
# be pending background work that needs a nailgun.
if kill_nailguns:
# TODO: This is JVM-specific and really doesn't belong here.
# TODO: Make this more selective? Only kill nailguns that affect state?
# E.g., checkstyle may not need to be killed.
NailgunProcessGroup().killall()
return result
def _do_run(self):
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
is_explain = self.global_options.explain
if self.reporting.global_instance().get_options().invalidation_report:
invalidation_report = InvalidationReport()
else:
invalidation_report = None
self.reporting.update_reporting(self.global_options,
is_quiet_task() or is_explain,
self.run_tracker,
invalidation_report=invalidation_report)
context = Context(
options=self.options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.spec_excludes,
invalidation_report=invalidation_report
)
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): {}\n'.format(' '.join(goal.name for goal in unknown)))
return 1
engine = RoundEngine()
result = engine.execute(context, self.goals)
if invalidation_report:
invalidation_report.report()
return result
def _setup_logging(self, global_options):
# NB: quiet help says 'Squelches all console output apart from errors'.
level = 'ERROR' if global_options.quiet else global_options.level.upper()
setup_logging(level, log_dir=global_options.logdir)
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot."""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path."""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='w'):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
return BuildFile(root_dir=self.build_root, relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
derived_from=None,
**kwargs):
address = SyntheticAddress.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.inject_target(target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from)
return target
@property
def alias_groups(self):
return BuildFileAliases.create(targets={'target': Dependencies})
def setUp(self):
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration, self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def config(self, overrides=''):
"""Returns a config valid for the test build root."""
if overrides:
with temporary_file() as fp:
fp.write(overrides)
fp.close()
with environment_as(PANTS_CONFIG_OVERRIDE=fp.name):
return Config.load()
else:
return Config.load()
def create_options(self, **kwargs):
return dict(**kwargs)
def context(self, config='', options=None, target_roots=None, **kwargs):
return create_context(config=self.config(overrides=config),
options=self.create_options(**(options or {})),
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
**kwargs)
def tearDown(self):
BuildRoot().reset()
SourceRoot.reset()
safe_rmtree(self.build_root)
BuildFile.clear_cache()
def target(self, spec):
"""Resolves the given target address to a Target object.
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
if self.build_graph.get_target_from_spec(spec) is None:
self.build_graph.inject_spec_closure(spec)
return self.build_graph.get_target_from_spec(spec)
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources and java_sources
"""
self.create_files(path, sources)
self.add_to_build_file(path, dedent('''
%(target_type)s(name='%(name)s',
sources=%(sources)s,
%(resources)s
%(java_sources)s
)
''' % dict(target_type=target_type,
name=name,
sources=repr(sources or []),
resources=('resources=[pants("%s")],' % kwargs.get('resources')
if kwargs.has_key('resources') else ''),
java_sources=('java_sources=[%s]'
% ','.join(map(lambda str_target: 'pants("%s")' % str_target,
kwargs.get('java_sources')))
if kwargs.has_key('java_sources') else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
return self.create_library(path, 'resources', name, sources)
@contextmanager
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot."""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path."""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
relpath: The relative path to the directory from the build root.
"""
safe_mkdir(os.path.join(self.build_root, relpath))
def create_file(self, relpath, contents='', mode='w'):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
with safe_open(os.path.join(self.build_root, relpath), mode=mode) as fp:
fp.write(contents)
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
derived_from=None,
**kwargs):
address = SyntheticAddress(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
self.build_graph.inject_target(target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from)
return target
def setUp(self):
self.build_root = mkdtemp(suffix='_BUILD_ROOT')
BuildRoot().path = self.build_root
self.create_file('pants.ini')
self.build_file_parser = make_default_build_file_parser(self.build_root)
self.build_graph = BuildGraph()
self.config = Config.load()
def tearDown(self):
BuildRoot().reset()
SourceRoot.reset()
safe_rmtree(self.build_root)
BuildFileCache.clear()
self.build_file_parser.clear_registered_context()
def target(self, spec):
"""Resolves the given target address to a Target object.
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
if self.build_graph.get_target_from_spec(spec) is None:
self.build_file_parser.inject_spec_closure_into_build_graph(spec, self.build_graph)
return self.build_graph.get_target_from_spec(spec)
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for provides, resources, java_sources
"""
self.create_files(path, sources)
self.add_to_build_file(path, dedent('''
%(target_type)s(name='%(name)s',
sources=%(sources)s,
%(resources)s
%(provides)s
%(java_sources)s
)
''' % dict(target_type=target_type,
name=name,
sources=repr(sources or []),
resources=('resources=[pants("%s")],' % kwargs.get('resources')
if kwargs.has_key('resources') else ''),
provides=(dedent('''provides=artifact(
org = 'com.twitter',
name = '%s',
repo = pants('build-support/ivy:ivy')
),
'''% name if kwargs.has_key('provides') else '')),
java_sources=('java_sources=[%s]'
% ','.join(map(lambda str_target: 'pants("%s")' % str_target,
kwargs.get('java_sources')))
if kwargs.has_key('java_sources') else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
return self.create_library(path, 'resources', name, sources)
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot."""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path."""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
return BuildFile(root_dir=self.build_root,
relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
resources=None,
derived_from=None,
**kwargs):
address = SyntheticAddress.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
dependencies.extend(resources or [])
self.build_graph.inject_target(
target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from)
return target
@property
def alias_groups(self):
return BuildFileAliases.create(targets={'target': Dependencies})
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.new_options = defaultdict(dict) # scope -> key-value mapping.
self.new_options[''] = {
'pants_workdir': os.path.join(self.build_root, '.pants.d'),
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist')
}
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration,
self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def config(self, overrides=''):
"""Returns a config valid for the test build root."""
if overrides:
with temporary_file() as fp:
fp.write(overrides)
fp.close()
with environment_as(PANTS_CONFIG_OVERRIDE=fp.name):
return Config.load()
else:
return Config.load()
def set_new_options_for_scope(self, scope, **kwargs):
self.new_options[scope].update(kwargs)
def context(self,
for_task_types=None,
config='',
options=None,
new_options=None,
target_roots=None,
**kwargs):
for_task_types = for_task_types or []
new_options = new_options or {}
new_option_values = defaultdict(dict)
# Get values for all new-style options registered by the tasks in for_task_types.
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError(
'You must set a scope on your task type before using it in tests.'
)
# We provide our own test-only registration implementation, bypassing argparse.
# When testing we set option values directly, so we don't care about cmd-line flags, config,
# env vars etc. In fact, for test isolation we explicitly don't want to look at those.
def register(*rargs, **rkwargs):
scoped_options = new_option_values[scope]
default = rkwargs.get('default')
if default is None and rkwargs.get('action') == 'append':
default = []
for flag_name in rargs:
option_name = flag_name.lstrip('-').replace('-', '_')
scoped_options[option_name] = default
task_type.register_options(register)
# Now override with any caller-specified values.
# TODO(benjy): Get rid of the new_options arg, and require tests to call set_new_options.
for scope, opts in new_options.items():
for key, val in opts.items():
new_option_values[scope][key] = val
for scope, opts in self.new_options.items():
for key, val in opts.items():
new_option_values[scope][key] = val
return create_context(config=self.config(overrides=config),
new_options=new_option_values,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
**kwargs)
def tearDown(self):
BuildRoot().reset()
SourceRoot.reset()
safe_rmtree(self.build_root)
BuildFile.clear_cache()
def target(self, spec):
"""Resolves the given target address to a Target object.
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
if self.build_graph.get_target_from_spec(spec) is None:
self.build_graph.inject_spec_closure(spec)
return self.build_graph.get_target_from_spec(spec)
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(
path,
dedent('''
%(target_type)s(name='%(name)s',
%(sources)s
%(resources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
''' % dict(
target_type=target_type,
name=name,
sources=('sources=%s,' % repr(sources) if sources else ''),
resources=('resources=["%s"],' % kwargs.get('resources')
if 'resources' in kwargs else ''),
java_sources=('java_sources=[%s],' % ','.join(
map(lambda str_target: '"%s"' % str_target,
kwargs.get('java_sources')))
if 'java_sources' in kwargs else ''),
provides=('provides=%s,' % kwargs.get('provides')
if 'provides' in kwargs else ''),
dependencies=('dependencies=%s,' % kwargs.get('dependencies')
if 'dependencies' in kwargs else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
return self.create_library(path, 'resources', name, sources)
@contextmanager
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
def populate_exclusive_groups(self,
context,
key=None,
classpaths=None,
target_predicate=None):
"""
Helps actual test cases to populate the "exclusives_groups" products data mapping
in the context, which holds the classpath values for targets.
:param context: The execution context where the products data mapping lives.
:param key: key for list of classpaths in the "exclusives_groups" data mapping.
None is the default value for most common cases.
:param classpaths: a list of classpath strings. If not specified, ['none'] will be used.
:param target_predicate: filter predicate for the context.targets(). For most common test
cases, None value is good enough.
"""
exclusives_mapping = ExclusivesMapping(context)
exclusives_mapping.set_base_classpath_for_group(
key or '<none>',
[('default', entry) for entry in classpaths or ['none']])
exclusives_mapping._populate_target_maps(
context.targets(target_predicate))
context.products.safe_create_data('exclusives_groups',
lambda: exclusives_mapping)
class GoalRunner(object):
"""Lists installed goals or else executes a named goal."""
def __init__(self, root_dir):
"""
:param root_dir: The root directory of the pants workspace.
"""
self.root_dir = root_dir
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(
plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scopes = ['']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.new_options = options_bootstrapper.get_full_options(
known_scopes=known_scopes)
self.register_options()
self.run_tracker = RunTracker.from_config(self.config)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(
Report.INFO,
'(To run a reporting server: ./pants goal server)')
self.build_file_parser = BuildFileParser(
build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir,
relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def get_spec_excludes(self):
# Note: Only call after register_options() has been called.
return [
os.path.join(self.root_dir, spec_exclude) for spec_exclude in
self.new_options.for_global_scope().spec_excludes
]
@property
def global_options(self):
return self.new_options.for_global_scope()
def register_options(self):
# Add a 'bootstrap' attribute to the register function, so that register_global can
# access the bootstrap option values.
def register_global(*args, **kwargs):
return self.new_options.register_global(*args, **kwargs)
register_global.bootstrap = self.new_options.bootstrap_option_values()
register_global_options(register_global)
for goal in Goal.all():
goal.register_options(self.new_options)
def _expand_goals_and_specs(self):
logger = logging.getLogger(__name__)
goals = self.new_options.goals
specs = self.new_options.target_specs
fail_fast = self.new_options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal,
must_exist=False).exists():
logger.warning(
" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
if self.new_options.is_help:
self.new_options.print_help(goals=goals)
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(
self.root_dir,
self.address_mapper,
spec_excludes=self.get_spec_excludes())
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(
spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(
self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
def run(self):
def fail():
self.run_tracker.set_root_outcome(WorkUnit.FAILURE)
kill_nailguns = self.new_options.for_global_scope().kill_nailguns
try:
result = self._do_run()
if result:
fail()
except KeyboardInterrupt:
fail()
# On ctrl-c we always kill nailguns, otherwise they might keep running
# some heavyweight compilation and gum up the system during a subsequent run.
kill_nailguns = True
raise
except Exception:
fail()
raise
finally:
self.run_tracker.end()
# Must kill nailguns only after run_tracker.end() is called, otherwise there may still
# be pending background work that needs a nailgun.
if kill_nailguns:
# TODO: This is JVM-specific and really doesn't belong here.
# TODO: Make this more selective? Only kill nailguns that affect state?
# E.g., checkstyle may not need to be killed.
NailgunTask.killall(log.info)
return result
def _do_run(self):
# TODO(John Sirois): Consider moving to straight python logging. The divide between the
# context/work-unit logging and standard python logging doesn't buy us anything.
# Enable standard python logging for code with no handle to a context/work-unit.
if self.global_options.level:
LogOptions.set_stderr_log_level((self.global_options.level
or 'info').upper())
logdir = self.global_options.logdir or self.config.get(
'goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
prev_log_level = None
# If quiet, temporarily change stderr log level to kill init's output.
if self.global_options.quiet:
prev_log_level = LogOptions.loglevel_name(
LogOptions.stderr_log_level())
# loglevel_name can fail, so only change level if we were able to get the current one.
if prev_log_level is not None:
LogOptions.set_stderr_log_level(
LogOptions._LOG_LEVEL_NONE_KEY)
log.init('goals')
if prev_log_level is not None:
LogOptions.set_stderr_log_level(prev_log_level)
else:
log.init()
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
# Target specs are mapped to the patterns which match them, if any. This variable is a key for
# specs which don't match any exclusion regexes. We know it won't already be in the list of
# patterns, because the asterisks in its name make it an invalid regex.
_UNMATCHED_KEY = '** unmatched **'
def targets_by_pattern(targets, patterns):
mapping = defaultdict(list)
for target in targets:
matched_pattern = None
for pattern in patterns:
if re.search(pattern, target.address.spec) is not None:
matched_pattern = pattern
break
if matched_pattern is None:
mapping[_UNMATCHED_KEY].append(target)
else:
mapping[matched_pattern].append(target)
return mapping
is_explain = self.global_options.explain
update_reporting(self.global_options,
is_quiet_task() or is_explain, self.run_tracker)
if self.global_options.exclude_target_regexp:
excludes = self.global_options.exclude_target_regexp
log.debug('excludes:\n {excludes}'.format(
excludes='\n '.join(excludes)))
by_pattern = targets_by_pattern(self.targets, excludes)
self.targets = by_pattern[_UNMATCHED_KEY]
# The rest of this if-statement is just for debug logging.
log.debug('Targets after excludes: {targets}'.format(
targets=', '.join(t.address.spec for t in self.targets)))
excluded_count = sum(len(by_pattern[p]) for p in excludes)
log.debug('Excluded {count} target{plural}.'.format(
count=excluded_count,
plural=('s' if excluded_count != 1 else '')))
for pattern in excludes:
log.debug('Targets excluded by pattern {pattern}\n {targets}'.
format(pattern=pattern,
targets='\n '.join(
t.address.spec
for t in by_pattern[pattern])))
context = Context(config=self.config,
new_options=self.new_options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.get_spec_excludes())
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): %s\n' %
' '.join(goal.name for goal in unknown))
return 1
engine = RoundEngine()
return engine.execute(context, self.goals)
class GoalRunner(object):
"""Lists installed goals or else executes a named goal."""
def __init__(self, root_dir):
"""
:param root_dir: The root directory of the pants workspace.
"""
self.root_dir = root_dir
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(
plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
# TODO: Create a 'Subsystem' abstraction instead of special-casing run-tracker here
# and in register_options().
known_scopes = ['', 'run-tracker']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(
known_scopes=known_scopes)
self.register_options()
# TODO(Eric Ayers) We are missing log messages. Set the log level earlier
# Enable standard python logging for code with no handle to a context/work-unit.
self._setup_logging() # NB: self.options are needed for this call.
self.run_tracker = RunTracker.from_options(self.options)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(
Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(
build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir,
relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
@property
def spec_excludes(self):
# Note: Only call after register_options() has been called.
return self.options.for_global_scope().spec_excludes
@property
def global_options(self):
return self.options.for_global_scope()
def register_options(self):
# Add a 'bootstrap' attribute to the register function, so that register_global can
# access the bootstrap option values.
def register_global(*args, **kwargs):
return self.options.register_global(*args, **kwargs)
register_global.bootstrap = self.options.bootstrap_option_values()
register_global_options(register_global)
# This is the first case we have of non-task, non-global options.
# The current implementation special-cases RunTracker, and is temporary.
# In the near future it will be replaced with a 'Subsystem' abstraction.
# But for now this is useful for kicking the tires.
def register_run_tracker(*args, **kwargs):
self.options.register('run-tracker', *args, **kwargs)
RunTracker.register_options(register_run_tracker)
for goal in Goal.all():
goal.register_options(self.options)
def _expand_goals_and_specs(self):
goals = self.options.goals
specs = self.options.target_specs
fail_fast = self.options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal,
must_exist=False).exists():
logger.warning(
" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}."
.format(goal))
if self.options.print_help_if_requested():
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup',
labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(
self.root_dir,
self.address_mapper,
spec_excludes=self.spec_excludes,
exclude_target_regexps=self.global_options.
exclude_target_regexp)
with self.run_tracker.new_workunit(name='parse',
labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(
spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(
self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
def run(self):
def fail():
self.run_tracker.set_root_outcome(WorkUnit.FAILURE)
kill_nailguns = self.options.for_global_scope().kill_nailguns
try:
result = self._do_run()
if result:
fail()
except KeyboardInterrupt:
fail()
# On ctrl-c we always kill nailguns, otherwise they might keep running
# some heavyweight compilation and gum up the system during a subsequent run.
kill_nailguns = True
raise
except Exception:
fail()
raise
finally:
self.run_tracker.end()
# Must kill nailguns only after run_tracker.end() is called, otherwise there may still
# be pending background work that needs a nailgun.
if kill_nailguns:
# TODO: This is JVM-specific and really doesn't belong here.
# TODO: Make this more selective? Only kill nailguns that affect state?
# E.g., checkstyle may not need to be killed.
NailgunTask.killall()
return result
def _do_run(self):
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
is_explain = self.global_options.explain
update_reporting(self.global_options,
is_quiet_task() or is_explain, self.run_tracker)
context = Context(config=self.config,
options=self.options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.spec_excludes)
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): %s\n' %
' '.join(goal.name for goal in unknown))
return 1
engine = RoundEngine()
return engine.execute(context, self.goals)
def _setup_logging(self):
# TODO(John Sirois): Consider moving to straight python logging. The divide between the
# context/work-unit logging and standard python logging doesn't buy us anything.
# TODO(John Sirois): Support logging.config.fileConfig so a site can setup fine-grained
# logging control and we don't need to be the middleman plumbing an option for each python
# standard logging knob.
# NB: quiet help says 'Squelches all console output apart from errors'.
level = 'ERROR' if self.global_options.quiet else self.global_options.level.upper(
)
logging_config = {
'version':
1, # required and there is only a version 1 format so far.
'disable_existing_loggers': False
}
formatters_config = {'brief': {'format': '%(levelname)s] %(message)s'}}
handlers_config = {
'console': {
'class': 'logging.StreamHandler',
'formatter': 'brief', # defined above
'level': level
}
}
log_dir = self.global_options.logdir
if log_dir:
safe_mkdir(log_dir)
# This is close to but not quite glog format. Namely the leading levelname is not a single
# character and the fractional second is only to millis precision and not micros.
glog_date_format = '%m%d %H:%M:%S'
glog_format = (
'%(levelname)s %(asctime)s.%(msecs)d %(process)d %(filename)s:%(lineno)d] '
'%(message)s')
formatters_config['glog'] = {
'format': glog_format,
'datefmt': glog_date_format
}
handlers_config['file'] = {
'class': 'logging.handlers.RotatingFileHandler',
'formatter': 'glog', # defined above
'level': level,
'filename': os.path.join(log_dir, 'pants.log'),
'maxBytes': 10 * 1024 * 1024,
'backupCount': 4
}
logging_config['formatters'] = formatters_config
logging_config['handlers'] = handlers_config
logging_config['root'] = {
'level': level,
'handlers': handlers_config.keys()
}
logging.config.dictConfig(logging_config)
class GoalRunnerFactory(object):
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp
)
def _get_buildfile_type(self, build_file_rev):
"""Selects the BuildFile type for use in a given pants run."""
if build_file_rev:
ScmBuildFile.set_rev(build_file_rev)
ScmBuildFile.set_scm(get_scm())
return ScmBuildFile
else:
return FilesystemBuildFile
def _expand_goals(self, goals):
"""Check and populate the requested goals for a given run."""
for goal in goals:
if self._address_mapper.from_cache(self._root_dir, goal, must_exist=False).file_exists():
logger.warning("Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
if self._help_request:
help_printer = HelpPrinter(self._options)
help_printer.print_help()
self._exiter(0)
self._goals.extend([Goal.by_name(goal) for goal in goals])
def _expand_specs(self, specs, fail_fast):
"""Populate the BuildGraph and target list from a set of input specs."""
with self._run_tracker.new_workunit(name='parse', labels=[WorkUnitLabel.SETUP]):
def filter_for_tag(tag):
return lambda target: tag in map(str, target.tags)
tag_filter = wrap_filters(create_filters(self._tag, filter_for_tag))
for spec in specs:
for address in self._spec_parser.parse_addresses(spec, fail_fast):
self._build_graph.inject_address_closure(address)
target = self._build_graph.get_target(address)
if tag_filter(target):
self._targets.append(target)
def _is_quiet(self):
return any(goal.has_task_of_type(QuietTaskMixin) for goal in self._goals) or self._explain
def _setup_context(self):
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self._run_tracker.new_workunit(name='bootstrap', labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self._address_mapper, self._build_file_parser)
with self._run_tracker.new_workunit(name='setup', labels=[WorkUnitLabel.SETUP]):
self._expand_goals(self._requested_goals)
self._expand_specs(self._target_specs, self._fail_fast)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(self._global_options,
self._is_quiet(),
self._run_tracker)
context = Context(options=self._options,
run_tracker=self._run_tracker,
target_roots=self._targets,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
spec_excludes=self._spec_excludes,
invalidation_report=invalidation_report)
return context, invalidation_report
def setup(self):
context, invalidation_report = self._setup_context()
return GoalRunner(context=context,
goals=self._goals,
kill_nailguns=self._kill_nailguns,
run_tracker=self._run_tracker,
invalidation_report=invalidation_report)
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot."""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path."""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
cls = self.address_mapper._build_file_type
return cls(root_dir=self.build_root, relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
resources=None,
derived_from=None,
**kwargs):
address = SyntheticAddress.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
dependencies.extend(resources or [])
self.build_graph.inject_target(target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from)
return target
@property
def alias_groups(self):
return BuildFileAliases.create(targets={'target': Dependencies})
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'cache_key_gen_version': '0-test',
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
# We need a pants.ini, even if empty. get_buildroot() uses its presence.
self.create_file('pants.ini')
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration, self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def set_options_for_scope(self, scope, **kwargs):
self.options[scope].update(kwargs)
def context(self, for_task_types=None, options=None, target_roots=None, console_outstream=None,
workspace=None, for_subsystems=None):
optionables = set()
extra_scopes = set()
for_subsystems = for_subsystems or ()
for subsystem in for_subsystems:
if subsystem.options_scope is None:
raise TaskError('You must set a scope on your subsystem type before using it in tests.')
optionables.add(subsystem)
for_task_types = for_task_types or ()
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError('You must set a scope on your task type before using it in tests.')
optionables.add(task_type)
extra_scopes.update([si.scope for si in task_type.known_scope_infos()])
optionables.update(Subsystem.closure(
set([dep.subsystem_cls for dep in task_type.subsystem_dependencies_iter()]) |
self._build_configuration.subsystems()))
# Now default the option values and override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
options = options.copy() if options else {}
for s, opts in self.options.items():
scoped_opts = options.setdefault(s, {})
scoped_opts.update(opts)
option_values = create_options_for_optionables(optionables,
extra_scopes=extra_scopes,
options=options)
context = create_context(options=option_values,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
Subsystem._options = context.options
return context
def tearDown(self):
SourceRoot.reset()
FilesystemBuildFile.clear_cache()
Subsystem.reset()
def target(self, spec):
"""Resolves the given target address to a Target object.
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
address = SyntheticAddress.parse(spec)
self.build_graph.inject_address_closure(address)
return self.build_graph.get_target(address)
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
addresses = list(spec_parser.parse_addresses(spec))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [self.build_graph.get_target(address) for address in addresses]
return targets
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(path, dedent('''
%(target_type)s(name='%(name)s',
%(sources)s
%(resources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
''' % dict(target_type=target_type,
name=name,
sources=('sources=%s,' % repr(sources)
if sources else ''),
resources=('resources=["%s"],' % kwargs.get('resources')
if 'resources' in kwargs else ''),
java_sources=('java_sources=[%s],'
% ','.join(map(lambda str_target: '"%s"' % str_target,
kwargs.get('java_sources')))
if 'java_sources' in kwargs else ''),
provides=('provides=%s,' % kwargs.get('provides')
if 'provides' in kwargs else ''),
dependencies=('dependencies=%s,' % kwargs.get('dependencies')
if 'dependencies' in kwargs else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
return self.create_library(path, 'resources', name, sources)
@contextmanager
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
def populate_compile_classpath(self, context, classpath=None):
"""
Helps actual test cases to populate the 'compile_classpath' products data mapping
in the context, which holds the classpath value for targets.
:param context: The execution context where the products data mapping lives.
:param classpath: a list of classpath strings. If not specified,
[os.path.join(self.buildroot, 'none')] will be used.
"""
classpath = classpath or [os.path.join(self.build_root, 'none')]
compile_classpaths = context.products.get_data('compile_classpath', lambda: UnionProducts())
compile_classpaths.add_for_targets(context.targets(),
[('default', entry) for entry in classpath])
@contextmanager
def add_data(self, context_products, data_type, target, *products):
make_products = lambda: defaultdict(MultipleRootedProducts)
data_by_target = context_products.get_data(data_type, make_products)
with temporary_dir() as outdir:
def create_product(product):
abspath = os.path.join(outdir, product)
with safe_open(abspath, mode='w') as fp:
fp.write(product)
return abspath
data_by_target[target].add_abs_paths(outdir, map(create_product, products))
yield temporary_dir
@contextmanager
def add_products(self, context_products, product_type, target, *products):
product_mapping = context_products.get(product_type)
with temporary_dir() as outdir:
def create_product(product):
with safe_open(os.path.join(outdir, product), mode='w') as fp:
fp.write(product)
return product
product_mapping.add(target, outdir, map(create_product, products))
yield temporary_dir
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot."""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path."""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='wb'):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
cls = self.address_mapper._build_file_type
return cls(root_dir=self.build_root, relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
resources=None,
derived_from=None,
**kwargs):
address = SyntheticAddress.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
dependencies.extend(resources or [])
self.build_graph.inject_target(
target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from)
return target
@property
def alias_groups(self):
return BuildFileAliases.create(targets={'target': Dependencies})
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'cache_key_gen_version': '0-test',
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
# We need a pants.ini, even if empty. get_buildroot() uses its presence.
self.create_file('pants.ini')
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration,
self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser,
FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(self.build_file_parser,
FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def set_options_for_scope(self, scope, **kwargs):
self.options[scope].update(kwargs)
def context(self,
for_task_types=None,
options=None,
target_roots=None,
console_outstream=None,
workspace=None):
for_task_types = for_task_types or []
options = options or {}
option_values = defaultdict(dict)
registered_subsystems = set()
bootstrap_option_values = None # We fill these in after registering bootstrap options.
# We provide our own test-only registration implementation, bypassing argparse.
# When testing we set option values directly, so we don't care about cmd-line flags, config,
# env vars etc. In fact, for test isolation we explicitly don't want to look at those.
# All this does is make the names available in code, with the default values.
# Individual tests can then override the option values they care about.
def register_func(on_scope):
def register(*rargs, **rkwargs):
scoped_options = option_values[on_scope]
default = rkwargs.get('default')
if default is None and rkwargs.get('action') == 'append':
default = []
for flag_name in rargs:
option_name = flag_name.lstrip('-').replace('-', '_')
scoped_options[option_name] = default
register.bootstrap = bootstrap_option_values
register.scope = on_scope
return register
# TODO: This sequence is a bit repetitive of the real registration sequence.
# Register bootstrap options and grab their default values for use in subsequent registration.
GlobalOptionsRegistrar.register_bootstrap_options(
register_func(Options.GLOBAL_SCOPE))
bootstrap_option_values = create_option_values(
copy.copy(option_values[Options.GLOBAL_SCOPE]))
# Now register the full global scope options.
GlobalOptionsRegistrar.register_options(
register_func(Options.GLOBAL_SCOPE))
# Now register task and subsystem options for relevant tasks.
for task_type in for_task_types:
scope = task_type.options_scope
if scope is None:
raise TaskError(
'You must set a scope on your task type before using it in tests.'
)
task_type.register_options(register_func(scope))
for subsystem in (set(task_type.global_subsystems())
| set(task_type.task_subsystems())
| self._build_configuration.subsystems()):
if subsystem not in registered_subsystems:
subsystem.register_options(
register_func(subsystem.options_scope))
registered_subsystems.add(subsystem)
# Now default option values override with any caller-specified values.
# TODO(benjy): Get rid of the options arg, and require tests to call set_options.
for scope, opts in options.items():
for key, val in opts.items():
option_values[scope][key] = val
for scope, opts in self.options.items():
for key, val in opts.items():
option_values[scope][key] = val
# Make inner scopes inherit option values from their enclosing scopes.
all_scopes = set(option_values.keys())
for task_type in for_task_types: # Make sure we know about pre-task subsystem scopes.
all_scopes.update(
[si.scope for si in task_type.known_scope_infos()])
# Iterating in sorted order guarantees that we see outer scopes before inner scopes,
# and therefore only have to inherit from our immediately enclosing scope.
for scope in sorted(all_scopes):
if scope != Options.GLOBAL_SCOPE:
enclosing_scope = scope.rpartition('.')[0]
opts = option_values[scope]
for key, val in option_values.get(enclosing_scope, {}).items():
if key not in opts: # Inner scope values override the inherited ones.
opts[key] = val
context = create_context(options=option_values,
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
console_outstream=console_outstream,
workspace=workspace)
Subsystem._options = context.options
return context
def tearDown(self):
SourceRoot.reset()
FilesystemBuildFile.clear_cache()
def target(self, spec):
"""Resolves the given target address to a Target object.
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
address = SyntheticAddress.parse(spec)
self.build_graph.inject_address_closure(address)
return self.build_graph.get_target(address)
def targets(self, spec):
"""Resolves a target spec to one or more Target objects.
spec: Either BUILD target address or else a target glob using the siblings ':' or
descendants '::' suffixes.
Returns the set of all Targets found.
"""
spec_parser = CmdLineSpecParser(self.build_root, self.address_mapper)
addresses = list(spec_parser.parse_addresses(spec))
for address in addresses:
self.build_graph.inject_address_closure(address)
targets = [
self.build_graph.get_target(address) for address in addresses
]
return targets
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources=None, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources, java_sources, provides
and dependencies.
"""
if sources:
self.create_files(path, sources)
self.add_to_build_file(
path,
dedent('''
%(target_type)s(name='%(name)s',
%(sources)s
%(resources)s
%(java_sources)s
%(provides)s
%(dependencies)s
)
''' % dict(
target_type=target_type,
name=name,
sources=('sources=%s,' % repr(sources) if sources else ''),
resources=('resources=["%s"],' % kwargs.get('resources')
if 'resources' in kwargs else ''),
java_sources=('java_sources=[%s],' % ','.join(
map(lambda str_target: '"%s"' % str_target,
kwargs.get('java_sources')))
if 'java_sources' in kwargs else ''),
provides=('provides=%s,' % kwargs.get('provides')
if 'provides' in kwargs else ''),
dependencies=('dependencies=%s,' % kwargs.get('dependencies')
if 'dependencies' in kwargs else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
return self.create_library(path, 'resources', name, sources)
@contextmanager
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
def populate_compile_classpath(self, context, classpath=None):
"""
Helps actual test cases to populate the 'compile_classpath' products data mapping
in the context, which holds the classpath value for targets.
:param context: The execution context where the products data mapping lives.
:param classpath: a list of classpath strings. If not specified, [os.path.join(self.buildroot, 'none')] will be used.
"""
classpath = classpath or [os.path.join(self.build_root, 'none')]
compile_classpaths = context.products.get_data('compile_classpath',
lambda: UnionProducts())
compile_classpaths.add_for_targets(context.targets(),
[('default', entry)
for entry in classpath])
@contextmanager
def add_data(self, context_products, data_type, target, *products):
make_products = lambda: defaultdict(MultipleRootedProducts)
data_by_target = context_products.get_data(data_type, make_products)
with temporary_dir() as outdir:
def create_product(product):
abspath = os.path.join(outdir, product)
with safe_open(abspath, mode='w') as fp:
fp.write(product)
return abspath
data_by_target[target].add_abs_paths(outdir,
map(create_product, products))
yield temporary_dir
@contextmanager
def add_products(self, context_products, product_type, target, *products):
product_mapping = context_products.get(product_type)
with temporary_dir() as outdir:
def create_product(product):
with safe_open(os.path.join(outdir, product), mode='w') as fp:
fp.write(product)
return product
product_mapping.add(target, outdir, map(create_product, products))
yield temporary_dir
class GoalRunner(object):
"""Lists installed goals or else executes a named goal."""
def __init__(self, root_dir):
"""
:param root_dir: The root directory of the pants workspace.
"""
self.root_dir = root_dir
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
# TODO: Create a 'Subsystem' abstraction instead of special-casing run-tracker here
# and in register_options().
known_scopes = ['', 'run-tracker']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scopes=known_scopes)
self.register_options()
self.run_tracker = RunTracker.from_options(self.options)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals', 'bootstrap_buildfiles', default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir, relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
@property
def spec_excludes(self):
# Note: Only call after register_options() has been called.
return self.options.for_global_scope().spec_excludes
@property
def global_options(self):
return self.options.for_global_scope()
def register_options(self):
# Add a 'bootstrap' attribute to the register function, so that register_global can
# access the bootstrap option values.
def register_global(*args, **kwargs):
return self.options.register_global(*args, **kwargs)
register_global.bootstrap = self.options.bootstrap_option_values()
register_global_options(register_global)
# This is the first case we have of non-task, non-global options.
# The current implementation special-cases RunTracker, and is temporary.
# In the near future it will be replaced with a 'Subsystem' abstraction.
# But for now this is useful for kicking the tires.
def register_run_tracker(*args, **kwargs):
self.options.register('run-tracker', *args, **kwargs)
RunTracker.register_options(register_run_tracker)
for goal in Goal.all():
goal.register_options(self.options)
def _expand_goals_and_specs(self):
goals = self.options.goals
specs = self.options.target_specs
fail_fast = self.options.for_global_scope().fail_fast
for goal in goals:
if BuildFile.from_cache(get_buildroot(), goal, must_exist=False).exists():
logger.warning(" Command-line argument '{0}' is ambiguous and was assumed to be "
"a goal. If this is incorrect, disambiguate it with ./{0}.".format(goal))
if self.options.print_help_if_requested():
sys.exit(0)
self.requested_goals = goals
with self.run_tracker.new_workunit(name='setup', labels=[WorkUnit.SETUP]):
spec_parser = CmdLineSpecParser(self.root_dir, self.address_mapper,
spec_excludes=self.spec_excludes,
exclude_target_regexps=self.global_options.exclude_target_regexp)
with self.run_tracker.new_workunit(name='parse', labels=[WorkUnit.SETUP]):
for spec in specs:
for address in spec_parser.parse_addresses(spec, fail_fast):
self.build_graph.inject_address_closure(address)
self.targets.append(self.build_graph.get_target(address))
self.goals = [Goal.by_name(goal) for goal in goals]
def run(self):
def fail():
self.run_tracker.set_root_outcome(WorkUnit.FAILURE)
kill_nailguns = self.options.for_global_scope().kill_nailguns
try:
result = self._do_run()
if result:
fail()
except KeyboardInterrupt:
fail()
# On ctrl-c we always kill nailguns, otherwise they might keep running
# some heavyweight compilation and gum up the system during a subsequent run.
kill_nailguns = True
raise
except Exception:
fail()
raise
finally:
self.run_tracker.end()
# Must kill nailguns only after run_tracker.end() is called, otherwise there may still
# be pending background work that needs a nailgun.
if kill_nailguns:
# TODO: This is JVM-specific and really doesn't belong here.
# TODO: Make this more selective? Only kill nailguns that affect state?
# E.g., checkstyle may not need to be killed.
NailgunTask.killall()
return result
def _do_run(self):
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
is_explain = self.global_options.explain
update_reporting(self.global_options, is_quiet_task() or is_explain, self.run_tracker)
context = Context(
config=self.config,
options=self.options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.spec_excludes
)
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): %s\n' % ' '.join(goal.name for goal in unknown))
return 1
engine = RoundEngine()
return engine.execute(context, self.goals)
def _setup_logging(self, global_options):
# NB: quiet help says 'Squelches all console output apart from errors'.
level = 'ERROR' if global_options.quiet else global_options.level.upper()
setup_logging(level, log_dir=global_options.logdir)
class BaseTest(unittest.TestCase):
"""A baseclass useful for tests requiring a temporary buildroot."""
def build_path(self, relpath):
"""Returns the canonical BUILD file path for the given relative build path."""
if os.path.basename(relpath).startswith('BUILD'):
return relpath
else:
return os.path.join(relpath, 'BUILD')
def create_dir(self, relpath):
"""Creates a directory under the buildroot.
relpath: The relative path to the directory from the build root.
"""
path = os.path.join(self.build_root, relpath)
safe_mkdir(path)
return path
def create_file(self, relpath, contents='', mode='w'):
"""Writes to a file under the buildroot.
relpath: The relative path to the file from the build root.
contents: A string containing the contents of the file - '' by default..
mode: The mode to write to the file in - over-write by default.
"""
path = os.path.join(self.build_root, relpath)
with safe_open(path, mode=mode) as fp:
fp.write(contents)
return path
def add_to_build_file(self, relpath, target):
"""Adds the given target specification to the BUILD file at relpath.
relpath: The relative path to the BUILD file from the build root.
target: A string containing the target definition as it would appear in a BUILD file.
"""
self.create_file(self.build_path(relpath), target, mode='a')
return BuildFile(root_dir=self.build_root,
relpath=self.build_path(relpath))
def make_target(self,
spec='',
target_type=Target,
dependencies=None,
resources=None,
derived_from=None,
**kwargs):
address = SyntheticAddress.parse(spec)
target = target_type(name=address.target_name,
address=address,
build_graph=self.build_graph,
**kwargs)
dependencies = dependencies or []
dependencies.extend(resources or [])
self.build_graph.inject_target(
target,
dependencies=[dep.address for dep in dependencies],
derived_from=derived_from)
return target
@property
def alias_groups(self):
return BuildFileAliases.create(targets={'target': Dependencies})
def setUp(self):
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration,
self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def config(self, overrides=''):
"""Returns a config valid for the test build root."""
if overrides:
with temporary_file() as fp:
fp.write(overrides)
fp.close()
with environment_as(PANTS_CONFIG_OVERRIDE=fp.name):
return Config.load()
else:
return Config.load()
def create_options(self, **kwargs):
return dict(**kwargs)
def context(self, config='', options=None, target_roots=None, **kwargs):
return create_context(config=self.config(overrides=config),
options=self.create_options(**(options or {})),
target_roots=target_roots,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
**kwargs)
def tearDown(self):
BuildRoot().reset()
SourceRoot.reset()
safe_rmtree(self.build_root)
BuildFile.clear_cache()
def target(self, spec):
"""Resolves the given target address to a Target object.
address: The BUILD target address to resolve.
Returns the corresponding Target or else None if the address does not point to a defined Target.
"""
if self.build_graph.get_target_from_spec(spec) is None:
self.build_graph.inject_spec_closure(spec)
return self.build_graph.get_target_from_spec(spec)
def create_files(self, path, files):
"""Writes to a file under the buildroot with contents same as file name.
path: The relative path to the file from the build root.
files: List of file names.
"""
for f in files:
self.create_file(os.path.join(path, f), contents=f)
def create_library(self, path, target_type, name, sources, **kwargs):
"""Creates a library target of given type at the BUILD file at path with sources
path: The relative path to the BUILD file from the build root.
target_type: valid pants target type.
name: Name of the library target.
sources: List of source file at the path relative to path.
**kwargs: Optional attributes that can be set for any library target.
Currently it includes support for resources and java_sources
"""
self.create_files(path, sources)
self.add_to_build_file(
path,
dedent('''
%(target_type)s(name='%(name)s',
sources=%(sources)s,
%(resources)s
%(java_sources)s
)
''' % dict(
target_type=target_type,
name=name,
sources=repr(sources or []),
resources=('resources=[pants("%s")],' % kwargs.get('resources')
if kwargs.has_key('resources') else ''),
java_sources=('java_sources=[%s]' % ','.join(
map(lambda str_target: 'pants("%s")' % str_target,
kwargs.get('java_sources')))
if kwargs.has_key('java_sources') else ''),
)))
return self.target('%s:%s' % (path, name))
def create_resources(self, path, name, *sources):
return self.create_library(path, 'resources', name, sources)
@contextmanager
def workspace(self, *buildfiles):
with temporary_dir() as root_dir:
with BuildRoot().temporary(root_dir):
with pushd(root_dir):
for buildfile in buildfiles:
touch(os.path.join(root_dir, buildfile))
yield os.path.realpath(root_dir)
def populate_exclusive_groups(self,
context,
key=None,
classpaths=None,
target_predicate=None):
"""
Helps actual test cases to populate the "exclusives_groups" products data mapping
in the context, which holds the classpath values for targets.
:param context: The execution context where the products data mapping lives.
:param key: key for list of classpaths in the "exclusives_groups" data mapping.
None is the default value for most common cases.
:param classpaths: a list of classpath strings. If not specified, ['none'] will be used.
:param target_predicate: filter predicate for the context.targets(). For most common test
cases, None value is good enough.
"""
exclusives_mapping = ExclusivesMapping(context)
exclusives_mapping.set_base_classpath_for_group(
key or '<none>',
[('default', entry) for entry in classpaths or ['none']])
exclusives_mapping._populate_target_maps(
context.targets(target_predicate))
context.products.safe_create_data('exclusives_groups',
lambda: exclusives_mapping)
