def test_transitive_closure_spec(self):
with self.workspace('./BUILD', 'a/BUILD', 'a/b/BUILD') as root_dir:
with open(os.path.join(root_dir, './BUILD'), 'w') as build:
build.write(
dedent('''
fake(name="foo",
dependencies=[
'a',
])
'''))
with open(os.path.join(root_dir, 'a/BUILD'), 'w') as build:
build.write(
dedent('''
fake(name="a",
dependencies=[
'a/b:bat',
])
'''))
with open(os.path.join(root_dir, 'a/b/BUILD'), 'w') as build:
build.write(dedent('''
fake(name="bat")
'''))
build_configuration = BuildConfiguration()
build_configuration.register_target_alias('fake', Target)
parser = BuildFileParser(build_configuration, root_dir=root_dir)
build_graph = BuildGraph(self.address_mapper)
parser.inject_spec_closure_into_build_graph(':foo', build_graph)
self.assertEqual(
len(build_graph.dependencies_of(
SyntheticAddress.parse(':foo'))), 1)
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'cache_key_gen_version': '0-test',
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
# We need a pants.ini, even if empty. get_buildroot() uses its presence.
self.create_file('pants.ini')
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration,
self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser,
FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def test_register_bad_target_alias(self):
with self.assertRaises(TypeError):
self.build_configuration.register_target_alias('fred', object())
target = Target('fred', SyntheticAddress.parse('a:b'), BuildGraph(address_mapper=None))
with self.assertRaises(TypeError):
self.build_configuration.register_target_alias('fred', target)
def setUp(self):
self.real_build_root = BuildRoot().path
self.build_root = mkdtemp(suffix='_BUILD_ROOT')
BuildRoot().path = self.build_root
self.create_file('pants.ini')
self.build_file_parser = BuildFileParser(self.build_root)
self.build_file_parser.register_alias_groups(self.alias_groups)
self.build_graph = BuildGraph()
def test_create_bad_targets(self):
with self.assertRaises(TypeError):
BuildFileAliases(targets={'fred': object()})
target = Target('fred', Address.parse('a:b'),
BuildGraph(address_mapper=None))
with self.assertRaises(TypeError):
BuildFileAliases(targets={'fred': target})
def setUp(self):
self.build_root = mkdtemp(suffix='_BUILD_ROOT')
BuildRoot().path = self.build_root
self.create_file('pants.ini')
self.build_file_parser = make_default_build_file_parser(
self.build_root)
self.build_graph = BuildGraph()
self.config = Config.load()
def setUp(self):
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration,
self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def scan(self, root=None):
"""Scans and parses all BUILD files found under ``root``.
Only BUILD files found under ``root`` are parsed as roots in the graph, but any dependencies of
targets parsed in the root tree's BUILD files will be followed and this may lead to BUILD files
outside of ``root`` being parsed and included in the returned build graph.
:param string root: The path to scan; by default, the build root.
:returns: A new build graph encapsulating the targets found.
"""
build_graph = BuildGraph(self.address_mapper)
for address in self.address_mapper.scan_addresses(root, spec_excludes=self.spec_excludes):
build_graph.inject_address_closure(address)
return build_graph
def test_file_have_coding_utf8(self):
"""
Look through all .py files and ensure they start with the line '# coding=utf8'
"""
config = Config.load()
backend_packages = config.getlist('backends', 'packages')
build_configuration = load_plugins_and_backends(
backends=backend_packages)
build_file_parser = BuildFileParser(
root_dir=get_buildroot(), build_configuration=build_configuration)
address_mapper = BuildFileAddressMapper(build_file_parser)
build_graph = BuildGraph(address_mapper=address_mapper)
for address in address_mapper.scan_addresses(
get_buildroot(),
spec_excludes=[config.getdefault('pants_workdir')]):
build_graph.inject_address_closure(address)
def has_hand_coded_python_files(tgt):
return (not tgt.is_synthetic
) and tgt.is_original and tgt.has_sources('.py')
nonconforming_files = []
for target in build_graph.targets(has_hand_coded_python_files):
for src in target.sources_relative_to_buildroot():
with open(os.path.join(get_buildroot(), src),
'r') as python_file:
coding_line = python_file.readline()
if '' == coding_line and os.path.basename(
src) == '__init__.py':
continue
if coding_line[0:2] == '#!':
# Executable file: look for the coding on the second line.
coding_line = python_file.readline()
if not coding_line.rstrip() == '# coding=utf-8':
nonconforming_files.append(src)
if len(nonconforming_files) > 0:
self.fail(
'Expected these files to contain first line "# coding=utf8": '
+ str(nonconforming_files))
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.new_options = defaultdict(dict) # scope -> key-value mapping.
self.new_options[''] = {
'pants_workdir': os.path.join(self.build_root, '.pants.d'),
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist')
}
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration,
self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp
)
def test_transitive_closure_spec(self):
class FakeTarget(Target):
def __init__(self, *args, **kwargs):
super(FakeTarget, self).__init__(*args, payload=None, **kwargs)
with self.workspace('./BUILD', 'a/BUILD', 'a/b/BUILD') as root_dir:
with open(os.path.join(root_dir, './BUILD'), 'w') as build:
build.write(
dedent('''
fake(name="foo",
dependencies=[
'a',
])
'''))
with open(os.path.join(root_dir, 'a/BUILD'), 'w') as build:
build.write(
dedent('''
fake(name="a",
dependencies=[
'a/b:bat',
])
'''))
with open(os.path.join(root_dir, 'a/b/BUILD'), 'w') as build:
build.write(dedent('''
fake(name="bat")
'''))
parser = BuildFileParser(root_dir=root_dir,
exposed_objects={},
path_relative_utils={},
target_alias_map={'fake': FakeTarget})
build_graph = BuildGraph()
parser.inject_spec_closure_into_build_graph(':foo', build_graph)
self.assertEqual(
len(build_graph.dependencies_of(SyntheticAddress(':foo'))), 1)
def _run():
"""
To add additional paths to sys.path, add a block to the config similar to the following:
[main]
roots: ['src/python/pants_internal/test/',]
"""
version = get_version()
if len(sys.argv) == 2 and sys.argv[1] == _VERSION_OPTION:
_do_exit(version)
root_dir = get_buildroot()
if not os.path.exists(root_dir):
_exit_and_fail('PANTS_BUILD_ROOT does not point to a valid path: %s' % root_dir)
if len(sys.argv) < 2:
argv = ['goal']
else:
argv = sys.argv[1:]
# Hack to force ./pants -h etc. to redirect to goal.
if argv[0] != 'goal' and set(['-h', '--help', 'help']).intersection(argv):
argv = ['goal'] + argv
parser = optparse.OptionParser(add_help_option=False, version=version)
RcFile.install_disable_rc_option(parser)
parser.add_option(_LOG_EXIT_OPTION,
action='store_true',
default=False,
dest='log_exit',
help = 'Log an exit message on success or failure.')
config = Config.load()
# XXX(wickman) This should be in the command goal, not un pants_exe.py!
run_tracker = RunTracker.from_config(config)
report = initial_reporting(config, run_tracker)
run_tracker.start(report)
url = run_tracker.run_info.get_info('report_url')
if url:
run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
run_tracker.log(Report.INFO, '(To run a reporting server: ./pants goal server)')
build_file_parser = BuildFileParser(root_dir=root_dir, run_tracker=run_tracker)
build_graph = BuildGraph(run_tracker=run_tracker)
additional_backends = config.getlist('backends', 'packages')
load_backends_from_source(build_file_parser, additional_backends=additional_backends)
command_class, command_args = _parse_command(root_dir, argv)
command = command_class(run_tracker,
root_dir,
parser,
command_args,
build_file_parser,
build_graph)
try:
if command.serialized():
def onwait(pid):
process = psutil.Process(pid)
print('Waiting on pants process %d (%s) to complete' %
(pid, ' '.join(process.cmdline)), file=sys.stderr)
return True
runfile = os.path.join(root_dir, '.pants.run')
lock = Lock.acquire(runfile, onwait=onwait)
else:
lock = Lock.unlocked()
try:
result = command.run(lock)
if result:
run_tracker.set_root_outcome(WorkUnit.FAILURE)
_do_exit(result)
except KeyboardInterrupt:
command.cleanup()
raise
finally:
lock.release()
finally:
run_tracker.end()
# Must kill nailguns only after run_tracker.end() is called, because there may still
# be pending background work that needs a nailgun.
if (hasattr(command.options, 'cleanup_nailguns') and command.options.cleanup_nailguns) \
or config.get('nailgun', 'autokill', default=False):
NailgunTask.killall(None)
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(
plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
# TODO: Create a 'Subsystem' abstraction instead of special-casing run-tracker here
# and in register_options().
known_scopes = ['', 'run-tracker']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(
known_scopes=known_scopes)
self.register_options()
# TODO(Eric Ayers) We are missing log messages. Set the log level earlier
# Enable standard python logging for code with no handle to a context/work-unit.
self._setup_logging() # NB: self.options are needed for this call.
self.run_tracker = RunTracker.from_options(self.options)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(
Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(
build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir,
relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def __init__(self, run_tracker, root_dir, parser, args):
"""run_tracker: The (already opened) RunTracker to track this run with
root_dir: The root directory of the pants workspace
parser: an OptionParser
args: the subcommand arguments to parse"""
self.run_tracker = run_tracker
self.root_dir = root_dir
# TODO(pl): Gross that we're doing a local import here, but this has dependendencies
# way down into specific Target subclasses, and I'd prefer to make it explicit that this
# import is in many ways similar to to third party plugin imports below.
from pants.base.build_file_aliases import (
target_aliases, object_aliases,
applicative_path_relative_util_aliases,
partial_path_relative_util_aliases)
for alias, target_type in target_aliases.items():
BuildFileParser.register_target_alias(alias, target_type)
for alias, obj in object_aliases.items():
BuildFileParser.register_exposed_object(alias, obj)
for alias, util in applicative_path_relative_util_aliases.items():
BuildFileParser.register_applicative_path_relative_util(
alias, util)
for alias, util in partial_path_relative_util_aliases.items():
BuildFileParser.register_partial_path_relative_util(alias, util)
config = Config.load()
# TODO(pl): This is awful but I need something quick and dirty to support
# injection of third party Targets and tools into BUILD file context
plugins = config.getlist('plugins', 'entry_points', default=[])
for entry_point_spec in plugins:
module, entry_point = entry_point_spec.split(':')
plugin_module = __import__(module, globals(), locals(),
[entry_point], 0)
getattr(plugin_module, entry_point)(config)
self.build_file_parser = BuildFileParser(root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.build_graph = BuildGraph(run_tracker=self.run_tracker)
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
# try:
build_file = BuildFile(root_dir=self.root_dir, relpath=path)
self.build_file_parser.parse_build_file_family(build_file)
# except (TypeError, ImportError):
# error(path, include_traceback=True)
# except (IOError, SyntaxError):
# error(path)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
# Override the OptionParser's error with more useful output
def error(message=None, show_help=True):
if message:
print(message + '\n')
if show_help:
parser.print_help()
parser.exit(status=1)
parser.error = error
self.error = error
self.setup_parser(parser, args)
self.options, self.args = parser.parse_args(args)
self.parser = parser
def setup(self):
options_bootstrapper = OptionsBootstrapper()
bootstrap_options = options_bootstrapper.get_bootstrap_options()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = bootstrap_options.for_global_scope().plugins
backend_packages = bootstrap_options.for_global_scope().backend_packages
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [ScopeInfo.for_global_scope()]
# Add scopes for all needed subsystems.
subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
