def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
Subsystem.reset()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.addCleanup(safe_rmtree, self.build_root)
self.pants_workdir = os.path.join(self.build_root, '.pants.d')
safe_mkdir(self.pants_workdir)
self.options = defaultdict(dict) # scope -> key-value mapping.
self.options[''] = {
'pants_workdir': self.pants_workdir,
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist'),
'pants_configdir': os.path.join(self.build_root, 'config'),
'cache_key_gen_version': '0-test',
}
BuildRoot().path = self.build_root
self.addCleanup(BuildRoot().reset)
# We need a pants.ini, even if empty. get_buildroot() uses its presence.
self.create_file('pants.ini')
self._build_configuration = BuildConfiguration()
self._build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(self._build_configuration,
self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser,
FilesystemBuildFile)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def setUp(self):
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration,
self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def test_file_have_coding_utf8(self):
"""
Look through all .py files and ensure they start with the line '# coding=utf8'
"""
config = Config.load()
backend_packages = config.getlist('backends', 'packages')
build_configuration = load_plugins_and_backends(
backends=backend_packages)
build_file_parser = BuildFileParser(
root_dir=get_buildroot(), build_configuration=build_configuration)
address_mapper = BuildFileAddressMapper(build_file_parser)
build_graph = BuildGraph(address_mapper=address_mapper)
for address in address_mapper.scan_addresses(
get_buildroot(),
spec_excludes=[config.getdefault('pants_workdir')]):
build_graph.inject_address_closure(address)
def has_hand_coded_python_files(tgt):
return (not tgt.is_synthetic
) and tgt.is_original and tgt.has_sources('.py')
nonconforming_files = []
for target in build_graph.targets(has_hand_coded_python_files):
for src in target.sources_relative_to_buildroot():
with open(os.path.join(get_buildroot(), src),
'r') as python_file:
coding_line = python_file.readline()
if '' == coding_line and os.path.basename(
src) == '__init__.py':
continue
if coding_line[0:2] == '#!':
# Executable file: look for the coding on the second line.
coding_line = python_file.readline()
if not coding_line.rstrip() == '# coding=utf-8':
nonconforming_files.append(src)
if len(nonconforming_files) > 0:
self.fail(
'Expected these files to contain first line "# coding=utf8": '
+ str(nonconforming_files))
def setUp(self):
super(BaseTest, self).setUp()
Goal.clear()
self.real_build_root = BuildRoot().path
self.build_root = os.path.realpath(mkdtemp(suffix='_BUILD_ROOT'))
self.new_options = defaultdict(dict) # scope -> key-value mapping.
self.new_options[''] = {
'pants_workdir': os.path.join(self.build_root, '.pants.d'),
'pants_supportdir': os.path.join(self.build_root, 'build-support'),
'pants_distdir': os.path.join(self.build_root, 'dist')
}
BuildRoot().path = self.build_root
self.create_file('pants.ini')
build_configuration = BuildConfiguration()
build_configuration.register_aliases(self.alias_groups)
self.build_file_parser = BuildFileParser(build_configuration,
self.build_root)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def __init__(self, root_dir, options, build_config, run_tracker, reporting, exiter=sys.exit):
"""
:param str root_dir: The root directory of the pants workspace (aka the "build root").
:param Options options: The global, pre-initialized Options instance.
:param BuildConfiguration build_config: A pre-initialized BuildConfiguration instance.
:param Runtracker run_tracker: The global, pre-initialized/running RunTracker instance.
:param Reporting reporting: The global, pre-initialized Reporting instance.
:param func exiter: A function that accepts an exit code value and exits (for tests, Optional).
"""
self._root_dir = root_dir
self._options = options
self._build_config = build_config
self._run_tracker = run_tracker
self._reporting = reporting
self._exiter = exiter
self._goals = []
self._targets = []
self._requested_goals = self._options.goals
self._target_specs = self._options.target_specs
self._help_request = self._options.help_request
self._global_options = options.for_global_scope()
self._tag = self._global_options.tag
self._fail_fast = self._global_options.fail_fast
self._spec_excludes = self._global_options.spec_excludes
self._explain = self._global_options.explain
self._kill_nailguns = self._global_options.kill_nailguns
self._build_file_type = self._get_buildfile_type(self._global_options.build_file_rev)
self._build_file_parser = BuildFileParser(self._build_config, self._root_dir)
self._address_mapper = BuildFileAddressMapper(self._build_file_parser, self._build_file_type)
self._build_graph = BuildGraph(self._address_mapper)
self._spec_parser = CmdLineSpecParser(
self._root_dir,
self._address_mapper,
spec_excludes=self._spec_excludes,
exclude_target_regexps=self._global_options.exclude_target_regexp
)
def setup(self):
options_bootstrapper = OptionsBootstrapper()
# Force config into the cache so we (and plugin/backend loading code) can use it.
# TODO: Plumb options in explicitly.
bootstrap_options = options_bootstrapper.get_bootstrap_options()
self.config = Config.from_cache()
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
backend_packages = self.config.getlist('backends', 'packages', [])
plugins = self.config.getlist('backends', 'plugins', [])
build_configuration = load_plugins_and_backends(
plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
# TODO: Create a 'Subsystem' abstraction instead of special-casing run-tracker here
# and in register_options().
known_scopes = ['', 'run-tracker']
for goal in Goal.all():
# Note that enclosing scopes will appear before scopes they enclose.
known_scopes.extend(filter(None, goal.known_scopes()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(
known_scopes=known_scopes)
self.register_options()
# TODO(Eric Ayers) We are missing log messages. Set the log level earlier
# Enable standard python logging for code with no handle to a context/work-unit.
self._setup_logging() # NB: self.options are needed for this call.
self.run_tracker = RunTracker.from_options(self.options)
report = initial_reporting(self.config, self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
self.run_tracker.log(
Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(
build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
with self.run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnit.SETUP]):
# construct base parameters to be filled in for BuildGraph
for path in self.config.getlist('goals',
'bootstrap_buildfiles',
default=[]):
build_file = BuildFile.from_cache(root_dir=self.root_dir,
relpath=path)
# TODO(pl): This is an unfortunate interface leak, but I don't think
# in the long run that we should be relying on "bootstrap" BUILD files
# that do nothing except modify global state. That type of behavior
# (e.g. source roots, goal registration) should instead happen in
# project plugins, or specialized configuration files.
self.build_file_parser.parse_build_file_family(build_file)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
self._expand_goals_and_specs()
def reset_build_graph(self):
"""Start over with a fresh build graph with no targets in it."""
self.address_mapper = BuildFileAddressMapper(self.build_file_parser)
self.build_graph = BuildGraph(address_mapper=self.address_mapper)
def _run():
# place the registration of the unhandled exception hook
# as early as possible in the code
sys.excepthook = _unhandled_exception_hook
"""
To add additional paths to sys.path, add a block to the config similar to the following:
[main]
roots: ['src/python/pants_internal/test/',]
"""
logging.basicConfig()
version = pants_version()
if len(sys.argv) == 2 and sys.argv[1] == _VERSION_OPTION:
_do_exit(msg=version, out=sys.stdout)
root_dir = get_buildroot()
if not os.path.exists(root_dir):
_exit_and_fail('PANTS_BUILD_ROOT does not point to a valid path: %s' %
root_dir)
if len(sys.argv) < 2:
argv = ['goal']
else:
argv = sys.argv[1:]
# Hack to force ./pants -h etc. to redirect to goal.
if argv[0] != 'goal' and set(['-h', '--help', 'help']).intersection(argv):
argv = ['goal'] + argv
parser = optparse.OptionParser(add_help_option=False, version=version)
RcFile.install_disable_rc_option(parser)
parser.add_option(_LOG_EXIT_OPTION,
action='store_true',
default=False,
dest='log_exit',
help='Log an exit message on success or failure.')
config = Config.load()
# XXX(wickman) This should be in the command goal, not in pants_exe.py!
run_tracker = RunTracker.from_config(config)
report = initial_reporting(config, run_tracker)
run_tracker.start(report)
url = run_tracker.run_info.get_info('report_url')
if url:
run_tracker.log(Report.INFO, 'See a report at: %s' % url)
else:
run_tracker.log(Report.INFO,
'(To run a reporting server: ./pants goal server)')
backend_packages = config.getlist('backends', 'packages')
build_configuration = load_build_configuration_from_source(
additional_backends=backend_packages)
build_file_parser = BuildFileParser(
build_configuration=build_configuration,
root_dir=root_dir,
run_tracker=run_tracker)
address_mapper = BuildFileAddressMapper(build_file_parser)
build_graph = BuildGraph(run_tracker=run_tracker,
address_mapper=address_mapper)
command_class, command_args = _parse_command(root_dir, argv)
command = command_class(run_tracker, root_dir, parser, command_args,
build_file_parser, address_mapper, build_graph)
try:
if command.serialized():
def onwait(pid):
process = psutil.Process(pid)
print('Waiting on pants process %d (%s) to complete' %
(pid, ' '.join(process.cmdline)),
file=sys.stderr)
return True
runfile = os.path.join(root_dir, '.pants.run')
lock = Lock.acquire(runfile, onwait=onwait)
else:
lock = Lock.unlocked()
try:
result = command.run(lock)
if result:
run_tracker.set_root_outcome(WorkUnit.FAILURE)
_do_exit(result)
except KeyboardInterrupt:
command.cleanup()
raise
finally:
lock.release()
finally:
run_tracker.end()
# Must kill nailguns only after run_tracker.end() is called, because there may still
# be pending background work that needs a nailgun.
if (hasattr(command.options, 'cleanup_nailguns') and command.options.cleanup_nailguns) \
or config.get('nailgun', 'autokill', default=False):
NailgunTask.killall(None)
def setup(self):
options_bootstrapper = OptionsBootstrapper()
bootstrap_options = options_bootstrapper.get_bootstrap_options()
# Get logging setup prior to loading backends so that they can log as needed.
self._setup_logging(bootstrap_options.for_global_scope())
# Add any extra paths to python path (eg for loading extra source backends)
for path in bootstrap_options.for_global_scope().pythonpath:
sys.path.append(path)
pkg_resources.fixup_namespace_packages(path)
# Load plugins and backends.
plugins = bootstrap_options.for_global_scope().plugins
backend_packages = bootstrap_options.for_global_scope().backend_packages
build_configuration = load_plugins_and_backends(plugins, backend_packages)
# Now that plugins and backends are loaded, we can gather the known scopes.
self.targets = []
known_scope_infos = [ScopeInfo.for_global_scope()]
# Add scopes for all needed subsystems.
subsystems = (set(self.subsystems) | Goal.subsystems() | build_configuration.subsystems())
for subsystem in subsystems:
known_scope_infos.append(ScopeInfo(subsystem.options_scope, ScopeInfo.GLOBAL_SUBSYSTEM))
# Add scopes for all tasks in all goals.
for goal in Goal.all():
known_scope_infos.extend(filter(None, goal.known_scope_infos()))
# Now that we have the known scopes we can get the full options.
self.options = options_bootstrapper.get_full_options(known_scope_infos)
self.register_options(subsystems)
# Make the options values available to all subsystems.
Subsystem._options = self.options
# Now that we have options we can instantiate subsystems.
self.run_tracker = RunTracker.global_instance()
self.reporting = Reporting.global_instance()
report = self.reporting.initial_reporting(self.run_tracker)
self.run_tracker.start(report)
url = self.run_tracker.run_info.get_info('report_url')
if url:
self.run_tracker.log(Report.INFO, 'See a report at: {}'.format(url))
else:
self.run_tracker.log(Report.INFO, '(To run a reporting server: ./pants server)')
self.build_file_parser = BuildFileParser(build_configuration=build_configuration,
root_dir=self.root_dir,
run_tracker=self.run_tracker)
rev = self.options.for_global_scope().build_file_rev
if rev:
ScmBuildFile.set_rev(rev)
ScmBuildFile.set_scm(get_scm())
build_file_type = ScmBuildFile
else:
build_file_type = FilesystemBuildFile
self.address_mapper = BuildFileAddressMapper(self.build_file_parser, build_file_type)
self.build_graph = BuildGraph(run_tracker=self.run_tracker,
address_mapper=self.address_mapper)
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self.run_tracker.new_workunit(name='bootstrap', labels=[WorkUnit.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self.address_mapper, self.build_file_parser)
self._expand_goals_and_specs()
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self.run_tracker.run_info.add_scm_info()
