def _setup_context(self):
# TODO(John Sirois): Kill when source root registration is lifted out of BUILD files.
with self._run_tracker.new_workunit(name='bootstrap',
labels=[WorkUnitLabel.SETUP]):
source_root_bootstrapper = SourceRootBootstrapper.global_instance()
source_root_bootstrapper.bootstrap(self._address_mapper,
self._build_file_parser)
with self._run_tracker.new_workunit(name='setup',
labels=[WorkUnitLabel.SETUP]):
self._expand_goals(self._requested_goals)
self._expand_specs(self._target_specs, self._fail_fast)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(
self._global_options, self._is_quiet(), self._run_tracker)
context = Context(options=self._options,
run_tracker=self._run_tracker,
target_roots=self._targets,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
spec_excludes=self._spec_excludes,
invalidation_report=invalidation_report)
return context, invalidation_report
def _setup_context(self):
with self._run_tracker.new_workunit(name='setup',
labels=[WorkUnitLabel.SETUP]):
self._build_graph, self._address_mapper, spec_roots = self._init_graph(
self._global_options.enable_v2_engine,
self._global_options.pants_ignore,
self._global_options.build_ignore,
self._global_options.exclude_target_regexp,
self._options.target_specs,
self._global_options.pants_workdir,
self._daemon_graph_helper,
self._global_options.subproject_roots,
)
goals = self._determine_goals(self._requested_goals)
is_quiet = self._should_be_quiet(goals)
target_roots = self._specs_to_targets(spec_roots)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(
self._global_options, is_quiet, self._run_tracker)
context = Context(options=self._options,
run_tracker=self._run_tracker,
target_roots=target_roots,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
invalidation_report=invalidation_report)
return goals, context
def _setup_context(self):
with self._run_tracker.new_workunit(name='setup',
labels=[WorkUnitLabel.SETUP]):
build_file_parser = BuildFileParser(self._build_config,
self._root_dir)
build_graph, address_mapper = self._graph_session.create_build_graph(
self._target_roots, self._root_dir)
goals = self._determine_v1_goals(address_mapper, self._options)
is_quiet = self._should_be_quiet(goals)
target_root_instances = self._roots_to_targets(
build_graph, self._target_roots)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(
self._global_options, is_quiet, self._run_tracker)
context = Context(options=self._options,
run_tracker=self._run_tracker,
target_roots=target_root_instances,
requested_goals=self._options.goals,
build_graph=build_graph,
build_file_parser=build_file_parser,
build_configuration=self._build_config,
address_mapper=address_mapper,
invalidation_report=invalidation_report,
scheduler=self._graph_session.scheduler_session)
return goals, context
def _do_run(self):
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
is_explain = self.global_options.explain
update_reporting(self.global_options,
is_quiet_task() or is_explain, self.run_tracker)
context = Context(config=self.config,
options=self.options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.spec_excludes)
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): %s\n' %
' '.join(goal.name for goal in unknown))
return 1
engine = RoundEngine()
return engine.execute(context, self.goals)
def prepare_task(self,
config=None,
args=None,
targets=None,
build_graph=None,
build_file_parser=None,
address_mapper=None,
console_outstream=None,
workspace=None):
"""Prepares a Task for execution.
task_type: The class of the Task to create.
config: An optional string representing the contents of a pants.ini config.
args: optional list of command line flags, these should be prefixed with '--test-'.
targets: optional list of Target objects passed on the command line.
Returns a new Task ready to execute.
"""
task_type = self.task_type()
assert issubclass(
task_type,
Task), 'task_type must be a Task subclass, got %s' % task_type
config = create_config(config or '')
workdir = os.path.join(config.getdefault('pants_workdir'), 'test',
task_type.__name__)
bootstrap_options = OptionsBootstrapper().get_bootstrap_options()
options = Options(env={},
config=config,
known_scopes=['', 'test'],
args=args or [])
# A lot of basic code uses these options, so always register them.
register_bootstrap_options(options.register_global)
# We need to wrap register_global (can't set .bootstrap attr on the bound instancemethod).
def register_global_wrapper(*args, **kwargs):
return options.register_global(*args, **kwargs)
register_global_wrapper.bootstrap = bootstrap_options.for_global_scope(
)
register_global_options(register_global_wrapper)
task_type.options_scope = 'test'
task_type.register_options_on_scope(options)
run_tracker = create_run_tracker()
context = Context(config,
options,
run_tracker,
targets or [],
build_graph=build_graph,
build_file_parser=build_file_parser,
address_mapper=address_mapper,
console_outstream=console_outstream,
workspace=workspace)
return task_type(context, workdir)
def _setup_context(self):
self._maybe_launch_pantsd()
with self._run_tracker.new_workunit(name='setup',
labels=[WorkUnitLabel.SETUP]):
self._expand_goals(self._requested_goals)
self._expand_specs(self._target_specs, self._fail_fast)
# Now that we've parsed the bootstrap BUILD files, and know about the SCM system.
self._run_tracker.run_info.add_scm_info()
# Update the Reporting settings now that we have options and goal info.
invalidation_report = self._reporting.update_reporting(
self._global_options, self._is_quiet(), self._run_tracker)
context = Context(options=self._options,
run_tracker=self._run_tracker,
target_roots=self._targets,
requested_goals=self._requested_goals,
build_graph=self._build_graph,
build_file_parser=self._build_file_parser,
address_mapper=self._address_mapper,
invalidation_report=invalidation_report)
return context, invalidation_report
def create_context(config='', options=None, target_roots=None, **kwargs):
"""Creates a ``Context`` with no config values, options, or targets by default.
:param config: Either a ``Context`` object or else a string representing the contents of the
pants.ini to parse the config from.
:param options: An optional dict of of option values.
:param target_roots: An optional list of target roots to seed the context target graph from.
:param ``**kwargs``: Any additional keyword arguments to pass through to the Context constructor.
"""
config = config if isinstance(config, Config) else create_config(config)
run_tracker = create_run_tracker()
target_roots = maybe_list(target_roots, Target) if target_roots else []
return Context(config, create_options(options or {}), run_tracker, target_roots, **kwargs)
def prepare_task(task_type,
config=None,
args=None,
targets=None,
build_graph=None,
build_file_parser=None,
address_mapper=None,
console_outstream=None,
workspace=None):
"""Prepares a Task for execution.
task_type: The class of the Task to create.
config: An optional string representing the contents of a pants.ini config.
args: optional list of command line flags, these should be prefixed with '--test-'.
targets: optional list of Target objects passed on the command line.
Returns a new Task ready to execute.
"""
assert issubclass(
task_type,
Task), 'task_type must be a Task subclass, got %s' % task_type
config = create_config(config or '')
workdir = os.path.join(config.getdefault('pants_workdir'), 'test',
task_type.__name__)
parser = OptionParser()
option_group = OptionGroup(parser, 'test')
mkflag = Mkflag('test')
task_type.setup_parser(option_group, args, mkflag)
options, _ = parser.parse_args(args or [])
run_tracker = create_run_tracker()
context = Context(config,
options,
run_tracker,
targets or [],
build_graph=build_graph,
build_file_parser=build_file_parser,
address_mapper=address_mapper,
console_outstream=console_outstream,
workspace=workspace)
return task_type(context, workdir)
def create_context(config='', options=None, target_roots=None, **kwargs):
"""Creates a ``Context`` with no config values, options, or targets by default.
:param config: Either a ``Context`` object or else a string representing the contents of the
pants.ini to parse the config from.
:param options: An optional dict of scope -> (dict of name -> new-style option values).
:param target_roots: An optional list of target roots to seed the context target graph from.
:param ``**kwargs``: Any additional keyword arguments to pass through to the Context constructor.
"""
config = config if isinstance(config, Config) else create_config(config)
# TODO: Get rid of this temporary hack after we plumb options through everywhere and can get
# rid of the config cache.
Config.cache(config)
run_tracker = create_run_tracker()
target_roots = maybe_list(target_roots, Target) if target_roots else []
return Context(config, create_options(options or {}), run_tracker,
target_roots, **kwargs)
def _do_run(self):
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
is_explain = self.global_options.explain
if self.reporting.global_instance().get_options().invalidation_report:
invalidation_report = InvalidationReport()
else:
invalidation_report = None
self.reporting.update_reporting(self.global_options,
is_quiet_task() or is_explain,
self.run_tracker,
invalidation_report=invalidation_report)
context = Context(
options=self.options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.spec_excludes,
invalidation_report=invalidation_report
)
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): {}\n'.format(' '.join(goal.name for goal in unknown)))
return 1
engine = RoundEngine()
result = engine.execute(context, self.goals)
if invalidation_report:
invalidation_report.report()
return result
def run(self):
# TODO(John Sirois): Consider moving to straight python logging. The divide between the
# context/work-unit logging and standard python logging doesn't buy us anything.
# Enable standard python logging for code with no handle to a context/work-unit.
if self.global_options.level:
LogOptions.set_stderr_log_level((self.global_options.level
or 'info').upper())
logdir = self.global_options.logdir or self.config.get(
'goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
prev_log_level = None
# If quiet, temporarily change stderr log level to kill init's output.
if self.global_options.quiet:
prev_log_level = LogOptions.loglevel_name(
LogOptions.stderr_log_level())
# loglevel_name can fail, so only change level if we were able to get the current one.
if prev_log_level is not None:
LogOptions.set_stderr_log_level(
LogOptions._LOG_LEVEL_NONE_KEY)
log.init('goals')
if prev_log_level is not None:
LogOptions.set_stderr_log_level(prev_log_level)
else:
log.init()
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
# Target specs are mapped to the patterns which match them, if any. This variable is a key for
# specs which don't match any exclusion regexes. We know it won't already be in the list of
# patterns, because the asterisks in its name make it an invalid regex.
_UNMATCHED_KEY = '** unmatched **'
def targets_by_pattern(targets, patterns):
mapping = defaultdict(list)
for target in targets:
matched_pattern = None
for pattern in patterns:
if re.search(pattern, target.address.spec) is not None:
matched_pattern = pattern
break
if matched_pattern is None:
mapping[_UNMATCHED_KEY].append(target)
else:
mapping[matched_pattern].append(target)
return mapping
is_explain = self.global_options.explain
update_reporting(self.global_options,
is_quiet_task() or is_explain, self.run_tracker)
if self.global_options.exclude_target_regexp:
excludes = self.global_options.exclude_target_regexp
log.debug('excludes:\n {excludes}'.format(
excludes='\n '.join(excludes)))
by_pattern = targets_by_pattern(self.targets, excludes)
self.targets = by_pattern[_UNMATCHED_KEY]
# The rest of this if-statement is just for debug logging.
log.debug('Targets after excludes: {targets}'.format(
targets=', '.join(t.address.spec for t in self.targets)))
excluded_count = sum(len(by_pattern[p]) for p in excludes)
log.debug('Excluded {count} target{plural}.'.format(
count=excluded_count,
plural=('s' if excluded_count != 1 else '')))
for pattern in excludes:
log.debug('Targets excluded by pattern {pattern}\n {targets}'.
format(pattern=pattern,
targets='\n '.join(
t.address.spec
for t in by_pattern[pattern])))
context = Context(config=self.config,
new_options=self.new_options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.get_spec_excludes())
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): %s\n' %
' '.join(goal.name for goal in unknown))
return 1
engine = RoundEngine()
return engine.execute(context, self.goals)
def _do_run(self):
# TODO(John Sirois): Consider moving to straight python logging. The divide between the
# context/work-unit logging and standard python logging doesn't buy us anything.
# TODO(Eric Ayers) We are missing log messages. Set the log level earlier
# Enable standard python logging for code with no handle to a context/work-unit.
if self.global_options.level:
LogOptions.set_stderr_log_level((self.global_options.level
or 'info').upper())
logdir = self.global_options.logdir or self.config.get(
'goals', 'logdir', default=None)
if logdir:
safe_mkdir(logdir)
LogOptions.set_log_dir(logdir)
prev_log_level = None
# If quiet, temporarily change stderr log level to kill init's output.
if self.global_options.quiet:
prev_log_level = LogOptions.loglevel_name(
LogOptions.stderr_log_level())
# loglevel_name can fail, so only change level if we were able to get the current one.
if prev_log_level is not None:
LogOptions.set_stderr_log_level(
LogOptions._LOG_LEVEL_NONE_KEY)
log.init('goals')
if prev_log_level is not None:
LogOptions.set_stderr_log_level(prev_log_level)
else:
log.init()
# Update the reporting settings, now that we have flags etc.
def is_quiet_task():
for goal in self.goals:
if goal.has_task_of_type(QuietTaskMixin):
return True
return False
is_explain = self.global_options.explain
update_reporting(self.global_options,
is_quiet_task() or is_explain, self.run_tracker)
context = Context(config=self.config,
options=self.options,
run_tracker=self.run_tracker,
target_roots=self.targets,
requested_goals=self.requested_goals,
build_graph=self.build_graph,
build_file_parser=self.build_file_parser,
address_mapper=self.address_mapper,
spec_excludes=self.get_spec_excludes())
unknown = []
for goal in self.goals:
if not goal.ordered_task_names():
unknown.append(goal)
if unknown:
context.log.error('Unknown goal(s): %s\n' %
' '.join(goal.name for goal in unknown))
return 1
engine = RoundEngine()
return engine.execute(context, self.goals)
