def initial_reporting(config, run_tracker):
"""Sets up the initial reporting configuration.
Will be changed after we parse cmd-line flags.
"""
reports_dir = os.path.join(config.getdefault('pants_workdir'), 'reports')
link_to_latest = os.path.join(reports_dir, 'latest')
run_id = run_tracker.run_info.get_info('id')
if run_id is None:
raise ReportingError('No run_id set')
run_dir = os.path.join(reports_dir, run_id)
safe_rmtree(run_dir)
html_dir = os.path.join(run_dir, 'html')
safe_mkdir(html_dir)
try:
if os.path.lexists(link_to_latest):
os.unlink(link_to_latest)
os.symlink(run_dir, link_to_latest)
except OSError as e:
# Another run may beat us to deletion or creation.
if not (e.errno == errno.EEXIST or e.errno == errno.ENOENT):
raise
report = Report()
# Capture initial console reporting into a buffer. We'll do something with it once
# we know what the cmd-line flag settings are.
outfile = StringIO()
capturing_reporter_settings = PlainTextReporter.Settings(
outfile=outfile,
log_level=Report.INFO,
color=False,
indent=True,
timing=False,
cache_stats=False)
capturing_reporter = PlainTextReporter(run_tracker,
capturing_reporter_settings)
report.add_reporter('capturing', capturing_reporter)
# Set up HTML reporting. We always want that.
template_dir = config.get('reporting', 'reports_template_dir')
html_reporter_settings = HtmlReporter.Settings(log_level=Report.INFO,
html_dir=html_dir,
template_dir=template_dir)
html_reporter = HtmlReporter(run_tracker, html_reporter_settings)
report.add_reporter('html', html_reporter)
# Add some useful RunInfo.
run_tracker.run_info.add_info('default_report',
html_reporter.report_path())
(_, port) = ReportingServerManager.get_current_server_pid_and_port()
if port:
run_tracker.run_info.add_info(
'report_url', 'http://localhost:%d/run/%s' % (port, run_id))
return report
def initial_reporting(self, run_tracker):
"""Sets up the initial reporting configuration.
Will be changed after we parse cmd-line flags.
"""
link_to_latest = os.path.join(self.get_options().reports_dir, 'latest')
run_id = run_tracker.run_info.get_info('id')
if run_id is None:
raise ReportingError('No run_id set')
run_dir = os.path.join(self.get_options().reports_dir, run_id)
safe_rmtree(run_dir)
html_dir = os.path.join(run_dir, 'html')
safe_mkdir(html_dir)
relative_symlink(run_dir, link_to_latest)
report = Report()
# Capture initial console reporting into a buffer. We'll do something with it once
# we know what the cmd-line flag settings are.
outfile = StringIO()
errfile = StringIO()
capturing_reporter_settings = PlainTextReporter.Settings(
outfile=outfile,
errfile=errfile,
log_level=Report.INFO,
color=False,
indent=True,
timing=False,
cache_stats=False,
label_format=self.get_options().console_label_format,
tool_output_format=self.get_options().console_tool_output_format)
capturing_reporter = PlainTextReporter(run_tracker,
capturing_reporter_settings)
report.add_reporter('capturing', capturing_reporter)
# Set up HTML reporting. We always want that.
html_reporter_settings = HtmlReporter.Settings(
log_level=Report.INFO,
html_dir=html_dir,
template_dir=self.get_options().template_dir)
html_reporter = HtmlReporter(run_tracker, html_reporter_settings)
report.add_reporter('html', html_reporter)
# Add some useful RunInfo.
run_tracker.run_info.add_info('default_report',
html_reporter.report_path())
port = ReportingServerManager().socket
if port:
run_tracker.run_info.add_info(
'report_url',
'http://localhost:{}/run/{}'.format(port, run_id))
return report
def update_reporting(self, global_options, is_quiet, run_tracker):
"""Updates reporting config once we've parsed cmd-line flags."""
# Get any output silently buffered in the old console reporter, and remove it.
removed_reporter = run_tracker.report.remove_reporter('capturing')
buffered_out = self._consume_stringio(removed_reporter.settings.outfile)
buffered_err = self._consume_stringio(removed_reporter.settings.errfile)
log_level = Report.log_level_from_string(global_options.level or 'info')
# Ideally, we'd use terminfo or somesuch to discover whether a
# terminal truly supports color, but most that don't set TERM=dumb.
color = global_options.colors and (os.getenv('TERM') != 'dumb')
timing = global_options.time
cache_stats = global_options.time # TODO: Separate flag for this?
if is_quiet:
console_reporter = QuietReporter(run_tracker,
QuietReporter.Settings(log_level=log_level, color=color,
timing=timing, cache_stats=cache_stats))
else:
# Set up the new console reporter.
stdout = sys.stdout.buffer if PY3 else sys.stdout
stderr = sys.stderr.buffer if PY3 else sys.stderr
settings = PlainTextReporter.Settings(log_level=log_level, outfile=stdout, errfile=stderr,
color=color, indent=True, timing=timing, cache_stats=cache_stats,
label_format=self.get_options().console_label_format,
tool_output_format=self.get_options().console_tool_output_format)
console_reporter = PlainTextReporter(run_tracker, settings)
console_reporter.emit(buffered_out, dest=ReporterDestination.OUT)
console_reporter.emit(buffered_err, dest=ReporterDestination.ERR)
console_reporter.flush()
run_tracker.report.add_reporter('console', console_reporter)
if global_options.logdir:
# Also write plaintext logs to a file. This is completely separate from the html reports.
safe_mkdir(global_options.logdir)
run_id = run_tracker.run_info.get_info('id')
outfile = open(os.path.join(global_options.logdir, '{}.log'.format(run_id)), 'wb')
errfile = open(os.path.join(global_options.logdir, '{}.err.log'.format(run_id)), 'wb')
settings = PlainTextReporter.Settings(log_level=log_level, outfile=outfile, errfile=errfile,
color=False, indent=True, timing=True, cache_stats=True,
label_format=self.get_options().console_label_format,
tool_output_format=self.get_options().console_tool_output_format)
logfile_reporter = PlainTextReporter(run_tracker, settings)
logfile_reporter.emit(buffered_out, dest=ReporterDestination.OUT)
logfile_reporter.emit(buffered_err, dest=ReporterDestination.ERR)
logfile_reporter.flush()
run_tracker.report.add_reporter('logfile', logfile_reporter)
invalidation_report = self._get_invalidation_report()
if invalidation_report:
run_id = run_tracker.run_info.get_info('id')
outfile = os.path.join(self.get_options().reports_dir, run_id, 'invalidation-report.csv')
invalidation_report.set_filename(outfile)
return invalidation_report
def initialize(self, run_tracker, start_time=None):
"""Initialize with the given RunTracker.
TODO: See `RunTracker.start`.
"""
run_id = run_tracker.initialize()
run_dir = os.path.join(self.get_options().reports_dir, run_id)
html_dir = os.path.join(run_dir, 'html')
safe_mkdir(html_dir)
relative_symlink(
run_dir, os.path.join(self.get_options().reports_dir, 'latest'))
report = Report()
# Capture initial console reporting into a buffer. We'll do something with it once
# we know what the cmd-line flag settings are.
outfile = StringIO()
errfile = StringIO()
capturing_reporter_settings = PlainTextReporter.Settings(
outfile=outfile,
errfile=errfile,
log_level=Report.INFO,
color=False,
indent=True,
timing=False,
cache_stats=False,
label_format=self.get_options().console_label_format,
tool_output_format=self.get_options().console_tool_output_format)
capturing_reporter = PlainTextReporter(run_tracker,
capturing_reporter_settings)
report.add_reporter('capturing', capturing_reporter)
# Set up HTML reporting. We always want that.
html_reporter_settings = HtmlReporter.Settings(
log_level=Report.INFO,
html_dir=html_dir,
template_dir=self.get_options().template_dir)
html_reporter = HtmlReporter(run_tracker, html_reporter_settings)
report.add_reporter('html', html_reporter)
# Add some useful RunInfo.
run_tracker.run_info.add_info('default_report',
html_reporter.report_path())
port = ReportingServerManager().socket
if port:
run_tracker.run_info.add_info(
'report_url',
'http://localhost:{}/run/{}'.format(port, run_id))
# And start tracking the run.
run_tracker.start(report, start_time)
def update_reporting(self, global_options, is_quiet, run_tracker):
"""Updates reporting config once we've parsed cmd-line flags."""
# Get any output silently buffered in the old console reporter, and remove it.
removed_reporter = run_tracker.report.remove_reporter('capturing')
buffered_out = self._consume_stringio(removed_reporter.settings.outfile)
buffered_err = self._consume_stringio(removed_reporter.settings.errfile)
log_level = Report.log_level_from_string(global_options.level or 'info')
# Ideally, we'd use terminfo or somesuch to discover whether a
# terminal truly supports color, but most that don't set TERM=dumb.
color = global_options.colors and (os.getenv('TERM') != 'dumb')
timing = global_options.time
cache_stats = global_options.time # TODO: Separate flag for this?
if is_quiet:
console_reporter = QuietReporter(run_tracker,
QuietReporter.Settings(log_level=log_level, color=color,
timing=timing, cache_stats=cache_stats))
else:
# Set up the new console reporter.
stdout = sys.stdout.buffer
stderr = sys.stderr.buffer
settings = PlainTextReporter.Settings(log_level=log_level, outfile=stdout, errfile=stderr,
color=color, indent=True, timing=timing, cache_stats=cache_stats,
label_format=self.get_options().console_label_format,
tool_output_format=self.get_options().console_tool_output_format)
console_reporter = PlainTextReporter(run_tracker, settings)
console_reporter.emit(buffered_out, dest=ReporterDestination.OUT)
console_reporter.emit(buffered_err, dest=ReporterDestination.ERR)
console_reporter.flush()
run_tracker.report.add_reporter('console', console_reporter)
if global_options.logdir:
# Also write plaintext logs to a file. This is completely separate from the html reports.
safe_mkdir(global_options.logdir)
run_id = run_tracker.run_info.get_info('id')
outfile = open(os.path.join(global_options.logdir, '{}.log'.format(run_id)), 'wb')
errfile = open(os.path.join(global_options.logdir, '{}.err.log'.format(run_id)), 'wb')
settings = PlainTextReporter.Settings(log_level=log_level, outfile=outfile, errfile=errfile,
color=False, indent=True, timing=True, cache_stats=True,
label_format=self.get_options().console_label_format,
tool_output_format=self.get_options().console_tool_output_format)
logfile_reporter = PlainTextReporter(run_tracker, settings)
logfile_reporter.emit(buffered_out, dest=ReporterDestination.OUT)
logfile_reporter.emit(buffered_err, dest=ReporterDestination.ERR)
logfile_reporter.flush()
run_tracker.report.add_reporter('logfile', logfile_reporter)
invalidation_report = self._get_invalidation_report()
if invalidation_report:
run_id = run_tracker.run_info.get_info('id')
outfile = os.path.join(self.get_options().reports_dir, run_id, 'invalidation-report.csv')
invalidation_report.set_filename(outfile)
return invalidation_report
def update_reporting(options, is_console_task, run_tracker):
"""Updates reporting config once we've parsed cmd-line flags."""
# Get any output silently buffered in the old console reporter, and remove it.
old_outfile = run_tracker.report.remove_reporter(
'capturing').settings.outfile
old_outfile.flush()
buffered_output = old_outfile.getvalue()
old_outfile.close()
log_level = Report.log_level_from_string(options.log_level or 'info')
# Ideally, we'd use terminfo or somesuch to discover whether a
# terminal truly supports color, but most that don't set TERM=dumb.
color = (not options.no_color) and (os.getenv('TERM') != 'dumb')
timing = options.time
cache_stats = options.time # TODO: Separate flag for this?
if options.quiet or is_console_task:
console_reporter = QuietReporter(
run_tracker,
QuietReporter.Settings(log_level=log_level, color=color))
else:
# Set up the new console reporter.
settings = PlainTextReporter.Settings(log_level=log_level,
outfile=sys.stdout,
color=color,
indent=True,
timing=timing,
cache_stats=cache_stats)
console_reporter = PlainTextReporter(run_tracker, settings)
console_reporter.emit(buffered_output)
console_reporter.flush()
run_tracker.report.add_reporter('console', console_reporter)
if options.logdir:
# Also write plaintext logs to a file. This is completely separate from the html reports.
safe_mkdir(options.logdir)
run_id = run_tracker.run_info.get_info('id')
outfile = open(os.path.join(options.logdir, '%s.log' % run_id), 'w')
settings = PlainTextReporter.Settings(log_level=log_level,
outfile=outfile,
color=False,
indent=True,
timing=True,
cache_stats=True)
logfile_reporter = PlainTextReporter(run_tracker, settings)
logfile_reporter.emit(buffered_output)
logfile_reporter.flush()
run_tracker.report.add_reporter('logfile', logfile_reporter)
def update_reporting(self, global_options, is_quiet_task, run_tracker, invalidation_report=None):
"""Updates reporting config once we've parsed cmd-line flags."""
# Get any output silently buffered in the old console reporter, and remove it.
old_outfile = run_tracker.report.remove_reporter('capturing').settings.outfile
old_outfile.flush()
buffered_output = old_outfile.getvalue()
old_outfile.close()
log_level = Report.log_level_from_string(global_options.level or 'info')
# Ideally, we'd use terminfo or somesuch to discover whether a
# terminal truly supports color, but most that don't set TERM=dumb.
color = global_options.colors and (os.getenv('TERM') != 'dumb')
timing = global_options.time
cache_stats = global_options.time # TODO: Separate flag for this?
if global_options.quiet or is_quiet_task:
console_reporter = QuietReporter(run_tracker,
QuietReporter.Settings(log_level=log_level, color=color))
else:
# Set up the new console reporter.
settings = PlainTextReporter.Settings(log_level=log_level, outfile=sys.stdout, color=color,
indent=True, timing=timing, cache_stats=cache_stats)
console_reporter = PlainTextReporter(run_tracker, settings)
console_reporter.emit(buffered_output)
console_reporter.flush()
run_tracker.report.add_reporter('console', console_reporter)
if global_options.logdir:
# Also write plaintext logs to a file. This is completely separate from the html reports.
safe_mkdir(global_options.logdir)
run_id = run_tracker.run_info.get_info('id')
outfile = open(os.path.join(global_options.logdir, '{}.log'.format(run_id)), 'w')
settings = PlainTextReporter.Settings(log_level=log_level, outfile=outfile, color=False,
indent=True, timing=True, cache_stats=True)
logfile_reporter = PlainTextReporter(run_tracker, settings)
logfile_reporter.emit(buffered_output)
logfile_reporter.flush()
run_tracker.report.add_reporter('logfile', logfile_reporter)
if invalidation_report:
run_id = run_tracker.run_info.get_info('id')
outfile = os.path.join(self.get_options().reports_dir, run_id, 'invalidation-report.csv')
invalidation_report.set_filename(outfile)
def create_run_tracker(info_dir=None):
"""Creates a ``RunTracker`` and starts it.
:param string info_dir: An optional director for the run tracker to store state; defaults to a
new temp dir that will be be cleaned up on interpreter exit.
"""
# TODO(John Sirois): Rework uses around a context manager for cleanup of the info_dir in a more
# disciplined manner
info_dir = info_dir or safe_mkdtemp()
run_tracker = RunTracker(info_dir)
report = Report()
settings = PlainTextReporter.Settings(outfile=sys.stdout,
log_level=Report.INFO,
color=False,
indent=True,
timing=False,
cache_stats=False)
report.add_reporter('test_debug', PlainTextReporter(run_tracker, settings))
run_tracker.start(report)
return run_tracker
def update_reporting(options, is_console_task, run_tracker):
"""Updates reporting config once we've parsed cmd-line flags."""
# Get any output silently buffered in the old console reporter, and remove it.
old_outfile = run_tracker.report.remove_reporter('capturing').settings.outfile
old_outfile.flush()
buffered_output = old_outfile.getvalue()
old_outfile.close()
log_level = Report.log_level_from_string(options.log_level or 'info')
color = not options.no_color
timing = options.time
cache_stats = options.time # TODO: Separate flag for this?
if options.quiet or is_console_task:
console_reporter = QuietReporter(run_tracker,
QuietReporter.Settings(log_level=log_level, color=color))
else:
# Set up the new console reporter.
settings = PlainTextReporter.Settings(log_level=log_level, outfile=sys.stdout, color=color,
indent=True, timing=timing, cache_stats=cache_stats)
console_reporter = PlainTextReporter(run_tracker, settings)
console_reporter.emit(buffered_output)
console_reporter.flush()
run_tracker.report.add_reporter('console', console_reporter)
if options.logdir:
# Also write plaintext logs to a file. This is completely separate from the html reports.
safe_mkdir(options.logdir)
run_id = run_tracker.run_info.get_info('id')
outfile = open(os.path.join(options.logdir, '%s.log' % run_id), 'w')
settings = PlainTextReporter.Settings(log_level=log_level, outfile=outfile, color=False,
indent=True, timing=True, cache_stats=True)
logfile_reporter = PlainTextReporter(run_tracker, settings)
logfile_reporter.emit(buffered_output)
logfile_reporter.flush()
run_tracker.report.add_reporter('logfile', logfile_reporter)
def initialize(self, run_tracker, all_options, start_time=None):
"""Initialize with the given RunTracker.
TODO: See `RunTracker.start`.
"""
run_id, run_uuid = run_tracker.initialize(all_options)
run_dir = os.path.join(self.get_options().reports_dir, run_id)
html_dir = os.path.join(run_dir, "html")
safe_mkdir(html_dir)
relative_symlink(
run_dir, os.path.join(self.get_options().reports_dir, "latest"))
report = Report()
# Capture initial console reporting into a buffer. We'll do something with it once
# we know what the cmd-line flag settings are.
outfile = BytesIO()
errfile = BytesIO()
capturing_reporter_settings = PlainTextReporter.Settings(
outfile=outfile,
errfile=errfile,
log_level=Report.INFO,
color=False,
indent=True,
timing=False,
cache_stats=False,
label_format=self.get_options().console_label_format,
tool_output_format=self.get_options().console_tool_output_format,
)
capturing_reporter = PlainTextReporter(run_tracker,
capturing_reporter_settings)
report.add_reporter("capturing", capturing_reporter)
# Set up HTML reporting. We always want that.
html_reporter_settings = HtmlReporter.Settings(
log_level=Report.INFO,
html_dir=html_dir,
template_dir=self.get_options().template_dir)
html_reporter = HtmlReporter(run_tracker, html_reporter_settings)
report.add_reporter("html", html_reporter)
# Set up Zipkin reporting.
zipkin_endpoint = self.get_options().zipkin_endpoint
trace_id = self.get_options().zipkin_trace_id
parent_id = self.get_options().zipkin_parent_id
sample_rate = self.get_options().zipkin_sample_rate
service_name_prefix = self.get_options().zipkin_service_name_prefix
if "{}" not in service_name_prefix:
service_name_prefix = service_name_prefix + "/{}"
max_span_batch_size = int(
self.get_options().zipkin_max_span_batch_size)
if zipkin_endpoint is None and trace_id is not None and parent_id is not None:
raise ValueError(
"The zipkin-endpoint flag must be set if zipkin-trace-id and zipkin-parent-id flags are given."
)
if (trace_id is None) != (parent_id is None):
raise ValueError(
"Flags zipkin-trace-id and zipkin-parent-id must both either be set or not set."
)
# If trace_id isn't set by a flag, use UUID from run_id
if trace_id is None:
trace_id = run_uuid
if trace_id and (len(trace_id) != 16 and len(trace_id) != 32
or not is_hex_string(trace_id)):
raise ValueError(
"Value of the flag zipkin-trace-id must be a 16-character or 32-character hex string. "
+ "Got {}.".format(trace_id))
if parent_id and (len(parent_id) != 16
or not is_hex_string(parent_id)):
raise ValueError(
"Value of the flag zipkin-parent-id must be a 16-character hex string. "
+ "Got {}.".format(parent_id))
if zipkin_endpoint is not None:
zipkin_reporter_settings = ZipkinReporter.Settings(
log_level=Report.INFO)
zipkin_reporter = ZipkinReporter(
run_tracker,
zipkin_reporter_settings,
zipkin_endpoint,
trace_id,
parent_id,
sample_rate,
service_name_prefix,
max_span_batch_size,
)
report.add_reporter("zipkin", zipkin_reporter)
# Add some useful RunInfo.
run_tracker.run_info.add_info("default_report",
html_reporter.report_path())
port = ReportingServerManager().socket
if port:
run_tracker.run_info.add_info(
"report_url",
"http://localhost:{}/run/{}".format(port, run_id))
# And start tracking the run.
run_tracker.start(report, start_time)
def initialize(self, run_tracker, all_options, start_time=None):
"""Initialize with the given RunTracker.
TODO: See `RunTracker.start`.
"""
run_id = run_tracker.initialize(all_options)
run_dir = os.path.join(self.get_options().reports_dir, run_id)
html_dir = os.path.join(run_dir, 'html')
safe_mkdir(html_dir)
relative_symlink(run_dir, os.path.join(self.get_options().reports_dir, 'latest'))
report = Report()
# Capture initial console reporting into a buffer. We'll do something with it once
# we know what the cmd-line flag settings are.
outfile = BytesIO()
errfile = BytesIO()
capturing_reporter_settings = PlainTextReporter.Settings(
outfile=outfile, errfile=errfile, log_level=Report.INFO,
color=False, indent=True, timing=False,
cache_stats=False,
label_format=self.get_options().console_label_format,
tool_output_format=self.get_options().console_tool_output_format)
capturing_reporter = PlainTextReporter(run_tracker, capturing_reporter_settings)
report.add_reporter('capturing', capturing_reporter)
# Set up HTML reporting. We always want that.
html_reporter_settings = HtmlReporter.Settings(log_level=Report.INFO,
html_dir=html_dir,
template_dir=self.get_options().template_dir)
html_reporter = HtmlReporter(run_tracker, html_reporter_settings)
report.add_reporter('html', html_reporter)
# Set up Zipkin reporting.
zipkin_endpoint = self.get_options().zipkin_endpoint
trace_id = self.get_options().zipkin_trace_id
parent_id = self.get_options().zipkin_parent_id
if zipkin_endpoint is None and trace_id is not None and parent_id is not None:
raise ValueError(
"The zipkin-endpoint flag must be set if zipkin-trace-id and zipkin-parent-id flags are given."
)
if (trace_id is None) != (parent_id is None):
raise ValueError(
"Flags zipkin-trace-id and zipkin-parent-id must both either be set or not set."
)
if zipkin_endpoint is not None:
zipkin_reporter_settings = ZipkinReporter.Settings(log_level=Report.INFO)
zipkin_reporter = ZipkinReporter(
run_tracker, zipkin_reporter_settings, zipkin_endpoint, trace_id, parent_id
)
report.add_reporter('zipkin', zipkin_reporter)
# Add some useful RunInfo.
run_tracker.run_info.add_info('default_report', html_reporter.report_path())
port = ReportingServerManager().socket
if port:
run_tracker.run_info.add_info('report_url', 'http://localhost:{}/run/{}'.format(port, run_id))
# And start tracking the run.
run_tracker.start(report, start_time)