def histogram_plot(experiment_log_dir, plot_spec_string, output_filename,
has_legend, x_limit, verbose):
queries = [
plot_utils.plot_spec_string_to_query(plot_spec_string, 0, "HIST")]
plot_data = metaprogram_utils.process_queries(
queries, experiment_log_dir, verbose)
if "plot_points" not in plot_data:
warnings.warn("No data to plot!")
return
histogram_data = plot_data["plot_points"][0]
cumulative_histogram = {}
layout = PlotLayout()
layout.dpi = 250
for stat_name in histogram_data:
plot = Plot()
plot.setTitle(stat_name)
if has_legend:
plot.hasLegend(labelSize=8)
if x_limit is not None:
plot.setXLimits(0, x_limit)
style_plot(plot, stat_name)
for key, points in sorted(histogram_data[stat_name].items()):
for size, count in itertools.izip(points["bin"], points["count"]):
if size not in cumulative_histogram:
cumulative_histogram[size] = 0
cumulative_histogram[size] += count
line = Line()
line.stepFunction("pre")
line.label = str(key)
line.xValues = points["bin"]
line.yValues = points["count"]
plot.add(line)
layout.addPlot(plot)
cumulative_plot = Plot()
if x_limit is not None:
cumulative_plot.setXLimits(0, x_limit)
cumulative_plot.setTitle("Cumulative Histogram for " + stat_name)
style_plot(cumulative_plot, stat_name)
line = Line()
line.stepFunction("pre")
line.xValues = sorted(cumulative_histogram.keys())
line.yValues = [cumulative_histogram[key] for key in line.xValues]
cumulative_plot.add(line)
layout.addPlot(cumulative_plot)
layout.save(output_filename)
def make_line_objects_for_stat_name(
query_number, stat_name, stat_name_data, min_timestamps):
lines = []
for points_info in sorted(stat_name_data):
lines_dict = {}
lines_dict["query_number"] = query_number
lines_dict["stat_name"] = stat_name
hostname = points_info[0].split('.')[0][-3:]
if len(points_info) == 4:
(phase, stage, worker_id) = points_info[1:]
lines_dict["stage"] = stage
lines_dict["worker_id"] = worker_id
label = "h:%s s:%s i:%d n:%s" % (
hostname, stage, worker_id, stat_name)
elif len(points_info) == 3:
(phase, logger_name) = points_info[1:]
lines_dict["logger_name"] = logger_name
label = "h:%s l:%s n:%s" % (
hostname, logger_name, stat_name)
lines_dict["hostname"] = hostname
lines_dict["phase"] = phase
min_timestamp = min_timestamps[phase]
def timestamp_adjuster(x):
"""
Adjust an absolute timestamp in microseconds to time in seconds
relative to the min timestamp
"""
return float(x - min_timestamp) / 1000000.0
time_series_line = Line()
time_series_line.label = label
time_series_line.stepFunction("pre")
time_series_line.xValues = map(
timestamp_adjuster, stat_name_data[points_info]["x_values"])
time_series_line.yValues = stat_name_data[points_info]["y_values"]
lines_dict["time_series_line"] = time_series_line
lines_dict["cdf_line"] = Utils.getCDF(
stat_name_data[points_info]["y_values"])
lines_dict["cdf_line"].label = label
lines.append(lines_dict)
return lines
def make_line_objects_for_stat_name(query_number, stat_name, stat_name_data,
min_timestamps):
lines = []
for points_info in sorted(stat_name_data):
lines_dict = {}
lines_dict["query_number"] = query_number
lines_dict["stat_name"] = stat_name
hostname = points_info[0].split('.')[0][-3:]
if len(points_info) == 4:
(phase, stage, worker_id) = points_info[1:]
lines_dict["stage"] = stage
lines_dict["worker_id"] = worker_id
label = "h:%s s:%s i:%d n:%s" % (hostname, stage, worker_id,
stat_name)
elif len(points_info) == 3:
(phase, logger_name) = points_info[1:]
lines_dict["logger_name"] = logger_name
label = "h:%s l:%s n:%s" % (hostname, logger_name, stat_name)
lines_dict["hostname"] = hostname
lines_dict["phase"] = phase
min_timestamp = min_timestamps[phase]
def timestamp_adjuster(x):
"""
Adjust an absolute timestamp in microseconds to time in seconds
relative to the min timestamp
"""
return float(x - min_timestamp) / 1000000.0
time_series_line = Line()
time_series_line.label = label
time_series_line.stepFunction("pre")
time_series_line.xValues = map(timestamp_adjuster,
stat_name_data[points_info]["x_values"])
time_series_line.yValues = stat_name_data[points_info]["y_values"]
lines_dict["time_series_line"] = time_series_line
lines_dict["cdf_line"] = Utils.getCDF(
stat_name_data[points_info]["y_values"])
lines_dict["cdf_line"].label = label
lines.append(lines_dict)
return lines
def generatePlot(stepType):
line = Line()
line.xValues = xVals
line.yValues = yVals
line.marker = 'o'
line.stepFunction(stepType)
plot = Plot()
plot.add(line)
plot.setTitle(r'"%s" Steps' % (stepType))
plot.setXLimits(0, 6)
plot.setYLimits(0, 6)
return plot
def generatePlot(stepType):
line = Line()
line.xValues = xVals
line.yValues = yVals
line.marker = 'o'
line.stepFunction(stepType)
plot = Plot()
plot.add(line)
plot.title = r'"%s" Steps' % (stepType)
plot.xLimits = (0, 6)
plot.yLimits = (0, 6)
return plot
def histogram_plot(experiment_log_dir, plot_spec_string, output_filename,
has_legend, x_limit, verbose):
queries = [
plot_utils.plot_spec_string_to_query(plot_spec_string, 0, "HIST")
]
plot_data = metaprogram_utils.process_queries(queries, experiment_log_dir,
verbose)
if "plot_points" not in plot_data:
warnings.warn("No data to plot!")
return
histogram_data = plot_data["plot_points"][0]
cumulative_histogram = {}
layout = PlotLayout()
layout.dpi = 250
for stat_name in histogram_data:
plot = Plot()
plot.setTitle(stat_name)
if has_legend:
plot.hasLegend(labelSize=8)
if x_limit is not None:
plot.setXLimits(0, x_limit)
style_plot(plot, stat_name)
for key, points in sorted(histogram_data[stat_name].items()):
for size, count in itertools.izip(points["bin"], points["count"]):
if size not in cumulative_histogram:
cumulative_histogram[size] = 0
cumulative_histogram[size] += count
line = Line()
line.stepFunction("pre")
line.label = str(key)
line.xValues = points["bin"]
line.yValues = points["count"]
plot.add(line)
layout.addPlot(plot)
cumulative_plot = Plot()
if x_limit is not None:
cumulative_plot.setXLimits(0, x_limit)
cumulative_plot.setTitle("Cumulative Histogram for " + stat_name)
style_plot(cumulative_plot, stat_name)
line = Line()
line.stepFunction("pre")
line.xValues = sorted(cumulative_histogram.keys())
line.yValues = [cumulative_histogram[key] for key in line.xValues]
cumulative_plot.add(line)
layout.addPlot(cumulative_plot)
layout.save(output_filename)
