def histogram_plot(experiment_log_dir, plot_spec_string, output_filename, has_legend, x_limit, verbose): queries = [ plot_utils.plot_spec_string_to_query(plot_spec_string, 0, "HIST")] plot_data = metaprogram_utils.process_queries( queries, experiment_log_dir, verbose) if "plot_points" not in plot_data: warnings.warn("No data to plot!") return histogram_data = plot_data["plot_points"][0] cumulative_histogram = {} layout = PlotLayout() layout.dpi = 250 for stat_name in histogram_data: plot = Plot() plot.setTitle(stat_name) if has_legend: plot.hasLegend(labelSize=8) if x_limit is not None: plot.setXLimits(0, x_limit) style_plot(plot, stat_name) for key, points in sorted(histogram_data[stat_name].items()): for size, count in itertools.izip(points["bin"], points["count"]): if size not in cumulative_histogram: cumulative_histogram[size] = 0 cumulative_histogram[size] += count line = Line() line.stepFunction("pre") line.label = str(key) line.xValues = points["bin"] line.yValues = points["count"] plot.add(line) layout.addPlot(plot) cumulative_plot = Plot() if x_limit is not None: cumulative_plot.setXLimits(0, x_limit) cumulative_plot.setTitle("Cumulative Histogram for " + stat_name) style_plot(cumulative_plot, stat_name) line = Line() line.stepFunction("pre") line.xValues = sorted(cumulative_histogram.keys()) line.yValues = [cumulative_histogram[key] for key in line.xValues] cumulative_plot.add(line) layout.addPlot(cumulative_plot) layout.save(output_filename)
def make_line_objects_for_stat_name( query_number, stat_name, stat_name_data, min_timestamps): lines = [] for points_info in sorted(stat_name_data): lines_dict = {} lines_dict["query_number"] = query_number lines_dict["stat_name"] = stat_name hostname = points_info[0].split('.')[0][-3:] if len(points_info) == 4: (phase, stage, worker_id) = points_info[1:] lines_dict["stage"] = stage lines_dict["worker_id"] = worker_id label = "h:%s s:%s i:%d n:%s" % ( hostname, stage, worker_id, stat_name) elif len(points_info) == 3: (phase, logger_name) = points_info[1:] lines_dict["logger_name"] = logger_name label = "h:%s l:%s n:%s" % ( hostname, logger_name, stat_name) lines_dict["hostname"] = hostname lines_dict["phase"] = phase min_timestamp = min_timestamps[phase] def timestamp_adjuster(x): """ Adjust an absolute timestamp in microseconds to time in seconds relative to the min timestamp """ return float(x - min_timestamp) / 1000000.0 time_series_line = Line() time_series_line.label = label time_series_line.stepFunction("pre") time_series_line.xValues = map( timestamp_adjuster, stat_name_data[points_info]["x_values"]) time_series_line.yValues = stat_name_data[points_info]["y_values"] lines_dict["time_series_line"] = time_series_line lines_dict["cdf_line"] = Utils.getCDF( stat_name_data[points_info]["y_values"]) lines_dict["cdf_line"].label = label lines.append(lines_dict) return lines
def make_line_objects_for_stat_name(query_number, stat_name, stat_name_data, min_timestamps): lines = [] for points_info in sorted(stat_name_data): lines_dict = {} lines_dict["query_number"] = query_number lines_dict["stat_name"] = stat_name hostname = points_info[0].split('.')[0][-3:] if len(points_info) == 4: (phase, stage, worker_id) = points_info[1:] lines_dict["stage"] = stage lines_dict["worker_id"] = worker_id label = "h:%s s:%s i:%d n:%s" % (hostname, stage, worker_id, stat_name) elif len(points_info) == 3: (phase, logger_name) = points_info[1:] lines_dict["logger_name"] = logger_name label = "h:%s l:%s n:%s" % (hostname, logger_name, stat_name) lines_dict["hostname"] = hostname lines_dict["phase"] = phase min_timestamp = min_timestamps[phase] def timestamp_adjuster(x): """ Adjust an absolute timestamp in microseconds to time in seconds relative to the min timestamp """ return float(x - min_timestamp) / 1000000.0 time_series_line = Line() time_series_line.label = label time_series_line.stepFunction("pre") time_series_line.xValues = map(timestamp_adjuster, stat_name_data[points_info]["x_values"]) time_series_line.yValues = stat_name_data[points_info]["y_values"] lines_dict["time_series_line"] = time_series_line lines_dict["cdf_line"] = Utils.getCDF( stat_name_data[points_info]["y_values"]) lines_dict["cdf_line"].label = label lines.append(lines_dict) return lines
def generatePlot(stepType): line = Line() line.xValues = xVals line.yValues = yVals line.marker = 'o' line.stepFunction(stepType) plot = Plot() plot.add(line) plot.setTitle(r'"%s" Steps' % (stepType)) plot.setXLimits(0, 6) plot.setYLimits(0, 6) return plot
def generatePlot(stepType): line = Line() line.xValues = xVals line.yValues = yVals line.marker = 'o' line.stepFunction(stepType) plot = Plot() plot.add(line) plot.title = r'"%s" Steps' % (stepType) plot.xLimits = (0, 6) plot.yLimits = (0, 6) return plot
def histogram_plot(experiment_log_dir, plot_spec_string, output_filename, has_legend, x_limit, verbose): queries = [ plot_utils.plot_spec_string_to_query(plot_spec_string, 0, "HIST") ] plot_data = metaprogram_utils.process_queries(queries, experiment_log_dir, verbose) if "plot_points" not in plot_data: warnings.warn("No data to plot!") return histogram_data = plot_data["plot_points"][0] cumulative_histogram = {} layout = PlotLayout() layout.dpi = 250 for stat_name in histogram_data: plot = Plot() plot.setTitle(stat_name) if has_legend: plot.hasLegend(labelSize=8) if x_limit is not None: plot.setXLimits(0, x_limit) style_plot(plot, stat_name) for key, points in sorted(histogram_data[stat_name].items()): for size, count in itertools.izip(points["bin"], points["count"]): if size not in cumulative_histogram: cumulative_histogram[size] = 0 cumulative_histogram[size] += count line = Line() line.stepFunction("pre") line.label = str(key) line.xValues = points["bin"] line.yValues = points["count"] plot.add(line) layout.addPlot(plot) cumulative_plot = Plot() if x_limit is not None: cumulative_plot.setXLimits(0, x_limit) cumulative_plot.setTitle("Cumulative Histogram for " + stat_name) style_plot(cumulative_plot, stat_name) line = Line() line.stepFunction("pre") line.xValues = sorted(cumulative_histogram.keys()) line.yValues = [cumulative_histogram[key] for key in line.xValues] cumulative_plot.add(line) layout.addPlot(cumulative_plot) layout.save(output_filename)