def histogram_plot(experiment_log_dir, plot_spec_string, output_filename,
has_legend, x_limit, verbose):
queries = [
plot_utils.plot_spec_string_to_query(plot_spec_string, 0, "HIST")]
plot_data = metaprogram_utils.process_queries(
queries, experiment_log_dir, verbose)
if "plot_points" not in plot_data:
warnings.warn("No data to plot!")
return
histogram_data = plot_data["plot_points"][0]
cumulative_histogram = {}
layout = PlotLayout()
layout.dpi = 250
for stat_name in histogram_data:
plot = Plot()
plot.setTitle(stat_name)
if has_legend:
plot.hasLegend(labelSize=8)
if x_limit is not None:
plot.setXLimits(0, x_limit)
style_plot(plot, stat_name)
for key, points in sorted(histogram_data[stat_name].items()):
for size, count in itertools.izip(points["bin"], points["count"]):
if size not in cumulative_histogram:
cumulative_histogram[size] = 0
cumulative_histogram[size] += count
line = Line()
line.stepFunction("pre")
line.label = str(key)
line.xValues = points["bin"]
line.yValues = points["count"]
plot.add(line)
layout.addPlot(plot)
cumulative_plot = Plot()
if x_limit is not None:
cumulative_plot.setXLimits(0, x_limit)
cumulative_plot.setTitle("Cumulative Histogram for " + stat_name)
style_plot(cumulative_plot, stat_name)
line = Line()
line.stepFunction("pre")
line.xValues = sorted(cumulative_histogram.keys())
line.yValues = [cumulative_histogram[key] for key in line.xValues]
cumulative_plot.add(line)
layout.addPlot(cumulative_plot)
layout.save(output_filename)
def constructImage(self):
lines = []
for i in xrange(3):
line = Line()
line.xValues = xrange(5)
line.yValues = [(i+1 / 2.0) * pow(x, i+1) for x in line.xValues]
line.label = "Line %d" % (i + 1)
lines.append(line)
plot = Plot()
plot.add(lines[0])
inset = Plot()
inset.add(lines[1])
inset.hideTickLabels()
inset.setTitle("Inset in Yo Inset\nSo You Can Inset\nWhile You Inset")
insideInset = Plot()
insideInset.hideTickLabels()
insideInset.add(lines[2])
inset.addInset(insideInset, width=0.4, height=0.3,
location="upper left")
plot.addInset(inset, width=0.4, height=0.4, location="lower right")
plot.save(self.imageName)
def generatePlot(stepType):
line = Line()
line.xValues = xVals
line.yValues = yVals
line.marker = 'o'
line.stepFunction(stepType)
plot = Plot()
plot.add(line)
plot.setTitle(r'"%s" Steps' % (stepType))
plot.setXLimits(0, 6)
plot.setYLimits(0, 6)
return plot
def constructImage(self):
plot = Plot()
line = Line()
line.yValues = [25, 40, 30, 23, 10, 50]
line.xValues = range(len(line.yValues))
line.xTickLabels = ["X 1", "X 2", "X 3", "X 4", "X 5"]
line.yTickLabels = ["Y Ten", "Y Twenty", "Y Thirty", "Y Forty",
"Y Fifty", "Y Sixty"]
line.yTickLabelPoints = [10, 20, 30, 40, 50, 60]
# You can set tick label properties with a dictionary ...
line.xTickLabelProperties = {
"color" : "blue",
"weight" : "bold",
"rotation" : 45
}
line.yTickLabelProperties = {
"style" : "italic",
"alpha" : 0.5,
"color" : "red"
}
# (clearing for demonstrative purposes)
line.xTickLabelProperties.clear()
line.yTickLabelProperties.clear()
# You can also set by direct elementwise access
line.xTickLabelProperties["color"] = "blue"
line.xTickLabelProperties["weight"] = "bold"
line.xTickLabelProperties["rotation"] = "45"
line.yTickLabelProperties["style"] = "italic"
line.yTickLabelProperties["alpha"] = 0.5
line.yTickLabelProperties["color"] = "red"
plot.add(line)
plot.setTitle("Craaazy Title")
plot.setTitleProperties(style="italic", weight="bold", rotation="5",
color="orange")
plot.setXLabel("X Label")
plot.setYLabel("Y Label")
plot.setYLimits(0, 60)
plot.setPlotParameters(bottom=.15, left=0.15)
plot.save(self.imageName)
def constructImage(self):
line = Line()
line.xValues = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
line.yValues = [0, 1, 4, 9, 16, 25, 36, 49, 64, 81, 100]
plot = Plot()
plot.useLatexLabels()
plot.setXLabel(r"$x$")
plot.setYLabel(r"$f(x) = x^2$")
plot.setTitle(r"LaTeX is Number $\sum_{n=1}^{\infty}\frac{-e^{i\pi}}{2^n}$")
plot.add(line)
layout = PlotLayout()
layout.useLatexLabels()
layout.addPlot(plot)
layout.setAxesLabelSize(18)
layout.setPlotParameters(top=0.84)
layout.save(self.imageName)
def constructImage(self):
line = Line()
line.xValues = range(5)
line.yValues = [2,3,5,7,9]
line.label = "A Line"
linePlot1 = Plot()
linePlot1.setTitle("Small Legend")
linePlot1.add(line)
linePlot1.hasLegend()
linePlot1.setLegendLabelSize(10)
linePlot2 = Plot()
linePlot2.setTitle("Large Legend")
linePlot2.add(line)
linePlot2.hasLegend()
linePlot2.setLegendLabelSize(30)
linePlot3 = Plot()
linePlot3.setTitle("Inherited from Layout")
linePlot3.add(line)
linePlot3.hasLegend()
layout = PlotLayout()
layout.setWidth(2)
layout.addPlot(linePlot1)
layout.addPlot(linePlot2)
layout.addPlot(linePlot3)
layout.setLegendLabelSize(15)
layout.setPlotParameters(left=0.03, bottom=0.03, right=0.98, top=0.94)
layout.save(self.imageName)
def constructImage(self):
line = Line()
line.xValues = range(5)
line.yValues = [2, 4, 6, 8, 10]
linePlot = Plot()
linePlot.add(line)
linePlot.setXLabel("X Data")
linePlot.setYLabel("Y Data")
linePlot.setTitle("Data as Line")
bar = Bar()
bar.xValues = range(5)
bar.yValues = [2, 4, 6, 8, 10]
barPlot = Plot()
barPlot.add(bar)
barPlot.setXLabel("X Data")
barPlot.setYLabel("Y Data")
barPlot.setTitle("Data as Bars")
scatter = Scatter()
scatter.xValues = range(5)
scatter.yValues = [2, 4, 6, 8, 10]
scatterPlot = Plot()
scatterPlot.add(scatter)
scatterPlot.setXLabel("X Data")
scatterPlot.setYLabel("Y Data")
scatterPlot.setTitle("Data as Points")
layout = PlotLayout()
# Plots in the same grouping are placed together on the same line
layout.addPlot(linePlot, grouping="topRow")
layout.addPlot(barPlot, grouping="topRow")
# Plots without a grouping are arranged as follows:
# * While you can make a row of N plots, where N is the size of the plot
# grouping with the largest size, do so.
# * If you can't make a row of N plots, make the plots stretch across a
# single row.
layout.addPlot(scatterPlot)
# Set values similar to those given in the "Configure subplots" sliders
# in the interactive figure
layout.setPlotParameters(hspace=0.48)
layout.save(self.imageName)
def histogram_plot(experiment_log_dir, plot_spec_string, output_filename,
has_legend, x_limit, verbose):
queries = [
plot_utils.plot_spec_string_to_query(plot_spec_string, 0, "HIST")
]
plot_data = metaprogram_utils.process_queries(queries, experiment_log_dir,
verbose)
if "plot_points" not in plot_data:
warnings.warn("No data to plot!")
return
histogram_data = plot_data["plot_points"][0]
cumulative_histogram = {}
layout = PlotLayout()
layout.dpi = 250
for stat_name in histogram_data:
plot = Plot()
plot.setTitle(stat_name)
if has_legend:
plot.hasLegend(labelSize=8)
if x_limit is not None:
plot.setXLimits(0, x_limit)
style_plot(plot, stat_name)
for key, points in sorted(histogram_data[stat_name].items()):
for size, count in itertools.izip(points["bin"], points["count"]):
if size not in cumulative_histogram:
cumulative_histogram[size] = 0
cumulative_histogram[size] += count
line = Line()
line.stepFunction("pre")
line.label = str(key)
line.xValues = points["bin"]
line.yValues = points["count"]
plot.add(line)
layout.addPlot(plot)
cumulative_plot = Plot()
if x_limit is not None:
cumulative_plot.setXLimits(0, x_limit)
cumulative_plot.setTitle("Cumulative Histogram for " + stat_name)
style_plot(cumulative_plot, stat_name)
line = Line()
line.stepFunction("pre")
line.xValues = sorted(cumulative_histogram.keys())
line.yValues = [cumulative_histogram[key] for key in line.xValues]
cumulative_plot.add(line)
layout.addPlot(cumulative_plot)
layout.save(output_filename)
#!/usr/bin/env python
from boomslang import Bar, Line, Scatter, Plot, PlotLayout
line = Line()
line.xValues = range(5)
line.yValues = [2, 4, 6, 8, 10]
linePlot = Plot()
linePlot.add(line)
linePlot.setXLabel("X Data")
linePlot.setYLabel("Y Data")
linePlot.setTitle("Data as Line")
bar = Bar()
bar.xValues = range(5)
bar.yValues = [2, 4, 6, 8, 10]
barPlot = Plot()
barPlot.add(bar)
barPlot.setXLabel("X Data")
barPlot.setYLabel("Y Data")
barPlot.setTitle("Data as Bars")
scatter = Scatter()
scatter.xValues = range(5)
scatter.yValues = [2, 4, 6, 8, 10]
scatterPlot = Plot()
scatterPlot.add(scatter)
#!/usr/bin/env python
from boomslang import Line, Plot, PlotLayout
line = Line()
line.xValues = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
line.yValues = [0, 1, 4, 9, 16, 25, 36, 49, 64, 81, 100]
plot = Plot()
plot.useLatexLabels()
plot.setXLabel(r"$x$")
plot.setYLabel(r"$f(x) = x^2$")
plot.setTitle(r"LaTeX is Number $\sum_{n=1}^{\infty}\frac{-e^{i\pi}}{2^n}$")
plot.add(line)
layout = PlotLayout()
layout.addPlot(plot)
layout.setAxesLabelSize(18)
layout.setPlotParameters(top=0.84)
layout.save("latex.png")
def plotExperiments(script_arguments_file):
script_arguments_fp = open(script_arguments_file, "r")
script_arguments = json.load(script_arguments_fp)
script_arguments_fp.close()
disk = script_arguments["disk"]
sort_strategies = script_arguments["sort_strategies"]
max_key_lengths = map(int, script_arguments["max_key_lengths"])
max_value_lengths = map(int, script_arguments["max_value_lengths"])
pareto_as = map(float, script_arguments["pareto_as"])
pareto_bs = map(int, script_arguments["pareto_bs"])
byte_counts = map(int, script_arguments["byte_counts"])
timeout = script_arguments["timeout"]
experiments = [(sort_strategy, max_key_length, max_value_length, pareto_a,
pareto_b, byte_count) for sort_strategy in sort_strategies
for max_key_length in max_key_lengths
for max_value_length in max_value_lengths
for pareto_a in pareto_as for pareto_b in pareto_bs
for byte_count in byte_counts]
# Open each file and store information about the experiment
file_id = 0
experiment_outcomes = {}
for experiment in experiments:
filename = os.path.join(disk, "stats_%d.json" % file_id)
sort_time = None
scratch_size = None
if os.path.exists(filename):
fp = open(filename, "r")
try:
stats_json = json.load(fp)
# time might not be present if the run timed out
if "time" in stats_json:
# sort time in seconds
sort_time = stats_json["time"] / 1000000.0
# scratch size in megabytes
scratch_size = stats_json["scratch_size"] / 1000000.0
except ValueError:
# If the JSON somehow doesn't parse, just skip this file
pass
experiment_outcomes[experiment] = (sort_time, scratch_size)
file_id += 1
# Specify colors manually so we can sync up a line to its scatter
colors = ["red", "green", "blue", "teal", "orange"]
if len(sort_strategies) > len(colors):
print >> sys.stderr, "Only %d colors but %d lines" % (
len(colors), len(sort_strategies))
sys.exit(1)
# Plot sort time vs scratch size
plot = Plot()
plot.setXLabel("Scratch Size (MB)")
plot.setYLabel("Sort Time (s)")
plot.hasLegend(labelSize=8, columns=3)
plot.setTitle("Sort Time vs Scratch Size")
lines = {}
for (sort_strategy, color) in zip(sort_strategies, colors):
line = LineAndScatter(sort_strategy, color)
lines[sort_strategy] = line
for experiment, (sort_time, scratch_size) in \
experiment_outcomes.iteritems():
# Mark this experiment on the line corresponding to its sort strategy
line = lines[experiment[0]]
if scratch_size is not None:
line.add_point(scratch_size, sort_time)
map(lambda l: l.add_to_plot(plot), lines.values())
layout = PlotLayout()
layout.addPlot(plot)
output_file = os.path.join(disk, "SortTimeVsScratchSize.pdf")
layout.save(output_file)
# Plot Sort time vs Scratch Size of a particular strategy:
# First sort the experiments by sort strategy
strategy_outcomes = {}
for experiment in experiments:
sort_strategy = experiment[0]
parameters = experiment[1:]
if parameters not in strategy_outcomes:
strategy_outcomes[parameters] = {}
strategy_outcomes[parameters][sort_strategy] = \
experiment_outcomes[experiment]
# Create one plot per strategy, with that strategy's scratch size
# on the x-axis
layout = PlotLayout()
for major_sort_strategy in sort_strategies:
# Set up plot boilerplate
plot = Plot()
plot.setXLabel("Scratch Size of %s (MB)" % (major_sort_strategy))
plot.setYLabel("Sort Time (s)")
plot.hasLegend(labelSize=8, columns=3)
plot.setTitle("Sort Time vs Scratch Size of %s" %
(major_sort_strategy))
lines = {}
for (sort_strategy, color) in zip(sort_strategies, colors):
line = LineAndScatter(sort_strategy, color)
lines[sort_strategy] = line
# For each configuration, plot the sort times on the y axis, and the
# scratch time for the major_sort_strategy on the x-axis
for outcome_by_strategy in strategy_outcomes.values():
scratch_size = outcome_by_strategy[major_sort_strategy][1]
if scratch_size is not None:
# Plot the sort time of each strategy using this scratch
# As the x-axis value.
for sort_strategy in sort_strategies:
sort_time = outcome_by_strategy[sort_strategy][0]
lines[sort_strategy].add_point(scratch_size, sort_time)
map(lambda l: l.add_to_plot(plot), lines.values())
layout.addPlot(plot)
output_file = os.path.join(disk, "SortTimeVsScratchSizeByStrategy.pdf")
layout.save(output_file)
# Plot individual parameter graphs using median parameters
median_max_key_length = median(max_key_lengths)
median_max_value_length = median(max_value_lengths)
median_pareto_a = median(pareto_as)
median_pareto_b = median(pareto_bs)
median_byte_count = median(byte_counts)
for (parameter_name, x_label, parameter_values, fixed_parameters) in [
("MaxKeyLength", "Max Key Length (bytes)", max_key_lengths,
(None, median_max_value_length, median_pareto_a, median_pareto_b,
median_byte_count)),
("MaxValueLength", "Max Value Length (bytes)", max_value_lengths,
(median_max_key_length, None, median_pareto_a, median_pareto_b,
median_byte_count)),
("ParetoA", "Pareto A", pareto_as,
(median_max_key_length, median_max_value_length, None,
median_pareto_b, median_byte_count)),
("ParetoB", "Pareto B (bytes)", pareto_bs,
(median_max_key_length, median_max_value_length, median_pareto_a,
None, median_byte_count)),
("FileSize", "File Size (bytes)", byte_counts,
(median_max_key_length, median_max_value_length, median_pareto_a,
median_pareto_b, None))
]:
time_plot = Plot()
size_plot = Plot()
for (plot, label, unit) in [(time_plot, "Sort Time", "s"),
(size_plot, "Scratch Size", "MB")]:
plot.setXLabel(x_label)
plot.setYLabel("%s (%s)" % (label, unit))
plot.hasLegend(labelSize=8, columns=3)
plot.setTitle("%s vs %s" % (label, parameter_name))
for (sort_strategy, color) in zip(sort_strategies, colors):
# Each sort strategy has its own line on each of the two plots
time_line = LineAndScatter(sort_strategy, color)
size_line = LineAndScatter(sort_strategy, color)
# Compute the set of parameters corresponding to this experiment.
for parameter in parameter_values:
# Apply this parameter to the None value in the tuple of
# fixed parameters.
parameters = [sort_strategy]
for fixed_parameter in fixed_parameters:
if fixed_parameter is None:
parameters.append(parameter)
else:
parameters.append(fixed_parameter)
# Plot sort time and scratch size for this experiment
(sort_time,
scratch_size) = experiment_outcomes[tuple(parameters)]
time_line.add_point(parameter, sort_time)
size_line.add_point(parameter, scratch_size)
# Add lines to plots
time_line.add_to_plot(time_plot)
size_line.add_to_plot(size_plot)
# Save the plot as a pdf
layout = PlotLayout()
layout.addPlot(time_plot)
layout.addPlot(size_plot)
output_file = os.path.join(disk, "%s.pdf" % (parameter_name))
layout.save(output_file)
def plot_timeline_for_phase(log_directory, job, phase, phase_data):
min_timestamp = phase_data["min_timestamp"]
max_timestamp = phase_data["max_timestamp"]
description = Description(os.path.join(log_directory, job, "description"))
stage_ordering = description.getStageOrdering(phase)
duration_lists = {}
for stage in stage_ordering:
duration_lists[stage] = []
for key in phase_data:
if key in ["min_timestamp", "max_timestamp"]:
continue
hostname, stage, worker_id = key
worker_duration_info = Duration(
hostname.split('.')[0], stage, worker_id,
(phase_data[key][0] - min_timestamp) / 1000000.0,
(phase_data[key][1] - min_timestamp) / 1000000.0)
duration_lists[stage].append(worker_duration_info)
def sort_function(x):
return (x.hostname, x.worker_id, x.start_time, x.stop_time)
layout = PlotLayout()
for stage in stage_ordering:
duration_list = duration_lists[stage]
duration_list.sort(key=sort_function)
bars = {}
# Set up a "padding" bar that will appear to move bars up so that they
# start when the worker starts
start_bar = Bar()
start_bar.linewidth = 0
start_bar.color = "white"
for i, duration in enumerate(duration_list):
if duration.hostname not in bars:
bars[duration.hostname] = Bar()
bars[duration.hostname].yValues.append(duration.stop_time -
duration.start_time)
start_bar.yValues.append(duration.start_time)
# Make sure that all bars have the same number of y-axis values,
# give them x-axis values and set their colors
start_bar.xValues = range(len(start_bar.yValues))
start_bar.xTickLabelProperties = {"rotation": 90}
bar_colors = [
"red", "blue", "green", "orange", "gray", "pink", "purple", "black"
]
offset = 0
for i, (hostname, bar) in enumerate(bars.items()):
# Pad y axis with zeroes so that bars can be laid out next to
# each other with a StackedBars
num_y_values = len(bar.yValues)
bar.yValues = (([0] * offset) + bar.yValues +
([0] * (len(duration_list) -
(num_y_values + offset))))
# Put the label for this hostname roughly in the middle of its bar
# cluster
start_bar.xTickLabels.append(hostname)
# Subtracting 0.5 to account for half the width of the bar
start_bar.xTickLabelPoints.append(offset + (num_y_values / 2.0) -
0.5)
offset += num_y_values
bar.xValues = range(len(bar.yValues))
bar.color = bar_colors[i % len(bar_colors)]
bar.label = hostname
stacked_bars = StackedBars()
stacked_bars.add(start_bar)
for hostname in sorted(bars.keys()):
stacked_bars.add(bars[hostname])
plot = Plot()
plot.setYLimits(0,
((max_timestamp - min_timestamp) / 1000000.0) * 1.05)
plot.setXLabel("Worker")
plot.setYLabel("Time (s)")
plot.setTitle(stage)
plot.add(stacked_bars)
layout.addPlot(plot)
return layout
def plot_timeline_for_phase(log_directory, job, phase, phase_data):
min_timestamp = phase_data["min_timestamp"]
max_timestamp = phase_data["max_timestamp"]
description = Description(os.path.join(log_directory, job, "description"))
stage_ordering = description.getStageOrdering(phase)
duration_lists = {}
for stage in stage_ordering:
duration_lists[stage] = []
for key in phase_data:
if key in ["min_timestamp", "max_timestamp"]:
continue
hostname, stage, worker_id = key
worker_duration_info = Duration(
hostname.split('.')[0], stage, worker_id,
(phase_data[key][0] - min_timestamp) / 1000000.0,
(phase_data[key][1] - min_timestamp) / 1000000.0)
duration_lists[stage].append(worker_duration_info)
def sort_function(x):
return (x.hostname, x.worker_id, x.start_time, x.stop_time)
layout = PlotLayout()
for stage in stage_ordering:
duration_list = duration_lists[stage]
duration_list.sort(key=sort_function)
bars = {}
# Set up a "padding" bar that will appear to move bars up so that they
# start when the worker starts
start_bar = Bar()
start_bar.linewidth = 0
start_bar.color = "white"
for i, duration in enumerate(duration_list):
if duration.hostname not in bars:
bars[duration.hostname] = Bar()
bars[duration.hostname].yValues.append(
duration.stop_time - duration.start_time)
start_bar.yValues.append(duration.start_time)
# Make sure that all bars have the same number of y-axis values,
# give them x-axis values and set their colors
start_bar.xValues = range(len(start_bar.yValues))
start_bar.xTickLabelProperties = {
"rotation" : 90
}
bar_colors = ["red", "blue", "green", "orange", "gray", "pink",
"purple", "black"]
offset = 0
for i, (hostname, bar) in enumerate(bars.items()):
# Pad y axis with zeroes so that bars can be laid out next to
# each other with a StackedBars
num_y_values = len(bar.yValues)
bar.yValues = (([0] * offset) + bar.yValues +
([0] *
(len(duration_list) - (num_y_values + offset))))
# Put the label for this hostname roughly in the middle of its bar
# cluster
start_bar.xTickLabels.append(hostname)
# Subtracting 0.5 to account for half the width of the bar
start_bar.xTickLabelPoints.append(offset + (num_y_values / 2.0)
- 0.5)
offset += num_y_values
bar.xValues = range(len(bar.yValues))
bar.color = bar_colors[i % len(bar_colors)]
bar.label = hostname
stacked_bars = StackedBars()
stacked_bars.add(start_bar)
for hostname in sorted(bars.keys()):
stacked_bars.add(bars[hostname])
plot = Plot()
plot.setYLimits(0, ((max_timestamp - min_timestamp) / 1000000.0) * 1.05)
plot.setXLabel("Worker")
plot.setYLabel("Time (s)")
plot.setTitle(stage)
plot.add(stacked_bars)
layout.addPlot(plot)
return layout
#!/usr/bin/env python
from boomslang import Line, Plot, PlotLayout
lines = []
for i in xrange(3):
line = Line()
line.xValues = xrange(5)
line.yValues = [(i + 1 / 2.0) * pow(x, i + 1) for x in line.xValues]
line.label = "Line %d" % (i + 1)
lines.append(line)
plot = Plot()
plot.add(lines[0])
inset = Plot()
inset.add(lines[1])
inset.hideTickLabels()
inset.setTitle("Inset in Yo Inset\nSo You Can Inset\nWhile You Inset")
insideInset = Plot()
insideInset.hideTickLabels()
insideInset.add(lines[2])
inset.addInset(insideInset, width=0.4, height=0.3, location="upper left")
plot.addInset(inset, width=0.4, height=0.4, location="lower right")
plot.save("inset.png")
