def fit_parameters(
self,
benchmark_data_file,
degree_of_confidence,
row_limit=None,
x0=None,
bounds=None,
used_arg_indices=None,
):
if isinstance(benchmark_data_file, str):
input_arr, runtime_arr = parse_benchmark_result(
benchmark_data_file,
row_limit=row_limit,
used_arg_indices=used_arg_indices,
)
elif isinstance(benchmark_data_file, list):
if len(benchmark_data_file) == 0:
raise Exception("Received empty list")
input_arr_ = []
runtime_arr_ = []
for path in benchmark_data_file:
input_arr, runtime_arr = parse_benchmark_result(
path, row_limit=row_limit, used_arg_indices=used_arg_indices
)
input_arr_.append(input_arr)
runtime_arr_.append(runtime_arr)
input_arr = np.concatenate(input_arr_)
runtime_arr = np.concatenate(runtime_arr_)
else:
raise Exception("Invalid argument")
n_model_param = self.get_n_model_param()
model_input_size = self.get_model_input_size()
if False in [len(i) == model_input_size for i in input_arr]:
raise Exception(
"Number of arguments in data file does not match operation definition: %s"
% (self.name)
)
if model_input_size == 0:
param = (fit_constant(runtime_arr, degree_of_confidence),)
else:
param = fit(
self,
input_arr,
runtime_arr,
degree_of_confidence,
x0=x0,
bounds=bounds,
)
self.latest_param = param
return param
def output_model_surface(
self,
param,
data_file,
basename,
row_limit=None,
used_arg_indices=None,
bench_label=None,
resolution=100,
):
input_arr, runtime_arr = parse_benchmark_result(
data_file, row_limit=row_limit, used_arg_indices=used_arg_indices
)
tmp = np.hstack((input_arr, runtime_arr[:, np.newaxis]))
input_arr = np.squeeze(input_arr)
if input_arr.ndim not in [1, 2] or input_arr.size == 0:
logging.debug(
"Model surface output supported only for models of 1 or 2 variables, skipping"
)
return None
# assert input_arr.ndim <= 2
write_csv_file(basename + "_orig.csv", tmp)
ndim = input_arr.ndim
if ndim == 1:
bboxes = [(input_arr.min(), input_arr.max())]
else:
bboxes = [
(i, j) for i, j in zip(input_arr.min(axis=0), input_arr.max(axis=0))
]
# for i in range(input_arr.ndim):
model = self.get_runtime_model()
axis_values = []
for i in bboxes:
axis_values.append(np.linspace(*i, resolution))
mesh_out = np.meshgrid(*axis_values, indexing="ij")
mesh = np.array([i.flatten() for i in mesh_out]).T
# X = np.linspace(input_arr.min(), input_arr.max(), 1000)
Y = model(param, mesh)
out_arr = np.hstack((mesh, Y[:, np.newaxis]))
first_row_labels = ["arg_%d" % i for i in range(ndim)]
first_row_labels += ["val"]
write_csv_file(basename + "_model.csv", out_arr, chunk_size=resolution)
import pandas as pd
import numpy as np
import sys
import matplotlib.pyplot as plt
from opbench.helper import parse_benchmark_result
input_arr, runtime_arr = parse_benchmark_result(sys.argv[1])
indices_0 = [i for i in range(input_arr.shape[0]) if input_arr[i, 1] == 0]
indices_1 = [i for i in range(input_arr.shape[0]) if input_arr[i, 1] == 1]
# import ipdb; ipdb.set_trace()
plt.scatter(
input_arr[indices_0, 0],
runtime_arr[indices_0],
marker="x",
color="red",
label="index = 0",
)
plt.scatter(
input_arr[indices_1, 0],
runtime_arr[indices_1],
marker="x",
color="blue",
label="index = 1",
)
plt.xlabel("dest_size [byte]")
def plot_argumentless_operation(
operation,
constant,
data_file,
output_file,
row_limit=None,
used_arg_indices=None,
bench_label=None,
):
assert operation.get_model_input_size() == 0
input_arr, runtime_arr = parse_benchmark_result(
data_file, row_limit=row_limit, used_arg_indices=used_arg_indices
)
mean = np.mean(runtime_arr)
std = np.std(runtime_arr)
y_left = []
y_right = []
for i in zip(runtime_arr):
if constant >= i:
y_left.append(i)
else:
y_right.append(i)
ratio = len(y_left) / (len(y_right) + len(y_left))
fig = plt.figure()
plt.grid()
plt.hist(runtime_arr, bins=64, range=(mean - 4 * std, mean + 4 * std), ec="k")
plt.axvline(constant, color="r", linewidth=2, label="Threshold = %.4g" % constant)
title = "Operation: %s" % (operation.get_name())
if bench_label is not None:
title += ", Label: %s" % bench_label
title += "\n"
title += "Points left: %d, Points right: %d, DoC: %.1f%%" % (
len(y_left),
len(y_right),
ratio * 100,
)
plt.title(title)
plt.xlabel("Runtime [%s]" % TIME_UNIT)
plt.ylabel("Number of points")
plt.xlim([max(0, mean - 4 * std), max(mean + 4 * std, constant)])
tick_vals = [
mean - 3 * std,
mean - 2 * std,
mean - std,
mean,
mean + std,
mean + 2 * std,
mean + 3 * std,
]
tick_labels = [
"%.2g\n$\\mu-3\\sigma$" % tick_vals[0],
"%.2g\n$\\mu-2\\sigma$" % tick_vals[1],
"%.2g\n$\\mu-\\sigma$" % tick_vals[2],
"%.2g\n$\\mu$" % tick_vals[3],
"%.2g\n$\\mu+\\sigma$" % tick_vals[4],
"%.2g\n$\\mu+2\\sigma$" % tick_vals[5],
"%.2g\n$\\mu+3\\sigma$" % tick_vals[6],
]
negative = False
while True:
if len(tick_vals) == 0:
break
if tick_vals[0] > 0:
break
tick_vals.pop(0)
tick_labels.pop(0)
negative = True
if negative:
tick_vals.insert(0, 0)
tick_labels.insert(0, "0")
plt.xticks(
tick_vals, tick_labels,
)
plt.legend()
plt.tight_layout()
plt.savefig(output_file, dpi=300)
plt.close()
def plot_single_input_operation(
operation,
param,
data_file,
output_file,
row_limit=None,
used_arg_indices=None,
bench_label=None,
):
assert operation.get_model_input_size() == 1
input_arr, runtime_arr = parse_benchmark_result(
data_file, row_limit=row_limit, used_arg_indices=used_arg_indices
)
model = operation.get_runtime_model()
model_runtime = model(param, input_arr)
X = np.linspace(input_arr.min(), input_arr.max(), 1000)
# Y = np.array([MODEL(param, i) for i in X])
Y = model(param, X[:, np.newaxis])
x_below = []
x_above = []
y_below = []
y_above = []
for i, j, k in zip(input_arr, model_runtime, runtime_arr):
if j >= k:
x_below.append(i)
y_below.append(k)
else:
x_above.append(i)
y_above.append(k)
ratio = len(x_below) / (len(x_above) + len(x_below))
plt.figure()
plt.scatter(x_below, y_below, color="green", marker="x")
plt.scatter(x_above, y_above, color="blue", marker="x")
fit_label = "Fit " + operation.get_model_definition() + " -> "
fit_label += " ".join(
[
"%s = %.3g" % (i, j)
for i, j in zip(operation.get_model_parameter_labels(), param)
]
)
plt.plot(X, Y, color="red", label=fit_label)
title = "Operation: %s" % (operation.get_name())
if bench_label is not None:
title += ", Label: %s" % bench_label
title += "\n"
title += "Points above: %d, Points below: %d, DoC: %.1f%%" % (
len(x_above),
len(x_below),
ratio * 100,
)
plt.title(title)
plt.xlabel(
operation.get_model_variable_descriptions()[0]
+ " ["
+ operation.get_model_variable_units()[0]
+ "]"
)
plt.ylabel("Runtime [%s]" % TIME_UNIT)
plt.grid()
plt.legend()
plt.tight_layout()
plt.savefig(output_file, dpi=300)
import argparse
import matplotlib.pyplot as plt
from opbench.helper import parse_benchmark_result
parser = argparse.ArgumentParser(
description=
"Given a benchmark data file, plot runtime against a single argument")
parser.add_argument("csv_file")
parser.add_argument("-i",
"--index",
type=int,
required=True,
help="Index of the arg, e.g. 0, 1, 2, ...")
args = parser.parse_args()
input_arr, runtime_arr = parse_benchmark_result(args.csv_file)
plt.scatter(input_arr[:, args.index], runtime_arr, marker="x")
plt.xlabel("arg at index %d" % args.index)
plt.ylabel("runtime [picosecond]")
plt.grid()
plt.show()
