def main():
prog = "python plot_test_of_best_train.py <WhatIsThis> one/or/many/" \
"*ClassicValidationResults*.csv"
description = "Plot a test error of best training trial"
parser = ArgumentParser(description=description, prog=prog)
# General Options
parser.add_argument("--logy",
action="store_true",
dest="logy",
default=False,
help="Plot y-axis on on log scale")
parser.add_argument("--ymax",
dest="ymax",
type=float,
default=None,
help="Maximum of the y-axis")
parser.add_argument("--ymin",
dest="ymin",
type=float,
default=None,
help="Minimum of the y-axis")
parser.add_argument("--xmax",
dest="xmax",
type=float,
default=None,
help="Maximum of the x-axis")
parser.add_argument("--xmin",
dest="xmin",
type=float,
default=None,
help="Minimum of the x-axis")
parser.add_argument("-s",
"--save",
dest="save",
default="",
help="Where to save plot instead of showing it?")
parser.add_argument("-t",
"--title",
dest="title",
default="",
help="Optional supertitle for plot")
parser.add_argument("--maxvalue",
dest="maxvalue",
type=float,
default=sys.maxint,
help="Replace all values higher than this?")
parser.add_argument("--ylabel",
dest="ylabel",
default="loss",
help="Label on y-axis")
parser.add_argument("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="print number of runs on plot")
# Properties
# We need this to show defaults for -h
defaults = plot_util.get_defaults()
for key in defaults:
parser.add_argument("--%s" % key,
dest=key,
default=None,
help="%s, default: %s" % (key, str(defaults[key])))
args, unknown = parser.parse_known_args()
if len(unknown) < 2:
print "To less arguments given"
parser.print_help()
sys.exit(1)
sys.stdout.write("Found " + str(len(unknown)) + " arguments\n")
# Get files and names
file_list, name_list = read_util.get_file_and_name_list(unknown,
match_file='.csv')
for idx in range(len(name_list)):
print "%20s contains %d file(s)" % (name_list[idx], len(
file_list[idx]))
if args.verbose:
name_list = [
name_list[i] + " (" + str(len(file_list[i])) + ")"
for i in range(len(name_list))
]
# Get data from csv
train_performance = list()
test_performance = list()
time_ = list()
for name in range(len(name_list)):
# We have a new experiment
train_performance.append(list())
test_performance.append(list())
for fl in file_list[name]:
_none, csv_data = read_util.read_csv(fl, has_header=True)
csv_data = np.array(csv_data)
# Replace too high values with args.maxint
train_performance[-1].append([
min([args.maxvalue, float(i.strip())]) for i in csv_data[:, 1]
])
test_performance[-1].append([
min([args.maxvalue, float(i.strip())]) for i in csv_data[:, 2]
])
time_.append([float(i.strip()) for i in csv_data[:, 0]])
# Check whether we have the same times for all runs
if len(time_) == 2:
if time_[0] == time_[1]:
time_ = [
time_[0],
]
else:
raise NotImplementedError(".csv's do not use the same "
"timesteps")
train_performance = [np.array(i) for i in train_performance]
test_performance = [np.array(i) for i in test_performance]
time_ = np.array(time_).flatten()
# Now get the test results for the best train performance
# All arrays are numExp x numTrials
test_of_best_train = list()
min_test = list()
max_test = list()
for i in range(len(train_performance)):
test_of_best_train.append(
get_test_of_best_train(train_performance[i], test_performance[i]))
min_test.append(np.min(test_performance[i], 0))
max_test.append(np.max(test_performance[i], 0))
prop = {}
args_dict = vars(args)
for key in defaults:
prop[key] = args_dict[key]
fig = plot_optimization_trace(times=time_,
performance_list=test_of_best_train,
min_test=min_test,
max_test=max_test,
name_list=name_list,
title=args.title,
logy=args.logy,
logx=True,
properties=prop,
y_min=args.ymin,
y_max=args.ymax,
x_min=args.xmin,
x_max=args.xmax,
ylabel=args.ylabel)
if args.save != "":
print "Save plot to %s" % args.save
plot_util.save_plot(fig, args.save, plot_util.get_defaults()['dpi'])
else:
fig.show()
def main():
prog = "python plot_scatter.py any.csv"
description = "Reads performances from a two-column .csv and creates a" \
" scatterplot"
parser = ArgumentParser(description=description, prog=prog)
# General Options
# parser.add_argument("-l", "--log", action="store_true", dest="log",
# default=False, help="Plot on log scale")
parser.add_argument("--max",
dest="max",
type=float,
default=1000,
help="Maximum of both axes")
parser.add_argument("--min",
dest="min",
type=float,
default=None,
help="Minimum of both axes")
parser.add_argument("-s",
"--save",
dest="save",
default="",
help="Where to save plot instead of showing it?")
parser.add_argument("--title",
dest="title",
default="",
help="Optional supertitle for plot")
parser.add_argument("--greyFactor",
dest="grey_factor",
type=float,
default=1,
help="If an algorithms is not greyFactor-times better"
" than the other, show this point less salient, > 1")
parser.add_argument("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="Plot some debug info")
parser.add_argument("-f",
"--lineFactors",
dest="linefactors",
default=None,
help="Plot X speedup/slowdown, format "
"'X,..,X' (no spaces)")
parser.add_argument(
"-c",
"--columns",
dest="columns",
default='1,2',
help="Use these columns from csv; starting at 1, format"
" 'xaxis,yaxis' (nospaces)")
parser.add_argument("--size",
dest="user_fontsize",
default=12,
type=int,
help="Standard fontsize")
parser.add_argument("--dpi",
dest="dpi",
default=100,
type=int,
help="DPI for saved figure")
args, unknown = parser.parse_known_args()
if len(unknown) != 1:
print "Wrong number of arguments"
parser.print_help()
sys.exit(1)
if args.grey_factor < 1:
print "A grey-factor lower than one makes no sense"
parser.print_help()
sys.exit(1)
# Check selected columns
columns = [int(float(i)) for i in args.columns.split(",")]
if len(columns) != 2:
raise ValueError("Selected more or less than two columns: %s" %
str(columns))
# As python starts with 0
columns = [i - 1 for i in columns]
# Load validationResults
res_header, res_data = read_util.read_csv(unknown[0],
has_header=True,
data_type=np.float)
res_data = np.array(res_data)
print "Found %s points" % (str(res_data.shape))
# Get data
if max(columns) > res_data.shape[1] - 1:
raise ValueError("You selected column %d, but there are only %d" %
(max(columns) + 1, res_data.shape[1]))
if min(columns) < 0:
raise ValueError("You selected a column number less than 1")
data_x = res_data[:, columns[0]]
data_y = res_data[:, columns[1]]
label_x = res_header[columns[0]]
label_y = res_header[columns[1]]
linefactors = list()
if args.linefactors is not None:
linefactors = [float(i) for i in args.linefactors.split(",")]
if len(linefactors) < 1:
print "Something is wrong with linefactors: %s" % args.linefactors
sys.exit(1)
if min(linefactors) < 1:
print "A line-factor lower than one makes no sense"
sys.exit(1)
if args.grey_factor > 1 and args.grey_factor not in linefactors:
linefactors.append(args.grey_factor)
fig = scatter.plot_scatter_plot(x_data=data_x,
y_data=data_y,
labels=[label_x, label_y],
title=args.title,
max_val=args.max,
min_val=args.min,
grey_factor=args.grey_factor,
linefactors=linefactors,
debug=args.verbose,
user_fontsize=args.user_fontsize,
dpi=args.dpi)
if args.save != "":
print "Save plot to %s" % args.save
plot_util.save_plot(fig, args.save, plot_util.get_defaults()['dpi'])
else:
fig.show()
def main():
prog = "python plot_scatter_train_test.py " \
"RANDOM_train one/or/many/validationResults-cli-*.csv " \
"RANDOM_test one/or/many/validationResults-cli-*.csv " \
"SMAC_train one/or/many/validationResults-traj-run-*.csv" \
"SMAC_test one/or/many/validationResults-traj-run-*.csv"
description = "Scatter PAR10 on train and test instances "
parser = ArgumentParser(description=description,
prog=prog,
formatter_class=ArgumentDefaultsHelpFormatter)
# General Options
parser.add_argument("--logx",
action="store_true",
dest="logx",
default=False,
help="Plot x-axis on log scale")
parser.add_argument("--logy",
action="store_true",
dest="logy",
default=False,
help="Plot y-axis on log scale")
parser.add_argument("--max",
dest="max",
type=float,
default=None,
help="Maximum of the axes")
parser.add_argument("--min",
dest="min",
type=float,
default=None,
help="Minimum of the axes")
parser.add_argument("-s",
"--save",
dest="save",
default="",
help="Where to save plot instead of "
"showing it?")
parser.add_argument("-t",
"--title",
dest="title",
default="",
help="Optional supertitle for plot")
parser.add_argument("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="print number of runs on plot")
parser.add_argument("--log",
dest="log",
action="store_true",
default=False,
help="Set axes to log scale")
parser.add_argument("--par",
dest="par",
type=int,
default=None,
help="Only used for ObjectiveMatrix to calculate PAR "
"score")
parser.add_argument("--cutoff",
dest="cutoff",
type=int,
default=None,
help="Only used for ObjectiveMatrix to calculate PAR "
"score")
parser.add_argument("--firstRandomAsDefault",
dest="firstRandomAsDefault",
default=False,
action="store_true",
help="If 'RANDOM' in name, then use the first config "
"as default")
parser.add_argument(
"--correlation",
dest="correlation",
default=False,
action="store_true",
help="Show Spearman rank-order correlation coefficient")
parser.add_argument("--default",
dest="default",
default=False,
action="store_true",
help="If 'default' in name use different marker style")
# Properties
# We need this to show defaults for -h
defaults = plot_util.get_defaults()
for key in defaults:
parser.add_argument("--%s" % key,
dest=key,
default=None,
help="%s, default: %s" % (key, str(defaults[key])))
args, unknown = parser.parse_known_args()
# Get files and names
file_list, name_list = read_util.get_file_and_name_list(unknown,
match_file='.csv')
if len(name_list) % 2 != 0:
raise ValueError(
"NOT A MULTIPLE OF TWO. ALWAYS NEED TEST AND TRAIN(%s)")
value_dict = collections.OrderedDict()
name_ls = []
min_ = macros.MAXINT
max_ = -macros.MAXINT
for name in range(len(name_list)):
base_name = "_".join(name_list[name].split("_")[:-1])
ext = name_list[name].split("_")[-1]
assert ext in ("train", "test"), "%s" % name_list[name]
name_ls.append(base_name)
value_dict[name_list[name]] = list()
for fl in file_list[name]:
try:
data = read_util.read_validationObjectiveMatrix_file(fl)
assert args.cutoff is not None, "If reading Objective Matrix " \
"you need to set --cutoff"
assert args.par is not None, "If reading Objective Matrix " \
"you need to set --par"
perf = list()
for cf in data:
row = np.array(data[cf])
row[row >= args.cutoff] = args.par * args.cutoff
perf.append(row)
perf = np.array(perf)
value_dict[name_list[name]].append(np.mean(perf, axis=0))
min_ = np.min((min_, np.min(np.mean(perf, axis=0))))
max_ = np.max((max_, np.max(np.mean(perf, axis=0))))
except ValueError:
print("Trying to read trajectory file")
data = read_util.read_trajectory_file(fl)
for entry in data:
value_dict[name_list[name]].append(
entry["Test Set Performance"])
min_ = np.min((min_, entry["Test Set Performance"]))
max_ = np.max((max_, entry["Test Set Performance"]))
value_dict[name_list[name]] = np.array(
value_dict[name_list[name]]).ravel()
print(value_dict[name_list[name]].shape)
name_ls = sorted(list(set(name_ls)))
################### Calculate correlation
if args.correlation:
for base_name in name_ls:
if len(value_dict[base_name + "_train"]) < 2:
print("% 15s has only one entry" % base_name)
continue
corr, pvalue = scipy.stats.spearmanr(
value_dict[base_name + "_train"],
value_dict[base_name + "_test"])
print("% 50s has a correlation of % 5g" % (base_name[:50], corr))
# Calc correlation for 20% best
idx = np.argsort(value_dict[base_name + "_train"])
idx = idx[:int(len(idx) * 0.2)]
corr, pvalue = scipy.stats.spearmanr(
value_dict[base_name + "_train"][idx],
value_dict[base_name + "_test"][idx])
print("Top 20%% (% 3d) of % 33s has a correlation of % 5g" %
(len(idx), base_name[:32], corr))
################### Plotting starts here
################### TODO: Put plotting code in separate script
properties = {}
args_dict = vars(args)
for key in defaults:
properties[key] = args_dict[key]
try:
properties[key] = float(properties[key])
if int(properties[key]) == properties[key]:
properties[key] = int(properties[key])
except:
continue
properties = plot_util.fill_with_defaults(properties)
size = 1
# Set up figure
ratio = 5
gs = matplotlib.gridspec.GridSpec(ratio, 1)
fig = figure(1, dpi=int(properties['dpi'])) #, figsize=(8, 4))
ax1 = subplot(gs[0:ratio, :])
ax1.grid(True,
linestyle='-',
which='major',
color=properties["gridcolor"],
alpha=float(properties["gridalpha"]))
for base_name in reversed(name_ls):
if (args.default and "DEFAULT" in base_name.upper()) or \
(base_name.upper() == "RANDOM" and args.firstRandomAsDefault):
# Do not plot using alpha
alpha = 1
zorder = 99
c = 'k'
marker = 'x'
edgecolor = c
size = properties["markersize"] * 2
label = "default configuration"
linewidth = 3
else:
alpha = 0.5
zorder = 1
c = next(properties["colors"])
edgecolor = ''
marker = next(properties["markers"])
size = properties["markersize"]
label = base_name.replace("_", " ")
linewidth = 0
ax1.scatter(value_dict[base_name + "_train"],
value_dict[base_name + "_test"],
label=label,
marker=marker,
c=c,
edgecolor=edgecolor,
s=size,
alpha=alpha,
zorder=zorder,
linewidth=linewidth)
if properties["legendlocation"] != 'None':
ax1.legend(loc=properties["legendlocation"],
framealpha=1,
fancybox=True,
ncol=1,
scatterpoints=1,
prop={'size': int(properties["legendsize"])})
ax1.set_xlabel("PAR10 on training set",
fontsize=properties["labelfontsize"])
ax1.set_ylabel("PAR10 on test set", fontsize=properties["labelfontsize"])
tick_params(axis='both',
which='major',
labelsize=properties["ticklabelsize"])
ax1.plot([0.1, 3500], [0.1, 3500], c='k', zorder=0)
if args.max is not None:
if args.min is not None:
ax1.set_xlim([args.min, args.max])
ax1.set_ylim([args.min, args.max])
else:
ax1.set_xlim([min_, args.max])
ax1.set_ylim([min_, args.max])
else:
if args.min is not None:
ax1.set_xlim([args.min, max_])
ax1.set_ylim([args.min, max_])
else:
ax1.set_xlim([min_, max_])
ax1.set_ylim([min_, max_])
ax1.set_aspect('equal')
if args.log:
ax1.set_xscale("log")
ax1.set_yscale("log")
if args.save != "":
print("Save plot to %s" % args.save)
plot_util.save_plot(fig, args.save, plot_util.get_defaults()['dpi'])
else:
fig.show()
def main():
prog = "python plot_bar_plots.py <Dataset> <model> " \
"exactly_one_.csv ... "
description = "Plot bar plots per dataset"
parser = ArgumentParser(description=description, prog=prog,
formatter_class=ArgumentDefaultsHelpFormatter)
# General Options
parser.add_argument("--logy", action="store_true", dest="logy",
default=False, help="Plot y-axis on log scale")
parser.add_argument("--limit", default=3600.0, type=float, help="Moment in time to assess the quality")
parser.add_argument("-s", "--save", dest="save",
default="",
help="Where to save plot instead of showing it?")
parser.add_argument("-t", "--title", dest="title",
default="", help="Optional supertitle for plot")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
default=False, help="print number of runs on plot")
# Properties
# We need this to show defaults for -h
defaults = plot_util.get_defaults()
for key in defaults:
parser.add_argument("--%s" % key, dest=key, default=None,
help="%s, default: %s" % (key, str(defaults[key])))
args, unknown = parser.parse_known_args()
limit = args.limit
sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n")
if len(unknown) < 2:
print("To less arguments given")
parser.print_help()
sys.exit(1)
# Get files and names
file_list, name_list = read_util.get_file_and_name_list(unknown,
match_file='.csv',
len_name=2)
datasets = set()
strategies = set()
strategy_dataset = {}
strategy_color = {}
colors = plot_util.get_plot_colors()
for idx in range(len(name_list)):
dataset = name_list[idx][0]
strategy = name_list[idx][1]
datasets.add(dataset)
strategies.add(strategy)
if strategy not in strategy_color:
strategy_color[strategy] = next(colors)
if strategy not in strategy_dataset:
strategy_dataset[strategy] = {}
strategy_dataset[strategy][dataset] = (dataset, file_list[idx][0])
assert len(file_list[idx]) == 1, "%s" % file_list[idx]
print("%20s contains %d file(s)" %
(name_list[idx], len(file_list[idx])))
results = OrderedDict()
for strategy in sorted(strategies):
means = list()
labels = []
for dataset in sorted(strategy_dataset[strategy]):
loss = 1.0
dataset_id = strategy_dataset[strategy][dataset][0]
file_path = strategy_dataset[strategy][dataset][1]
labels.append(dataset_id)
with open(file_path, 'r') as csvfile:
csvreader = csv.reader(csvfile, delimiter=',', quotechar='"')
for idx, row in enumerate(csvreader):
if idx > 0:
time = float(row[0])
test = float(row[2])
if time < limit:
loss = test
means.append(loss)
results[strategy] = means
ind = np.arange(len(datasets)) # the x locations for the groups
width = 0.15 # the width of the bars
fig, ax = plt.subplots()
rects = []
legends = []
for idx, strategy in enumerate(results):
rects.append(ax.bar(ind + idx * width, results[strategy], width, color=strategy_color[strategy]))
legends.append(strategy)
# add some text for labels, title and axes ticks
ax.set_ylabel('Result')
ax.set_xticks(ind + width / 2)
ax.set_xticklabels(labels, rotation='vertical')
ax.legend(rects, legends)
def autolabel(rects):
"""
Attach a text label above each bar displaying its height
"""
for rect in rects:
height = rect.get_height()
ax.text(rect.get_x() + rect.get_width() / 2., 1.05 * height,
'%f' % height,
ha='center', va='bottom')
# for rect in rects:
# autolabel(rect)
if args.save != "":
print("Save plot to %s" % args.save)
plot_util.save_plot(fig, args.save, plot_util.get_defaults()['dpi'])
else:
fig.show()
def main():
prog = "python plot_performance <WhatIsThis> one/or/many/runs_and_res*.csv"
description = "Plot a median trace with quantiles for multiple experiments"
parser = ArgumentParser(description=description,
prog=prog,
formatter_class=ArgumentDefaultsHelpFormatter)
# General Options
parser.add_argument("-l",
"--log",
action="store_true",
dest="log",
default=False,
help="Plot on log scale")
parser.add_argument("--ymax",
dest="ymax",
type=float,
default=None,
help="Maximum of the y-axis")
parser.add_argument("--ymin",
dest="ymin",
type=float,
default=None,
help="Minimum of the y-axis")
parser.add_argument("--xmax",
dest="xmax",
type=float,
default=None,
help="Maximum of the x-axis")
parser.add_argument("--xmin",
dest="xmin",
type=float,
default=None,
help="Minimum of the x-axis")
parser.add_argument("-s",
"--save",
dest="save",
default="",
help="Where to save plot instead of showing it?")
parser.add_argument("-t",
"--title",
dest="title",
default="",
help="Optional supertitle for plot")
parser.add_argument("--maxvalue",
dest="maxvalue",
type=float,
default=plottingscripts.utils.macros.MAXINT,
help="Replace all values higher than this?")
parser.add_argument("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="print number of runs on plot")
parser.add_argument("--agglomeration",
dest="agglomeration",
type=str,
default="median",
help="Show mean or median",
choices=("mean", "median"))
parser.add_argument("--ylabel", dest="ylabel", default="Performance")
parser.add_argument("--optimum",
dest="optimum",
default=0,
type=float,
help="Plot difference to optimum")
args, unknown = parser.parse_known_args()
sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n")
if len(unknown) < 2:
print("To less arguments given")
parser.print_help()
sys.exit(1)
# Get files and names
file_list, name_list = read_util.get_file_and_name_list(unknown,
match_file=".")
for idx in range(len(name_list)):
print("%20s contains %d file(s)" %
(name_list[idx], len(file_list[idx])))
if args.verbose:
name_list = [
name_list[i] + " (" + str(len(file_list[i])) + ")"
for i in range(len(name_list))
]
# Get data from csv
performance_list = list()
time_list = list()
show_from = -plottingscripts.utils.macros.MAXINT
for name in range(len(name_list)):
# We have a new experiment
performance = list()
time_ = list()
for fl in file_list[name]:
_none, csv_data = read_util.read_csv(fl, has_header=True)
csv_data = np.array(csv_data)
# Replace too high values with args.maxint
data = [
min([args.maxvalue,
float(i.strip()) - args.optimum]) for i in csv_data[:, 1]
]
# do we have only non maxint data?
show_from = max(data.count(args.maxvalue), show_from)
performance.append(data)
time_.append([float(i.strip()) for i in csv_data[:, 0]])
if len(time_) > 1:
performance, time_ = merge_test_performance_different_times.\
fill_trajectory(performance_list=performance, time_list=time_)
print(performance[0][:10])
else:
time_ = time_[0]
performance = [np.array(i) for i in performance]
time_ = np.array(time_)
performance_list.append(performance)
time_list.append(time_)
if args.xmin is None and show_from != 0:
args.xmin = show_from
fig = plot_methods.\
plot_optimization_trace_mult_exp(time_list=time_list,
performance_list=performance_list,
title=args.title,
name_list=name_list,
logx=args.log, logy=False,
agglomeration=args.agglomeration,
y_min=args.ymin,
y_max=args.ymax,
x_min=args.xmin,
x_max=args.xmax,
ylabel=args.ylabel)
if args.save != "":
print("Save plot to %s" % args.save)
plot_util.save_plot(fig, args.save, plot_util.get_defaults()['dpi'])
else:
fig.show()
def main():
prog = "python plot_scatter.py"
description = "Plots performances of the best config at one time vs " \
"another in a scatter plot"
parser = ArgumentParser(description=description, prog=prog)
# General Options
parser.add_argument("--max", dest="max", type=float,
default=1000, help="Maximum of both axes")
parser.add_argument("--min", dest="min", type=float,
default=None, help="Minimum of both axes")
parser.add_argument("-s", "--save", dest="save",
default="", help="Where to save plot instead of "
"showing it?")
parser.add_argument("--title", dest="title",
default="", help="Optional supertitle for plot")
parser.add_argument("--greyFactor", dest="grey_factor", type=float,
default=1, help="If an algorithms is not greyFactor-"
"times better than the other, show this"
" point less salient, > 1")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
default=False, help="Plot some debug info")
parser.add_argument("-f", "--lineFactors", dest="linefactors",
default=None, help="Plot X speedup/slowdown,"
" format 'X,..,X' (no spaces)")
parser.add_argument("--time", dest="time", default=None,
help="Plot config at which time?, format 'time1,time2'")
parser.add_argument("--obj", dest="obj", default=None, required=True,
help="Path to validationObjectiveMatrix-traj-* file")
parser.add_argument("--res", dest="res", required=True,
help="Path to validationResults-traj-run-* file")
parser.add_argument("--minvalue", dest="minvalue", type=float,
help="Replace all values smaller than this",)
parser.add_argument("--fontsize", dest="fontsize", type=int, default=20,
help="Use this fontsize for plotting",)
args, unknown = parser.parse_known_args()
if len(unknown) != 0:
print("Wrong number of arguments")
parser.print_help()
sys.exit(1)
if args.grey_factor < 1:
print("A grey-factor lower than one makes no sense")
parser.print_help()
sys.exit(1)
# Load validationResults
res_header, res_data = read_util.read_csv(args.res, has_header=True)
av_times = [float(row[0]) for row in res_data]
if args.time is None:
# Print available times and quit
print("Choose a time from")
print("\n".join(["* %s" % i for i in av_times]))
sys.exit(0)
time_arr = args.time.split(",")
if len(time_arr) != 2 or \
(len(time_arr) == 2 and (time_arr[1] == "" or time_arr[0] == "")):
print("Something wrong with %s, should be 'a,b'" % args.time)
print("Choose a time from")
print("\n".join(["* %s" % i for i in av_times]))
sys.exit(0)
time_1 = float(time_arr[0])
time_2 = float(time_arr[1])
# Now extract data
config_1 = [int(float(row[len(res_header)-2].strip('"'))) for row in
res_data if int(float(row[0])) == int(time_1)]
config_2 = [int(float(row[len(res_header)-2].strip('"'))) for row in
res_data if int(float(row[0])) == int(time_2)]
if len(config_1) == 0 or len(config_2) == 0:
print("Time int(%s) or int(%s) not found. Choose a time from:" %
(time_1, time_2))
print("\n".join(["* %s" % i for i in av_times]))
sys.exit(1)
config_1 = config_1[0]
config_2 = config_2[0]
obj_header, obj_data = read_util.read_csv(args.obj, has_header=True)
head_template = '"Objective of validation config #%s"'
idx_1 = obj_header.index(head_template % config_1)
idx_2 = obj_header.index(head_template % config_2)
data_one = np.array([float(row[idx_1].strip('"')) for row in obj_data])
data_two = np.array([float(row[idx_2].strip('"')) for row in obj_data])
print("Found %s points for config %d and %s points for config %d" %
(str(data_one.shape), config_1, str(data_two.shape), config_2))
linefactors = list()
if args.linefactors is not None:
linefactors = [float(i) for i in args.linefactors.split(",")]
if len(linefactors) < 1:
print("Something is wrong with linefactors: %s" % args.linefactors)
sys.exit(1)
if min(linefactors) < 1:
print("A line-factor lower than one makes no sense")
sys.exit(1)
if args.grey_factor > 1 and args.grey_factor not in linefactors:
linefactors.append(args.grey_factor)
label_template = 'Objective of validation config #%s, best at %s sec'
# This might produce overhead for large .csv files
times = [int(float(row[0])) for row in res_data]
time_1 = res_data[times.index(int(time_1))][0]
time_2 = res_data[times.index(int(time_2))][0]
data_one = np.array([max(args.minvalue, i) for i in data_one])
data_two = np.array([max(args.minvalue, i) for i in data_two])
fig = scatter.plot_scatter_plot(x_data=data_one, y_data=data_two,
labels=[label_template %
(config_1, str(time_1)),
label_template %
(config_2, str(time_2))],
title=args.title,
max_val=args.max, min_val=args.min,
grey_factor=args.grey_factor,
linefactors=linefactors,
user_fontsize=args.fontsize,
debug=args.verbose)
if args.save != "":
print("Save plot to %s" % args.save)
plot_util.save_plot(fig, args.save, plot_util.get_defaults()['dpi'])
else:
fig.show()
def main():
prog = "python plot_ranks_from_csv.py <Dataset> <model> " \
"*.csv ... "
description = "Plot ranks over different datasets"
parser = ArgumentParser(description=description,
prog=prog,
formatter_class=ArgumentDefaultsHelpFormatter)
# General Options
parser.add_argument("--logy",
action="store_true",
dest="logy",
default=False,
help="Plot y-axis on log scale")
parser.add_argument("--logx",
action="store_true",
dest="logx",
default=False,
help="Plot x-axis on log scale")
parser.add_argument("--ymax",
dest="ymax",
type=float,
default=None,
help="Maximum of the y-axis")
parser.add_argument("--ymin",
dest="ymin",
type=float,
default=None,
help="Minimum of the y-axis")
parser.add_argument("--xmax",
dest="xmax",
type=float,
default=None,
help="Maximum of the x-axis")
parser.add_argument("--xmin",
dest="xmin",
type=float,
default=None,
help="Minimum of the x-axis")
parser.add_argument("-s",
"--save",
dest="save",
default="",
help="Where to save plot instead of showing it?")
parser.add_argument("-t",
"--title",
dest="title",
default="",
help="Optional supertitle for plot")
parser.add_argument("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="print number of runs on plot")
parser.add_argument("--samples",
dest="samples",
type=int,
default=10,
help="Number of bootstrap samples to plot")
parser.add_argument("--xlabel",
type=str,
default=None,
help="x label (overrides default)")
parser.add_argument("--ylabel",
type=str,
default=None,
help="y label (overrides default)")
# Properties
# We need this to show defaults for -h
defaults = plot_util.get_defaults()
for key in defaults:
parser.add_argument("--%s" % key,
dest=key,
default=None,
help="%s, default: %s" % (key, str(defaults[key])))
args, unknown = parser.parse_known_args()
sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n")
if len(unknown) < 2:
print("To less arguments given")
parser.print_help()
sys.exit(1)
# Get files and names
file_list, name_list = read_util.get_file_and_name_list(unknown,
match_file='.csv',
len_name=2)
for idx in range(len(name_list)):
assert len(file_list[idx]) == 1, "%s" % file_list[idx]
print("%20s contains %d file(s)" %
(name_list[idx], len(file_list[idx])))
dataset_dict, dataset_list, estimator_list = read_data(
file_list, name_list) # Make lists
estimator_list = sorted(list(estimator_list))
dataset_list = sorted(list(dataset_list))
print("Found datasets: %s" % str(dataset_list))
print("Found estimators: %s" % str(estimator_list))
for dataset in dataset_list:
# In order to use fill trajectory for all runs on one dataset,
# the trajectories for each run need to be in one array -> flatten the
# array and put it together afterwards
print("Processing dataset: %s" % dataset)
if dataset not in dataset_dict:
# This should never happen
raise ValueError("Dataset %s lost" % dataset)
# Merge the times of this dataset
performances_per_estimator = list()
times_per_estimator = list()
num_performances_per_estimator = list()
for est in dataset_dict[dataset]:
num_performances = len(dataset_dict[dataset][est]['performances'])
num_performances_per_estimator.extend([est] * num_performances)
performances_per_estimator.extend(
dataset_dict[dataset][est]['performances'])
times_per_estimator.extend([dataset_dict[dataset][est]['times']] *
num_performances)
performances, times = fill_trajectory(
performance_list=performances_per_estimator,
time_list=times_per_estimator)
assert performances.shape[0] == times.shape[0], \
(performances.shape[0], times.shape[0])
for i, est in enumerate(dataset_dict[dataset]):
dataset_dict[dataset][est]['performances'] = list()
dataset_dict[dataset][est]['times'] = list()
for est, perf in zip(num_performances_per_estimator,
performances.transpose()):
dataset_dict[dataset][est]['performances'].append(perf)
dataset_dict[dataset][est]['times'].append(times)
# Calculate rankings
ranking_list = list()
time_list = list()
for dataset in dataset_list:
ranking, e_list = calculate_ranking(performances=dataset_dict[dataset],
estimators=estimator_list,
bootstrap_samples=args.samples)
ranking_list.extend(ranking)
assert len(e_list) == len(estimator_list)
time_list.extend(
[dataset_dict[dataset][e]["times"][0] for e in e_list])
# Fill trajectories as ranks are calculated on different time steps
# sanity check
assert len(ranking_list) == len(time_list), (len(ranking_list),
len(time_list))
assert len(ranking_list[0]) == len(time_list[0]), "%d is not %d" % \
(len(ranking_list[0]),
len(time_list[0]))
p, times = fill_trajectory(performance_list=ranking_list,
time_list=time_list)
del ranking_list, dataset_dict
p = p.transpose()
performance_list = [list() for e in estimator_list]
time_list = [times for e in estimator_list]
for idd, dataset in enumerate(dataset_list):
for ide, est in enumerate(estimator_list):
performance_list[ide].append(p[idd * (len(estimator_list)) + ide])
for entry in performance_list:
assert np.array(entry).shape[1] == time_list[0].shape[0], \
(np.array(entry).shape[1], time_list[0].shape)
prop = {}
args_dict = vars(args)
for key in defaults:
prop[key] = args_dict[key]
#prop['linestyles'] = itertools.cycle(["-", ":"])
ylabel = "average rank (%d bootstrap samples)" % args.samples
if args.ylabel:
ylabel = args.ylabel
fig = plot_methods.plot_optimization_trace_mult_exp(
time_list=time_list,
performance_list=performance_list,
title=args.title,
name_list=estimator_list,
logy=args.logy,
logx=args.logx,
y_min=args.ymin,
y_max=args.ymax,
x_min=args.xmin,
x_max=args.xmax,
xlabel=args.xlabel,
ylabel=ylabel,
scale_std=0,
properties=prop)
if args.save != "":
print("Save plot to %s" % args.save)
plot_util.save_plot(fig, args.save, plot_util.get_defaults()['dpi'])
else:
fig.show()
def main():
prog = "python plot_performance_wo_Timeouts <WhatIsThis> " \
"one/or/many/validationObjectiveMatrix*.csv"
description = "Plot a median trace with quantiles for multiple " \
"experiments, but ignore Timeouts"
parser = ArgumentParser(description=description, prog=prog)
# General Options
parser.add_argument("--logx",
action="store_true",
dest="logx",
default=False,
help="Plot x-axis on log scale")
parser.add_argument("--logy",
action="store_true",
dest="logy",
default=False,
help="Plot y-axis on log scale")
parser.add_argument("--ymax",
dest="ymax",
type=float,
default=None,
help="Maximum of the y-axis")
parser.add_argument("--ymin",
dest="ymin",
type=float,
default=None,
help="Minimum of the y-axis")
parser.add_argument("--ylabel",
dest="ylabel",
default=None,
help="Label on y-axis")
parser.add_argument("-s",
"--save",
dest="save",
default="",
help="Where to save plot instead of showing it?")
parser.add_argument("-t",
"--title",
dest="title",
default="",
help="Optional supertitle for plot")
parser.add_argument("--maxvalue",
dest="maxvalue",
type=float,
default=sys.maxint,
help="Replace all values higher than this?")
parser.add_argument("--agglomeration",
dest="agglomeration",
type=str,
default="median",
choices=("median", "mean"),
help="Plot mean or median")
parser.add_argument("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="print number of runs on plot")
parser.add_argument("-c",
"--cutoff",
dest="cutoff",
required=True,
type=float,
help="Cutoff of this scenario")
args, unknown = parser.parse_known_args()
sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n")
if len(unknown) < 2:
print "To less arguments given"
parser.print_help()
sys.exit(1)
if args.ylabel is None:
args.ylabel = "%s performance on instances" % args.agglomeration
# Get files and names
file_list, name_list = read_util.get_file_and_name_list(unknown,
match_file='.csv')
for idx in range(len(name_list)):
print "%20s contains %d file(s)" % (name_list[idx], len(
file_list[idx]))
if args.verbose:
name_list = [
name_list[i] + " (" + str(len(file_list[i])) + ")"
for i in range(len(name_list))
]
# Get data from csv
performance = list()
time_ = list()
for name in range(len(name_list)):
# We have a new experiment
performance.append(list())
time_.append([1, 2])
for fl in file_list[name]:
data_dict = read_util.read_validationObjectiveMatrix_file(fl)
# Get Performance for each step
tmp_performance = list()
for idx in [0, -1]:
summer = [data_dict[inst][idx] for inst in data_dict.keys()]
summer = np.array(summer)
notTimeout_idx = summer < args.cutoff
summer = summer[notTimeout_idx]
tmp_performance.append(np.mean(summer))
performance[-1].append(tmp_performance)
performance = [np.array(i) for i in performance]
# This plotting function requires a time array for each experiment
fig = plot_methods.plot_optimization_trace_mult_exp(
time_list=time_,
performance_list=performance,
title=args.title,
name_list=name_list,
logx=args.logx,
logy=args.logy,
y_max=args.ymax,
y_min=args.ymin,
x_min=0.8,
x_max=2.2,
agglomeration=args.agglomeration,
ylabel=args.ylabel)
if args.save != "":
print "Save plot to %s" % args.save
plot_util.save_plot(fig, args.save, plot_util.get_defaults()['dpi'])
else:
fig.show()
def main():
prog = "python plot_ac_overhead <WhatIsThis> one/or/many/*ValidationResults-traj-run*.csv"
description = "Plot a median trace with quantiles for multiple experiments"
parser = ArgumentParser(description=description, prog=prog)
# General Options
parser.add_argument("--logx", action="store_true", dest="logx",
default=False, help="Plot x-axis on log scale")
parser.add_argument("--logy", action="store_true", dest="logy",
default=False, help="Plot y-axis on log scale")
parser.add_argument("--ymax", dest="ymax", type=float,
default=None, help="Maximum of the y-axis")
parser.add_argument("--ymin", dest="ymin", type=float,
default=None, help="Minimum of the y-axis")
parser.add_argument("--xmax", dest="xmax", type=float,
default=None, help="Maximum of the x-axis")
parser.add_argument("--xmin", dest="xmin", type=float,
default=None, help="Minimum of the x-axis")
parser.add_argument("--ylabel", dest="ylabel", default=None,
help="Label on y-axis")
parser.add_argument("-s", "--save", dest="save",
default="", help="Where to save plot instead of showing it?")
parser.add_argument("-t", "--title", dest="title",
default="", help="Optional supertitle for plot")
parser.add_argument("--maxvalue", dest="maxvalue", type=float,
default=sys.maxint, help="Replace all values higher than this?")
parser.add_argument("--agglomeration", dest="agglomeration", type=str,
default="median", choices=("median", "mean"),
help="Plot mean or median")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=False,
help="print number of runs on plot")
args, unknown = parser.parse_known_args()
sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n")
if len(unknown) < 2:
print "To less arguments given"
parser.print_help()
sys.exit(1)
if args.ylabel is None:
args.ylabel = "%s AC overhead [sec]" % args.agglomeration
# Get files and names
file_list, name_list = read_util.get_file_and_name_list(unknown, match_file='.csv')
for idx in range(len(name_list)):
print "%20s contains %d file(s)" % (name_list[idx], len(file_list[idx]))
if args.verbose:
name_list = [name_list[i] + " (" + str(len(file_list[i])) + ")" for i in range(len(name_list))]
# Get data from csv
overhead = list()
time_ = list()
show_from = -sys.maxint
for name in range(len(name_list)):
# We have a new experiment
overhead.append(list())
for fl in file_list[name]:
_none, csv_data = read_util.read_csv(fl, has_header=True)
csv_data = np.array(csv_data)
overhead[-1].append(csv_data[:, 3])
time_.append([float(i.strip()) for i in csv_data[:, 0]])
# Check whether we have the same times for all runs
if len(time_) == 2:
if time_[0] == time_[1]:
time_ = [time_[0], ]
else:
raise NotImplementedError(".csv are not using the same times")
overhead = [np.array(i, dtype=np.float32) for i in overhead]
# print time_
time_ = np.array(time_).flatten()
# This plotting function requires a time array for each experiment
new_time_list = [time_ for i in range(len(overhead))]
fig = plot_methods.plot_optimization_trace_mult_exp(time_list=new_time_list,
performance_list=overhead,
title=args.title,
name_list=name_list,
logx=args.logx,
logy=args.logy,
y_min=args.ymin,
y_max=args.ymax,
x_min=args.xmin,
x_max=args.xmax,
agglomeration=args.agglomeration,
ylabel=args.ylabel)
if args.save != "":
print "Save plot to %s" % args.save
plot_util.save_plot(fig, args.save, plot_util.get_defaults()['dpi'])
else:
fig.show()
def main():
prog = "python plot_TestvsTrainPerformance.py <WhatIsThis> " \
"one/or/many/*ClassicValidationResults*.csv"
description = "Plot a median trace with quantiles for multiple experiments"
parser = ArgumentParser(description=description, prog=prog)
# General Options
parser.add_argument("--logx",
action="store_true",
dest="logx",
default=False,
help="Plot x-axis on log scale")
parser.add_argument("--logy",
action="store_true",
dest="logy",
default=False,
help="Plot y-axis on log scale")
parser.add_argument("--ymax",
dest="ymax",
type=float,
default=None,
help="Maximum of the y-axis")
parser.add_argument("--ymin",
dest="ymin",
type=float,
default=None,
help="Minimum of the y-axis")
parser.add_argument("--xmax",
dest="xmax",
type=float,
default=None,
help="Maximum of the x-axis")
parser.add_argument("--xmin",
dest="xmin",
type=float,
default=None,
help="Minimum of the x-axis")
parser.add_argument("--ylabel",
dest="ylabel",
default=None,
help="Label on y-axis")
parser.add_argument("-s",
"--save",
dest="save",
default="",
help="Where to save plot instead of showing it?")
parser.add_argument("-t",
"--title",
dest="title",
default="",
help="Optional supertitle for plot")
parser.add_argument("--maxvalue",
dest="maxvalue",
type=float,
default=plottingscripts.utils.macros.MAXINT,
help="Replace all values higher than this?")
parser.add_argument("--agglomeration",
dest="agglomeration",
type=str,
default="median",
choices=("median", "mean"),
help="Plot mean or median")
parser.add_argument("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="print number of runs on plot")
# Properties
# We need this to show defaults for -h
defaults = plot_util.get_defaults()
for key in defaults:
parser.add_argument("--%s" % key,
dest=key,
default=None,
help="%s, default: %s" % (key, str(defaults[key])))
args, unknown = parser.parse_known_args()
sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n")
if len(unknown) < 2:
print("To less arguments given")
parser.print_help()
sys.exit(1)
if args.ylabel is None:
args.ylabel = "%s performance on instances" % args.agglomeration
# Get files and names
file_list, name_list = read_util.get_file_and_name_list(unknown,
match_file='.csv')
for idx in range(len(name_list)):
print("%20s contains %d file(s)" %
(name_list[idx], len(file_list[idx])))
if args.verbose:
name_list = [
name_list[i] + " (" + str(len(file_list[i])) + ")"
for i in range(len(name_list))
]
name_list_test_train, new_time_list, performance = get_performance_data(
file_list, name_list, args.maxvalue)
properties = helper.fill_property_dict(arguments=args, defaults=defaults)
if len(name_list) > 1:
properties["linestyles"] = itertools.cycle([":", "-"])
c = []
cycle = plot_util.get_defaults()["colors"]
for i in range(10):
color = next(cycle)
c.extend([color, color])
properties["colors"] = itertools.cycle(c)
properties["markers"] = itertools.cycle([""])
fig = plot_methods.\
plot_optimization_trace_mult_exp(time_list=new_time_list,
performance_list=performance,
title=args.title,
name_list=name_list_test_train,
logx=args.logx, logy=args.logy,
y_min=args.ymin,
y_max=args.ymax,
x_min=args.xmin,
x_max=args.xmax,
agglomeration=args.agglomeration,
ylabel=args.ylabel,
properties=properties)
if args.save != "":
print("Save plot to %s" % args.save)
plot_util.save_plot(fig, args.save, plot_util.get_defaults()['dpi'])
else:
fig.show()
def main():
prog = "python plot_ValidationPerformance.py <WhatIsThis> " \
"one/or/many/*ClassicValidationResults*.csv"
description = "Plot a median trace with quantiles for multiple experiments"
parser = ArgumentParser(description=description,
prog=prog,
formatter_class=ArgumentDefaultsHelpFormatter)
# General Options
parser.add_argument("--logx",
action="store_true",
dest="logx",
default=False,
help="Plot x-axis on log scale")
parser.add_argument("--logy",
action="store_true",
dest="logy",
default=False,
help="Plot y-axis on log scale")
parser.add_argument("--ymax",
dest="ymax",
type=float,
default=None,
help="Maximum of the y-axis")
parser.add_argument("--ymin",
dest="ymin",
type=float,
default=None,
help="Minimum of the y-axis")
parser.add_argument("--xmax",
dest="xmax",
type=float,
default=None,
help="Maximum of the x-axis")
parser.add_argument("--xmin",
dest="xmin",
type=float,
default=None,
help="Minimum of the x-axis")
parser.add_argument("--ylabel",
dest="ylabel",
default=None,
help="Label on y-axis")
parser.add_argument("--xlabel",
dest="xlabel",
default="time [sec]",
help="Label on x-axis")
parser.add_argument("-s",
"--save",
dest="save",
default="",
help="Where to save plot instead of showing it?")
parser.add_argument("-t",
"--title",
dest="title",
default="",
help="Optional supertitle for plot")
parser.add_argument("--maxvalue",
dest="maxvalue",
type=float,
default=plottingscripts.utils.macros.MAXINT,
help="Replace all values higher than this?")
parser.add_argument("--agglomeration",
dest="agglomeration",
type=str,
default="median",
choices=("median", "mean"),
help="Plot mean or median")
parser.add_argument("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="print number of runs on plot")
parser.add_argument("--scaleY",
dest="scale_y",
default=1,
type=float,
help="Multiply all Y values with this factor")
group = parser.add_mutually_exclusive_group()
group.add_argument('--train',
dest="train",
default=False,
action='store_true')
group.add_argument('--test',
dest="test",
default=True,
action='store_true')
# Properties
# We need this to show defaults for -h
defaults = plot_util.get_defaults()
for key in defaults:
parser.add_argument("--%s" % key,
dest=key,
default=None,
help="%s, default: %s" % (key, str(defaults[key])))
args, unknown = parser.parse_known_args()
sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n")
if len(unknown) < 2:
print("To few arguments given")
parser.print_help()
sys.exit(1)
if args.ylabel is None:
if args.train:
args.ylabel = "%s performance on train instances" % \
args.agglomeration
else:
args.ylabel = "%s performance on test instances" % \
args.agglomeration
# Get files and names
file_list, name_list = read_util.get_file_and_name_list(unknown,
match_file='.csv')
for idx in range(len(name_list)):
print("%20s contains %d file(s)" %
(name_list[idx], len(file_list[idx])))
if args.verbose:
name_list = [
name_list[i] + " (" + str(len(file_list[i])) + ")"
for i in range(len(name_list))
]
# Get data from csv
performance = list()
time_ = list()
show_from = -plottingscripts.utils.macros.MAXINT
for name in range(len(name_list)):
# We have a new experiment
performance.append(list())
for fl in file_list[name]:
_none, csv_data = read_util.read_csv(fl, has_header=True)
csv_data = np.array(csv_data)
# Replace too high values with args.maxint
if args.train:
data = [
min([args.maxvalue, float(i.strip())])
for i in csv_data[:, 1]
]
elif args.test:
data = [
min([args.maxvalue, float(i.strip())])
for i in csv_data[:, 2]
]
else:
print("This should not happen")
# do we have only non maxint data?
show_from = max(data.count(args.maxvalue), show_from)
performance[-1].append(data)
time_.append([float(i.strip()) for i in csv_data[:, 0]])
# Check whether we have the same times for all runs
if len(time_) == 2:
if time_[0] == time_[1]:
time_ = [
time_[0],
]
else:
raise NotImplementedError(".csv are not using the same "
"times")
performance = [np.array(i) for i in performance]
# print time_
time_ = np.array(time_).flatten()
if args.train:
print("Plot TRAIN performance")
elif args.test:
print("Plot TEST performance")
else:
print("Don't know what I'm printing")
if args.xmin is None and show_from != 0:
args.xmin = show_from
if args.scale_y != 1:
print("Scale Y values with %g", args.scale_y)
performance = np.array(performance)
performance = args.scale_y * performance
properties = helper.fill_property_dict(arguments=args, defaults=defaults)
print(properties)
new_time_list = [time_ for i in range(len(performance))]
fig = plot_methods.\
plot_optimization_trace_mult_exp(time_list=new_time_list,
performance_list=performance,
title=args.title,
name_list=name_list,
logx=args.logx, logy=args.logy,
y_min=args.ymin,
y_max=args.ymax,
x_min=args.xmin,
x_max=args.xmax,
agglomeration=args.agglomeration,
legend=(args.legendlocation is None),
ylabel=args.ylabel, xlabel=args.xlabel,
properties=properties)
if args.save != "":
print("Save plot to %s" % args.save)
plot_util.save_plot(fig, args.save, plot_util.get_defaults()['dpi'])
else:
fig.show()
def main():
"""Plot several validation runs which do not share time steps!
"""
prog = "python plot_test_performance_from_csv.py <WhatIsThis> " \
"one/or/many/*ClassicValidationResults*.csv"
description = "Merge results to one csv"
parser = ArgumentParser(description=description,
prog=prog,
formatter_class=ArgumentDefaultsHelpFormatter)
# General Options
parser.add_argument("--logy",
action="store_true",
dest="logy",
default=False,
help="Plot y-axis on log scale")
parser.add_argument("--logx",
action="store_true",
dest="logx",
default=False,
help="Plot x-axis on log scale")
parser.add_argument("--ymax",
dest="ymax",
type=float,
default=None,
help="Maximum of the y-axis")
parser.add_argument("--ymin",
dest="ymin",
type=float,
default=None,
help="Minimum of the y-axis")
parser.add_argument("--xmax",
dest="xmax",
type=float,
default=None,
help="Maximum of the x-axis")
parser.add_argument("--xmin",
dest="xmin",
type=float,
default=None,
help="Minimum of the x-axis")
parser.add_argument("-s",
"--save",
dest="save",
default="",
help="Where to save plot instead of showing it?")
parser.add_argument("-t",
"--title",
dest="title",
default=None,
help="Optional supertitle for plot")
parser.add_argument("--xlabel",
dest="xlabel",
default="time [sec]",
help="x label")
parser.add_argument("--ylabel",
dest="ylabel",
default="Minfunction value",
help="y label")
parser.add_argument("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="print number of runs on plot")
# Properties
# We need this to show defaults for -h
defaults = plot_util.get_defaults()
for key in defaults:
parser.add_argument("--%s" % key,
dest=key,
default=None,
help="%s, default: %s" % (key, str(defaults[key])))
args, unknown = parser.parse_known_args()
sys.stdout.write("\nFound " + str(len(unknown)) + " arguments\n")
if len(unknown) < 2:
print("To less arguments given")
parser.print_help()
sys.exit(1)
# Get files and names
file_list, name_list = read_util.get_file_and_name_list(unknown,
match_file='.csv')
for idx in range(len(name_list)):
print("%20s contains %d file(s)" %
(name_list[idx], len(file_list[idx])))
times = list()
performances = list()
for idx, name in enumerate(name_list):
trajectories = []
times_ = []
print("Processing %s" % name)
for csv_file in file_list[idx]:
# print(file_list[idx][0])
fh = open(csv_file, 'r')
reader = csv.reader(fh)
p = list()
t = list()
for i, row in enumerate(reader):
if i == 0:
continue
if float(row[0]) < 0:
warnings.warn('Found time stamp < 0 in file %s' % csv_file)
continue
t.append(float(row[0]))
p.append(float(row[2]))
if len(t) == 0:
print('Found empty file %s' % csv_file)
continue
times_.append(t)
trajectories.append(p)
trajectories, times_ = fill_trajectory(trajectories, times_)
times.append(times_)
performances.append(trajectories.transpose())
# Sort names alphabetical as done here:
# http://stackoverflow.com/questions/15610724/sorting-multiple-lists-in-python-based-on-sorting-of-a-single-list
sorted_lists = sorted(zip(name_list, times, performances),
key=lambda x: x[0])
name_list, times, performances = [[x[i] for x in sorted_lists]
for i in range(3)]
prop = {}
args_dict = vars(args)
for key in defaults:
prop[key] = args_dict[key]
fig = plot_methods.plot_optimization_trace_mult_exp(
time_list=times,
performance_list=performances,
title=args.title,
name_list=name_list,
ylabel=args.ylabel,
xlabel=args.xlabel,
logy=args.logy,
logx=args.logx,
y_min=args.ymin,
y_max=args.ymax,
x_min=args.xmin,
x_max=args.xmax,
properties=prop,
scale_std=1)
if args.save != "":
print("Save plot to %s" % args.save)
plot_util.save_plot(fig, args.save, plot_util.get_defaults()['dpi'])
else:
fig.show()
def main():
prog = "python plot_scatter.py"
description = "Plots performances of the best config at one time for two " \
"configuration runs"
parser = ArgumentParser(description=description, prog=prog)
# General Options
parser.add_argument("--max",
dest="max",
type=float,
default=1000,
help="Maximum of both axes")
parser.add_argument("--min",
dest="min",
type=float,
default=None,
help="Minimum of both axes")
parser.add_argument("-s",
"--save",
dest="save",
default="",
help="Where to save plot instead of showing it?")
parser.add_argument("--title",
dest="title",
default="",
help="Optional supertitle for plot")
parser.add_argument("--greyFactor",
dest="grey_factor",
type=float,
default=1,
help="If an algorithms is not greyFactor-times better"
" than the other, show this point less salient, > 1")
parser.add_argument("-v",
"--verbose",
dest="verbose",
action="store_true",
default=False,
help="Plot some debug info")
parser.add_argument(
"-f",
"--lineFactors",
dest="linefactors",
default=None,
help="Plot X speedup/slowdown, format 'X,..,X' (no spaces)")
parser.add_argument("--time",
dest="time",
default=None,
type=float,
help="Plot config at which time?")
parser.add_argument(
"--obj1",
dest="obj1",
default=None,
required=True,
help="Path to validationObjectiveMatrix-traj-run-* file")
parser.add_argument("--res1",
dest="res1",
required=True,
help="Path to validationResults-traj-run-* file")
parser.add_argument(
"--obj2",
dest="obj2",
default=None,
required=True,
help="Path to validationObjectiveMatrix-traj-run-* file")
parser.add_argument("--res2",
dest="res2",
required=True,
help="Path to validationResults-traj-run-* file")
parser.add_argument(
"--minvalue",
dest="minvalue",
type=float,
default=None,
help="Replace all values smaller than this",
)
parser.add_argument(
"--fontsize",
dest="fontsize",
type=int,
default=20,
help="Use this fontsize for plotting",
)
args, unknown = parser.parse_known_args()
if len(unknown) != 0:
print "Wrong number of arguments"
parser.print_help()
sys.exit(1)
if args.grey_factor < 1:
print "A grey-factor lower than one makes no sense"
parser.print_help()
sys.exit(1)
# Load validationResults
res1_header, res1_data = read_util.read_csv(args.res1, has_header=True)
res2_header, res2_data = read_util.read_csv(args.res2, has_header=True)
av_times = [float(row[0]) for row in res1_data]
if args.time is None:
# Print available times and quit
print "Choose a time from"
print "\n".join(["* %s" % i for i in av_times])
sys.exit(0)
# Now extract data
config_1 = [
int(float(row[len(res1_header) - 2].strip('"'))) for row in res1_data
if int(float(row[0])) == int(args.time)
]
config_2 = [
int(float(row[len(res2_header) - 2].strip('"'))) for row in res2_data
if int(float(row[0])) == int(args.time)
]
if len(config_1) == 0 or len(config_2) == 0:
print "Time int(%s) not found. Choose a time from:" % (args.time)
print "\n".join(["* %s" % i for i in av_times])
sys.exit(1)
config_1 = config_1[0]
config_2 = config_2[0]
obj1_header, obj1_data = read_util.read_csv(args.obj1, has_header=True)
obj2_header, obj2_data = read_util.read_csv(args.obj2, has_header=True)
head_template = '"Objective of validation config #%s"'
idx_1 = obj1_header.index(head_template % config_1)
idx_2 = obj2_header.index(head_template % config_2)
data_one = np.array([float(row[idx_1].strip('"')) for row in obj1_data])
data_two = np.array([float(row[idx_2].strip('"')) for row in obj2_data])
print "Found %s points for config %d and %s points for config %d" % \
(str(data_one.shape), config_1, str(data_two.shape), config_2)
linefactors = list()
if args.linefactors is not None:
linefactors = [float(i) for i in args.linefactors.split(",")]
if len(linefactors) < 1:
print "Something is wrong with linefactors: %s" % args.linefactors
sys.exit(1)
if min(linefactors) < 1:
print "A line-factor lower than one makes no sense"
sys.exit(1)
if args.grey_factor > 1 and args.grey_factor not in linefactors:
linefactors.append(args.grey_factor)
label_template = '%s %20s at %s sec'
l1 = label_template % ("obj1", os.path.basename(
args.obj1)[:20], str(args.time))
l2 = label_template % ("obj2", os.path.basename(
args.obj2)[:20], str(args.time))
if args.minvalue is not None:
print "Replace all values lower than %f" % args.minvalue
data_one = np.array([max(args.minvalue, i) for i in data_one])
data_two = np.array([max(args.minvalue, i) for i in data_two])
fig = scatter.plot_scatter_plot(x_data=data_one,
y_data=data_two,
labels=[l1, l2],
title=args.title,
max_val=args.max,
min_val=args.min,
grey_factor=args.grey_factor,
linefactors=linefactors,
user_fontsize=args.fontsize,
debug=args.verbose)
if args.save != "":
print "Save plot to %s" % args.save
plot_util.save_plot(fig=fig,
save=args.save,
dpi=plot_util.get_defaults()['dpi'])
else:
plt.show()
