def do_anova(domain, dep):
if 'every' == domain:
for dom in FOND_DOMAINS:
do_anova(dom)
return
elif 'all' in domain:
prp_data = []
dom_type = []
if 'fond' in domain:
dom_type = FOND_DOMAINS
elif 'new' in domain:
dom_type = NEW_DOMAINS
elif 'ipc' in domain:
dom_type = IPC06_DOMAINS
elif 'interesting' in domain:
dom_type = INTERESTING_DOMAINS
for dom in dom_type:
prp_data.extend(load_CSV("RESULTS/prp-%s-results.csv" % dom))
else:
prp_data = load_CSV("RESULTS/prp-%s-results.csv" % domain)
solved_prp_data = filter(lambda x: x[-2] == 'True', prp_data)
anova([prp_data[0]] + solved_prp_data,
['fullstate', 'planlocal', 'usepolicy'],
dependent=dep)
def prp_compare_two(domain, type1, type2, name1, name2):
if 'every' == domain:
for dom in FOND_DOMAINS:
prp_compare_two(dom, type1, type2)
return
elif ('all' in domain) and ('redundant' not in domain):
prp_data = []
dom_type = []
if 'fond' in domain:
dom_type = FOND_DOMAINS
elif 'new' in domain:
dom_type = NEW_DOMAINS
elif 'ipc' in domain:
dom_type = IPC06_DOMAINS
elif 'interesting' in domain:
dom_type = INTERESTING_DOMAINS
for dom in dom_type:
prp_data.extend(load_CSV("RESULTS/prp-%s-results.csv" % dom))
elif 'all-redundant' == domain:
prp_data = []
for i in range(1, 6):
prp_data.extend(
load_CSV("RESULTS/prp-blocksworld-redundant-%d-results.csv" %
i))
else:
prp_data = load_CSV("RESULTS/prp-%s-results.csv" % domain)
return _prp_compare_two(domain, type1, type2, name1, name2, prp_data)
def checkit(filename):
data_kd = load_CSV(filename + "_kd.csv")[1:]
data_kt = load_CSV(filename + "_kt.csv")[1:]
# Merge the two sets row by row
assert len(data_kd) == len(data_kt)
data = []
for i in range(0, len(data_kd)):
data.append(data_kd[i] + data_kt[i])
for row in data:
for i in range(len(row)):
if row[i] == '0.000000':
row[i] = '0.000001'
def plot_data(data, inds, labels, zlabel, fname):
data_map = {}
for size in range(SIZE[0], SIZE[1] + 10, 10):
data_map[size] = {}
for dep in range(DEPTH[0], DEPTH[1]+1):
data_map[size][dep] = {}
for row in data:
data_map[int(row[0])][int(row[1])][inds[0]] = float(row[inds[0]])
data_map[int(row[0])][int(row[1])][inds[1]] = float(row[inds[1]])
data_map[int(row[0])][int(row[1])][inds[2]] = float(row[inds[2]])
data_map[int(row[0])][int(row[1])][inds[3]] = float(row[inds[3]])
from mpl_toolkits.mplot3d import axes3d
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
X, Y = np.meshgrid(np.arange(SIZE[0], SIZE[1] + 10, 10), np.arange(DEPTH[0], DEPTH[1]+1))
zs0 = np.array([data_map[x][y][inds[0]] for x,y in zip(np.ravel(X), np.ravel(Y))])
zs1 = np.array([data_map[x][y][inds[1]] for x,y in zip(np.ravel(X), np.ravel(Y))])
zs2 = np.array([data_map[x][y][inds[2]] for x,y in zip(np.ravel(X), np.ravel(Y))])
zs3 = np.array([data_map[x][y][inds[3]] for x,y in zip(np.ravel(X), np.ravel(Y))])
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
#if 'Query Time ($log_e(sec)$)' == zlabel or 'Update Time ($log_e(sec)$)' == zlabel:
Z0 = np.log(zs0).reshape(X.shape)
Z1 = np.log(zs1).reshape(X.shape)
Z2 = np.log(zs2).reshape(X.shape)
Z3 = np.log(zs3).reshape(X.shape)
# else:
#ax.set_zticks([])
'''
Z0 = (zs0 / 1).reshape(X.shape)
Z1 = (zs1 / 1).reshape(X.shape)
Z2 = (zs2 / 1).reshape(X.shape)
Z3 = (zs3 / 1).reshape(X.shape)
'''
cols = []
for i in range(0, len(labels)):
cols.append(COLOURS[labels[i]])
ax.plot_wireframe(X, Y, Z0, color=cols[0])
ax.plot_wireframe(X, Y, Z1, color=cols[1])
ax.plot_wireframe(X, Y, Z2, color=cols[2])
ax.plot_wireframe(X, Y, Z3, color=cols[3])
#cset = ax.contourf(X, Y, Z0, zdir='z', offset=-100, cmap=matplotlib.cm.coolwarm)
#cset = ax.contourf(X, Y, Z0, zdir='x', offset=0, cmap=matplotlib.cm.coolwarm)
#cset = ax.contourf(X, Y, Z0, zdir='z', offset=0, cmap=matplotlib.cm.coolwarm)
#cset = ax.contourf(X, Y, Z0, zdir='y', offset=40, cmap=cm.coolwarm)
ax.set_xlabel('# of Agents')
ax.set_ylabel('Maximum Depth')
ax.set_zlabel(zlabel)
scatter1_proxy = matplotlib.lines.Line2D([0],[0], linestyle="none", c=cols[0], marker = 's')
scatter2_proxy = matplotlib.lines.Line2D([0],[0], linestyle="none", c=cols[1], marker = 's')
scatter3_proxy = matplotlib.lines.Line2D([0],[0], linestyle="none", c=cols[2], marker = 's')
scatter4_proxy = matplotlib.lines.Line2D([0],[0], linestyle="none", c=cols[3], marker = 's')
ax.legend([scatter1_proxy, scatter2_proxy, scatter3_proxy, scatter4_proxy], labels, numpoints = 1)
ax.get_xaxis().set_major_locator(matplotlib.ticker.MaxNLocator(integer=True))
ax.get_yaxis().set_major_locator(matplotlib.ticker.MaxNLocator(integer=True))
plt.show()
print "Plotting query time..."
plot_data(data, [V_KT_QUERY, V_KD_QUERY, CLOSURE_KD_QUERY, CLOSURE_KT_QUERY], [V_KT, V_KD, CLOSURE_KD, CLOSURE_KT], 'Query Time ($log_e(sec)$)', 'time.eps')
print "Plotting size..."
plot_data(data, [CLOSURE_KT_SIZE, CLOSURE_KD_SIZE, V_KD_SIZE, V_KT_SIZE], [CLOSURE_KT, CLOSURE_KD, V_KD, V_KT], 'Size (x1000)', 'size.eps')
print "Plotting update time..."
plot_data(data, [CLOSURE_KT_UPDATE, CLOSURE_KD_UPDATE, V_KT_UPDATE, V_KD_UPDATE], [CLOSURE_KT, CLOSURE_KD, V_KT, V_KD], 'Update Time ($log_e(sec)$)', 'update_time.eps')
def plot_from_file(csvfile,
title=None,
use_header=True,
xlog=False,
ylog=False,
no_x_tics=False,
noxy=False,
no_scatter=False,
bw=False,
mult_plots=False,
xlabel="",
ylabel="",
legendname="Legend",
nosymbols=False):
from krrt.utils import load_CSV
data = load_CSV(csvfile)
if use_header:
header = data.pop(0)
else:
header = ['x-axis', 'y-axis', 'Legend']
if mult_plots:
num_plots = len(data[0]) - 1
x = [float(item[0]) for item in data]
xs = []
ys = []
ps = []
assert use_header
for i in range(1, num_plots + 1):
y = [float(item[i]) for item in filter(lambda x: x[i] != '', data)]
xs.append(x[:len(y)])
ys.append(y)
ps.append(header[i])
plot(xs,
ys,
x_label=header[0],
y_label=ylabel,
x_log=xlog,
y_log=ylog,
names=ps,
graph_name=title,
legend_name=legendname,
disable_x_tics=no_x_tics,
xyline=not noxy,
no_scatter=no_scatter,
col=not bw,
nosymbols=nosymbols)
return
if len(data[0]) > 2:
data_keys = sorted(list(set([item[2] for item in data])))
xs = []
ys = []
ps = []
for key in data_keys:
relevant_data = filter(lambda x: x[2] == key, data)
xs.append([])
ys.append([])
ps.append(key)
for line in relevant_data:
xs[-1].append(float(line[0]))
ys[-1].append(float(line[1]))
plot(xs,
ys,
x_label=header[0],
y_label=header[1],
x_log=xlog,
y_log=ylog,
names=ps,
graph_name=title,
legend_name=header[2],
disable_x_tics=no_x_tics,
xyline=not noxy,
no_scatter=no_scatter,
col=not bw,
nosymbols=nosymbols)
else:
xs = []
ys = []
for d in data:
xs.append(float(d[0]))
ys.append(float(d[1]))
plot([xs], [ys],
x_label=header[0],
y_label=header[1],
x_log=xlog,
y_log=ylog,
graph_name=title,
disable_x_tics=no_x_tics,
xyline=not noxy,
no_scatter=no_scatter,
col=not bw,
nosymbols=nosymbols)
def compare_results(dirs):
from krrt.utils import load_CSV
print
print "Comparing results for %s..." % (dirs)
coverage = [0] * len(dirs)
time_better = [0] * len(dirs)
time_score = [0] * len(dirs)
pruned_score = [0] * len(dirs)
generated_score = [0] * len(dirs)
quality_better = [0] * len(dirs)
quality_score = [0] * len(dirs)
nodes_expanded = [0] * len(dirs)
nodes_generated = [0] * len(dirs)
nodes_pruned = [0] * len(dirs)
time_total = [0] * len(dirs)
search_time = [0] * len(dirs)
pruned_avg = [0] * len(dirs)
baseline = None
stats_results = False
if 'stats' in dirs[0]:
stats_results = True
for i in range(len(dirs)):
if dirs[i] == 'results_brfs':
baseline = i
break
for dom in domains:
data = []
cov = []
t_score = []
st_score = []
q_score = []
p_score = []
g_score = []
p_avg = []
for i in range(len(dirs)):
direc = dirs[i]
data.append(load_CSV("%s/%s.csv" % (direc, dom))[1:])
cov.append(len(filter(lambda x: 'ok' == x[1], data[-1])))
coverage[i] += cov[-1]
shared_data = []
for prob in zip(*data):
if 0 != len(filter(lambda x: 'ok' == x[1], prob)):
for x in prob:
if x[1] != 'ok':
x[2] = sys.maxint
if x[1] != 'ok':
x[3] = sys.maxint
shared_data.append(list(prob))
for i in range(len(dirs)):
tsum = 0
for x in shared_data:
if x[i][1] != 'ok': continue
tstar = max(1.0, min([float(x[j][2]) for j in range(len(x))]))
t = float(x[i][2])
if float(x[i][2]) < 1.0:
tsum+=1.0
else:
tsum += 1/(1+log10(t/tstar))
#tsum += float(tstar/t)
#tsum += log10(1+tstar)/log10(1+t)
t_score.append(tsum)
time_score[i] += t_score[i]
for i in range(len(dirs)):
tsum = 0
for x in shared_data:
if x[i][1] != 'ok': continue
tstar = float("inf")
for j in range(len(x)):
if x[j][1] != 'ok':continue
if baseline == j:
if tstar > float(x[j][2]):
tstar = float(x[j][2])
else:
if tstar > float(x[j][11]):
tstar = float(x[j][11])
tstar = max(1.0, tstar)
t = 0
if baseline == i:
t = float(x[i][2])
else:
t = float(x[i][11])
if t < 1.0:
tsum+=1.0
else:
tsum += 1/(1+log10(t/tstar))
#tsum += float(tstar/t)
#tsum += log10(1+tstar)/log10(1+t)
st_score.append(tsum)
search_time[i] += st_score[i]
for i in range(len(dirs)):
tsum = 0
for x in shared_data:
if x[i][1] != 'ok': continue
all_solved = True
for j in range(len(dirs)):
if x[j][1] != 'ok':
all_solved = False
break
if all_solved is False: continue
tstar = 1.0
for j in range(len(x)):
if x[j][1] != 'ok':continue
if tstar < float(x[j][6]): tstar = float(x[j][6])
t = float(x[i][6])
if float(x[i][6]) < 1.0:
tsum+=0.0
else:
tsum += (1+log10(t/tstar))/(1)
#tsum += float(t/tstar)
#tsum += log10(1+tstar)/log10(1+t)
p_score.append(tsum)
pruned_score[i] += p_score[i]
#Pruned Average
for i in range(len(dirs)):
tsum = 0
total_solved = 0
xprun = 0.0
for x in shared_data:
if x[i][1] != 'ok': continue
all_solved = True
for j in range(len(dirs)):
if x[j][1] != 'ok':
all_solved = False
break
if all_solved is False: continue
total_solved+=1
xprun += float(x[i][6])
if total_solved == 0:
p_avg.append( 0 )
else:
p_avg.append( xprun / total_solved )
pruned_avg[i] += p_avg[i]
if baseline is None:
for i in range(len(dirs)):
tsum = 0
for x in shared_data:
if x[i][1] != 'ok': continue
tstar = float("inf")
for j in range(len(x)):
if x[j][1] != 'ok':continue
if tstar > float(x[j][4]): tstar = float(x[j][4])
tstar = max(1.0, tstar)
t = float(x[i][4])
if t < 1.0:
tsum+=1.0
else:
tsum += 1/(1+log10(t/tstar))
#tsum += float(t/tstar)
#tsum += log10(1+tstar)/log10(1+t)
g_score.append(tsum)
generated_score[i] += g_score[i]
else:
#Generated Reduction percentage
for i in range(len(dirs)):
tsum = 0
total_solved = 0
baseline_generated = 0.0
xgen = 0.0
for x in shared_data:
if x[baseline][1] != 'ok': continue
if x[i][1] != 'ok': continue
total_solved+=1
baseline_generated += float(x[baseline][4])
xgen += float(x[i][4])
if xgen == 0:
g_score.append( 1 )
else:
g_score.append( baseline_generated / xgen )
generated_score[i] += g_score[i]
#n_gen_1 = sum([int(x[0][4]) for x in shared_data])
#n_gen_2 = sum([int(x[1][4]) for x in shared_data])
#nodes_generated[0] += n_gen_1
#nodes_generated[1] += n_gen_2
#n_exp_1 = sum([int(x[0][5]) for x in shared_data])
#n_exp_2 = sum([int(x[1][5]) for x in shared_data])
#nodes_expanded[0] += n_exp_1
#nodes_expanded[1] += n_exp_2
#time_1 = sum([float(x[0][2]) for x in shared_data])
#time_2 = sum([float(x[1][2]) for x in shared_data])
#time_total[0] += time_1
#time_total[1] += time_2
print "\nDomain: %s" % dom
print "Coverage: %s" % ' -vs- '.join(["%d" % c for c in cov])
print "Search Time score: %s" % ' -vs- '.join(["%.2f" % t for t in st_score])
print "Total Time score: %s" % ' -vs- '.join(["%.2f" % t for t in t_score])
if stats_results is False:
print "Generated Reduction Factor: %s" % ' -vs- '.join(["%.2f" % t for t in g_score])
else:
print "Avg. States Pruned: %s" % ' -vs- '.join(["%.2f" % (t/float(len(dirs))) for t in p_avg])
if baseline is None:
print "Pruned score: %s" % ' -vs- '.join(["%.2f" % t for t in p_score])
print "\nDomain: all"
print "Coverage: %s" % ' -vs- '.join(["%d" % c for c in coverage])
print "Search Time score: %s" % ' -vs- '.join(["%.2f" % t for t in search_time])
print "Total Time score: %s" % ' -vs- '.join(["%.2f" % t for t in time_score])
if stats_results is False:
print "Generated Reduction Factor: %s" % ' -vs- '.join(["%.2f" % t for t in generated_score])
else:
print "Avg. States Pruned: %s" % ' -vs- '.join(["%.2f" % (t/float(len(dirs))) for t in pruned_avg])
if baseline is None:
print "Pruned score: %s" % ' -vs- '.join(["%.2f" % t for t in pruned_score])
def checkit(filename):
data = load_CSV(filename)[1:]
for row in data:
for i in range(len(row)):
if row[i] == '0.000000':
row[i] = '0.000001'
def plot_data(data, inds, labs, cols, zlabel, fname):
data_map = {}
for ag in range(AGENTS[0], AGENTS[1] + 1):
data_map[ag] = {}
for dep in range(DEPTH[0], DEPTH[1] + 1):
data_map[ag][dep] = {}
for row in data:
data_map[int(row[0])][int(row[1])][inds[0]] = float(row[inds[0]])
data_map[int(row[0])][int(row[1])][inds[1]] = float(row[inds[1]])
data_map[int(row[0])][int(row[1])][inds[2]] = float(row[inds[2]])
from mpl_toolkits.mplot3d import axes3d
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
X, Y = np.meshgrid(np.arange(AGENTS[0], AGENTS[1] + 1),
np.arange(DEPTH[0], DEPTH[1] + 1))
#zs0 = np.array([1 for x,y in zip(np.ravel(X), np.ravel(Y))])
#zs1 = np.array([data_map[x][y][ind1] / data_map[x][y][indnorm] for x,y in zip(np.ravel(X), np.ravel(Y))])
#zs2 = np.array([data_map[x][y][ind2] / data_map[x][y][indnorm] for x,y in zip(np.ravel(X), np.ravel(Y))])
zs0 = np.array([
data_map[x][y][inds[0]] for x, y in zip(np.ravel(X), np.ravel(Y))
])
zs1 = np.array([
data_map[x][y][inds[1]] for x, y in zip(np.ravel(X), np.ravel(Y))
])
zs2 = np.array([
data_map[x][y][inds[2]] for x, y in zip(np.ravel(X), np.ravel(Y))
])
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
if 'Query Time ($log_e(sec)$)' == zlabel or 'Update Time ($log_e(sec)$)' == zlabel:
print "za = " + str(zs0)
Z0 = np.log(zs0).reshape(X.shape)
print "Z0 = " + str(Z0)
Z1 = np.log(zs1).reshape(X.shape)
Z2 = np.log(zs2).reshape(X.shape)
else:
#ax.set_zticks([])
Z0 = (zs0 / 1000).reshape(X.shape)
Z1 = (zs1 / 1000).reshape(X.shape)
Z2 = (zs2 / 1000).reshape(X.shape)
#ax.plot_wireframe(X, Y, Z0, color='0.75')
ax.plot_wireframe(X, Y, Z0, color=cols[0])
ax.plot_wireframe(X, Y, Z1, color=cols[1])
ax.plot_wireframe(X, Y, Z2, color=cols[2])
#cset = ax.contourf(X, Y, Z0, zdir='z', offset=-100, cmap=matplotlib.cm.coolwarm)
#cset = ax.contourf(X, Y, Z0, zdir='x', offset=0, cmap=matplotlib.cm.coolwarm)
#cset = ax.contourf(X, Y, Z0, zdir='z', offset=0, cmap=matplotlib.cm.coolwarm)
#cset = ax.contourf(X, Y, Z0, zdir='y', offset=40, cmap=cm.coolwarm)
ax.set_xlabel('# of Agents')
ax.set_ylabel('Maximum Depth')
ax.set_zlabel(zlabel)
scatter1_proxy = matplotlib.lines.Line2D([0], [0],
linestyle="none",
c=cols[0],
marker='s')
scatter2_proxy = matplotlib.lines.Line2D([0], [0],
linestyle="none",
c=cols[1],
marker='s')
scatter3_proxy = matplotlib.lines.Line2D([0], [0],
linestyle="none",
c=cols[2],
marker='s')
ax.legend([scatter1_proxy, scatter2_proxy, scatter3_proxy],
[labs[0], labs[1], labs[2]],
numpoints=1)
ax.get_xaxis().set_major_locator(
matplotlib.ticker.MaxNLocator(integer=True))
ax.get_yaxis().set_major_locator(
matplotlib.ticker.MaxNLocator(integer=True))
plt.show()
col1 = '#1b9e77'
col2 = '#d95f02'
col3 = '#7570b3'
print "Plotting query time..."
plot_data(data, [6, 8, 7], ['INF', '$V_{RML}$', 'Closure'],
[col1, col3, col2], 'Query Time ($log_e(sec)$)', 'time.eps')
print "Plotting size..."
plot_data(data, [4, 3, 5], ['Closure', 'INF', '$V_{RML}$'],
[col2, col1, col3], 'Size (x1000)', 'size.eps')
print "Plotting update time..."
plot_data(data, [9, 11, 10], ['INF', '$V_{RML}$', 'Closure'],
[col1, col3, col2], 'Update Time ($log_e(sec)$)',
'update_time.eps')
def compare_results(dirs):
from krrt.utils import load_CSV
print
print "Comparing results for %s..." % (dirs)
coverage = [0] * len(dirs)
time_better = [0] * len(dirs)
time_score = [0] * len(dirs)
quality_better = [0] * len(dirs)
quality_score = [0] * len(dirs)
nodes_expanded = [0] * len(dirs)
nodes_generated = [0] * len(dirs)
time_total = [0] * len(dirs)
for dom in domains:
data = []
cov = []
t_score = []
q_score = []
for i in range(len(dirs)):
direc = dirs[i]
data.append(load_CSV("%s/%s.csv" % (direc, dom))[1:])
cov.append(len(filter(lambda x: 'ok' == x[1], data[-1])))
coverage[i] += cov[-1]
shared_data = []
for prob in zip(*data):
if 0 != len(filter(lambda x: 'ok' == x[1], prob)):
for x in prob:
if x[1] != 'ok':
x[2] = sys.maxint
if x[1] != 'ok':
x[3] = sys.maxint
shared_data.append(list(prob))
for i in range(len(dirs)):
tsum = 0
for x in shared_data:
if x[i][1] != 'ok': continue
tstar = max(1.0, min([float(x[j][2]) for j in range(len(x))]))
t = float(x[i][2])
if float(x[i][2]) < 1.0:
tsum+=1.0
else:
tsum += 1/(1+log10(t/tstar))
#tsum += float(tstar/t)
#tsum += log10(1+tstar)/log10(1+t)
t_score.append(tsum)
time_score[i] += t_score[i]
for i in range(len(dirs)):
q_score.append( sum([ min([float(x[j][3]) for j in range(len(x)) ]) / float(x[i][3]) for x in shared_data ]))
quality_score[i] += q_score[i]
#n_gen_1 = sum([int(x[0][4]) for x in shared_data])
#n_gen_2 = sum([int(x[1][4]) for x in shared_data])
#nodes_generated[0] += n_gen_1
#nodes_generated[1] += n_gen_2
#n_exp_1 = sum([int(x[0][5]) for x in shared_data])
#n_exp_2 = sum([int(x[1][5]) for x in shared_data])
#nodes_expanded[0] += n_exp_1
#nodes_expanded[1] += n_exp_2
#time_1 = sum([float(x[0][2]) for x in shared_data])
#time_2 = sum([float(x[1][2]) for x in shared_data])
#time_total[0] += time_1
#time_total[1] += time_2
print "\nDomain: %s" % dom
print "Coverage: %s" % ' -vs- '.join(["%d" % c for c in cov])
#print "Time better: %d -vs- %d" % (time_better_1, time_better_2)
print "Time score: %s" % ' -vs- '.join(["%.2f" % t for t in t_score])
#print "Quality better: %d -vs- %d" % (quality_better_1, quality_better_2)
print "Quality score: %s" % ' -vs- '.join(["%.2f" % t for t in q_score])
#print "Time per node generated: %.6f -vs- %.6f" % ((float(time_1) / float(max(1,n_gen_1))), (float(time_2) / float(max(1,n_gen_2))))
#print "Time per node expanded: %.6f -vs- %.6f" % ((float(time_1) / float(max(1,n_exp_1))), (float(time_2) / float(max(1,n_exp_2))))
print "\nDomain: all"
print "Coverage: %s" % ' -vs- '.join(["%d" % c for c in coverage])
#print "Time better: %d -vs- %d" % (time_better[0], time_better[1])
print "Time score: %s" % ' -vs- '.join(["%.2f" % t for t in time_score])
#print "Quality better: %d -vs- %d" % (quality_better[0], quality_better[1])
print "Quality score: %s" % ' -vs- '.join(["%.2f" % t for t in quality_score])
def compare_results(dirs):
from krrt.utils import load_CSV
print
print "Comparing results for %s..." % (dirs)
coverage = [0] * len(dirs)
time_better = [0] * len(dirs)
time_score = [0] * len(dirs)
quality_better = [0] * len(dirs)
quality_score = [0] * len(dirs)
nodes_expanded = [0] * len(dirs)
nodes_generated = [0] * len(dirs)
time_total = [0] * len(dirs)
for dom in domains:
data = []
cov = []
t_score = []
q_score = []
for i in range(len(dirs)):
direc = dirs[i]
data.append(load_CSV("%s/%s.csv" % (direc, dom))[1:])
cov.append(len(filter(lambda x: 'ok' == x[1], data[-1])))
coverage[i] += cov[-1]
shared_data = []
for prob in zip(*data):
if 0 != len(filter(lambda x: 'ok' == x[1], prob)):
for x in prob:
if x[1] != 'ok':
x[2] = sys.maxint
if x[1] != 'ok':
x[3] = sys.maxint
shared_data.append(list(prob))
for i in range(len(dirs)):
tsum = 0
for x in shared_data:
if x[i][1] != 'ok': continue
tstar = max(1.0, min([float(x[j][2]) for j in range(len(x))]))
t = float(x[i][2])
if float(x[i][2]) < 1.0:
tsum += 1.0
else:
tsum += 1 / (1 + log10(t / tstar))
#tsum += float(tstar/t)
#tsum += log10(1+tstar)/log10(1+t)
t_score.append(tsum)
time_score[i] += t_score[i]
for i in range(len(dirs)):
q_score.append(
sum([
min([float(x[j][3])
for j in range(len(x))]) / float(x[i][3])
for x in shared_data
]))
quality_score[i] += q_score[i]
#n_gen_1 = sum([int(x[0][4]) for x in shared_data])
#n_gen_2 = sum([int(x[1][4]) for x in shared_data])
#nodes_generated[0] += n_gen_1
#nodes_generated[1] += n_gen_2
#n_exp_1 = sum([int(x[0][5]) for x in shared_data])
#n_exp_2 = sum([int(x[1][5]) for x in shared_data])
#nodes_expanded[0] += n_exp_1
#nodes_expanded[1] += n_exp_2
#time_1 = sum([float(x[0][2]) for x in shared_data])
#time_2 = sum([float(x[1][2]) for x in shared_data])
#time_total[0] += time_1
#time_total[1] += time_2
print "\nDomain: %s" % dom
print "Coverage: %s" % ' -vs- '.join(["%d" % c for c in cov])
#print "Time better: %d -vs- %d" % (time_better_1, time_better_2)
print "Time score: %s" % ' -vs- '.join(["%.2f" % t for t in t_score])
#print "Quality better: %d -vs- %d" % (quality_better_1, quality_better_2)
print "Quality score: %s" % ' -vs- '.join(
["%.2f" % t for t in q_score])
#print "Time per node generated: %.6f -vs- %.6f" % ((float(time_1) / float(max(1,n_gen_1))), (float(time_2) / float(max(1,n_gen_2))))
#print "Time per node expanded: %.6f -vs- %.6f" % ((float(time_1) / float(max(1,n_exp_1))), (float(time_2) / float(max(1,n_exp_2))))
print "\nDomain: all"
print "Coverage: %s" % ' -vs- '.join(["%d" % c for c in coverage])
#print "Time better: %d -vs- %d" % (time_better[0], time_better[1])
print "Time score: %s" % ' -vs- '.join(["%.2f" % t for t in time_score])
#print "Quality better: %d -vs- %d" % (quality_better[0], quality_better[1])
print "Quality score: %s" % ' -vs- '.join(
["%.2f" % t for t in quality_score])
def ab_compare(res1, res2):
def get_time(line):
if line[4] == '-':
return max(0.1, float(line[2]))
else:
return float('inf')
def get_size(line):
if line[4] == '-':
return float(line[3])
else:
return float('inf')
def get_score(_x, y):
x = max(_x, 0.001)
if x == float('inf') and y == float('inf'):
return 0.0
else:
return min(x, y) / x
res1_experiments = set(
map(lambda x: x.split('/')[-1], get_file_list(res1,
match_list=['.csv'])))
res2_experiments = set(
map(lambda x: x.split('/')[-1], get_file_list(res2,
match_list=['.csv'])))
shared_results = list(res1_experiments & res2_experiments)
print
print "Found %d overlapping experiments to compare." % len(shared_results)
time_score = [0.0, 0.0]
size_score = [0.0, 0.0]
coverage = [0, 0]
for res in shared_results:
data1 = load_CSV(res1 + '/' + res)[1:]
data2 = load_CSV(res2 + '/' + res)[1:]
if len(data1) != len(data2):
print "Error with %s experiment -- different number of data points." % res
continue
times1 = [get_time(line) for line in data1]
times2 = [get_time(line) for line in data2]
time1 = sum(
[get_score(times1[i], times2[i]) for i in range(len(data1))])
time2 = sum(
[get_score(times2[i], times1[i]) for i in range(len(data1))])
sizes1 = [get_size(line) for line in data1]
sizes2 = [get_size(line) for line in data2]
size1 = sum(
[get_score(sizes1[i], sizes2[i]) for i in range(len(data1))])
size2 = sum(
[get_score(sizes2[i], sizes1[i]) for i in range(len(data1))])
cov1 = len(filter(lambda x: x[4] == '-', data1))
cov2 = len(filter(lambda x: x[4] == '-', data2))
time_score[0] += time1
time_score[1] += time2
size_score[0] += size1
size_score[1] += size2
coverage[0] += cov1
coverage[1] += cov2
print "\n [ %s ]\n" % res.split('-results.csv')[0]
print " Coverage: %d -vs- %d" % (cov1, cov2)
print "Size Score: %.2f -vs- %.2f" % (size1, size2)
print "Time Score: %.2f -vs- %.2f\n" % (time1, time2)
print "\n [ OVERALL ]"
print " Coverage: %d -vs- %d" % (coverage[0], coverage[1])
print "Size Score: %.2f -vs- %.2f" % (size_score[0], size_score[1])
print "Time Score: %.2f -vs- %.2f" % (time_score[0], time_score[1])
print
def online_compare(domain):
if 'every' in domain:
dom_type = []
if 'fond' in domain:
dom_type = FOND_DOMAINS
elif 'new' in domain:
dom_type = NEW_DOMAINS
elif 'ipc' in domain:
dom_type = IPC06_DOMAINS
elif 'interesting' in domain:
dom_type = INTERESTING_DOMAINS
for dom in dom_type:
online_compare(dom)
return
elif 'all' in domain:
prp_data = []
ffr_data = []
dom_type = []
if 'fond' in domain:
dom_type = FOND_DOMAINS
elif 'new' in domain:
dom_type = NEW_DOMAINS
elif 'ipc' in domain:
dom_type = IPC06_DOMAINS
elif 'interesting' in domain:
dom_type = INTERESTING_DOMAINS
for dom in dom_type:
prp_data.extend(load_CSV("RESULTS/bo-prp-%s-results.csv" % dom))
ffr_data.extend(load_CSV("RESULTS/ffr-prp-%s-results.csv" % dom))
else:
prp_data = load_CSV("RESULTS/bo-prp-%s-results.csv" % domain)
ffr_data = load_CSV("RESULTS/ffr-prp-%s-results.csv" % domain)
# Load both sets
solved_prp_data = filter(lambda x: x[4] == '-', prp_data)
solved_ffr_data = filter(lambda x: x[4] == '-', ffr_data)
prp_mapping = {}
ffr_mapping = {}
for line in solved_ffr_data:
assert (line[0], line[1]) not in ffr_mapping
ffr_mapping[(line[0], line[1])] = line
for line in solved_prp_data:
assert (line[0], line[1]) not in prp_mapping
prp_mapping[(line[0], line[1])] = line
ffr_solved = set(ffr_mapping.keys())
prp_solved = set(prp_mapping.keys())
both_solved = ffr_solved & prp_solved
ffr_actions = []
prp_actions = []
ffr_replans = []
prp_replans = []
ffr_times = []
prp_times = []
ffr_success = []
prp_success = []
for (dom, prob) in both_solved:
ffr_actions.append(float(ffr_mapping[(dom, prob)][-4]))
prp_actions.append(float(prp_mapping[(dom, prob)][-4]))
ffr_replans.append(float(ffr_mapping[(dom, prob)][-5]))
prp_replans.append(float(prp_mapping[(dom, prob)][-5]))
ffr_success.append(float(ffr_mapping[(dom, prob)][-1]))
prp_success.append(float(prp_mapping[(dom, prob)][-1]))
ffr_times.append(
float(ffr_mapping[(dom, prob)][-7]) /
float(ffr_mapping[(dom, prob)][6]))
prp_times.append(
float(prp_mapping[(dom, prob)][-7]) /
float(prp_mapping[(dom, prob)][6]))
print "Online replanning for domain %s:\n" % domain
print "Total ffr successful runs: %d" % len(ffr_solved)
print "Total prp successful runs: %d" % len(prp_solved)
print "Total shared successful runs: %d\n" % len(both_solved)
print "FFR Average Actions: %f" % (sum(ffr_actions) /
float(len(ffr_actions)))
print "PRP Average Actions: %f\n" % (sum(prp_actions) /
float(len(prp_actions)))
print "FFR Average Replans: %f" % (sum(ffr_replans) /
float(len(ffr_replans)))
print "PRP Average Replans: %f\n" % (sum(prp_replans) /
float(len(prp_replans)))
print "FFR Average Success: %f" % (sum(ffr_success) /
float(len(ffr_success)))
print "PRP Average Success: %f\n" % (sum(prp_success) /
float(len(prp_success)))
print "FFR Total Success: %d" % int(sum(ffr_success))
print "PRP Total Success: %d\n" % int(sum(prp_success))
print "FFR Average Time: %f" % (sum(ffr_times) / float(len(ffr_times)))
print "PRP Average Time: %f\n" % (sum(prp_times) / float(len(prp_times)))
return
def fip_vs_prp(domain):
if 'every' in domain:
dom_type = []
if 'fond' in domain:
dom_type = FOND_DOMAINS
elif 'new' in domain:
dom_type = NEW_DOMAINS
elif 'ipc' in domain:
dom_type = IPC06_DOMAINS
elif 'interesting' in domain:
dom_type = INTERESTING_DOMAINS
for dom in dom_type:
fip_vs_prp(dom)
return
elif 'all' in domain:
fip_data = []
prp_data = []
dom_type = []
if 'fond' in domain:
dom_type = FOND_DOMAINS
elif 'new' in domain:
dom_type = NEW_DOMAINS
elif 'ipc' in domain:
dom_type = IPC06_DOMAINS
elif 'interesting' in domain:
dom_type = INTERESTING_DOMAINS
for dom in dom_type:
prp_data.extend(load_CSV("RESULTS/prp-%s-results.csv" % dom))
fip_data = []
prp_data = []
for dom in dom_type:
fip_data.extend(load_CSV("RESULTS/fip-%s-results.csv" % dom))
prp_data.extend(load_CSV("RESULTS/prp-%s-results.csv" % dom))
else:
fip_data = load_CSV("RESULTS/fip-%s-results.csv" % domain)
prp_data = load_CSV("RESULTS/prp-%s-results.csv" % domain)
print "\nAnalyzing FIP vs PRP for %s:" % domain
# Load both sets
solved_fip_data = filter(lambda x: x[-1] == '-' and x[-2] != '0', fip_data)
nosol_fip_data = filter(lambda x: x[-1] == 'N', fip_data)
prp_data = [prp_data[0]] + filter_prp_settings(
prp_data, '18000', '0', '0', '1', '1', '1', '1', '1', '1', '1', '1')
solved_prp_data = filter(lambda x: x[-2] == 'True', prp_data)
solved_prp_data = average_prp_data(solved_prp_data)
nosol_prp_data = filter(lambda x: x[4] == 'N' or x[-2] == 'False',
prp_data)
nosol_prp_data = average_prp_data(nosol_prp_data)
fip_mapping = {}
prp_mapping = {}
for line in solved_fip_data:
assert (line[0], line[1]) not in fip_mapping
fip_mapping[(line[0], line[1])] = line
for line in solved_prp_data:
assert (line[0], line[1]) not in prp_mapping
prp_mapping[(line[0], line[1])] = line
fip_solved = set(fip_mapping.keys())
prp_solved = set(prp_mapping.keys())
both_solved = fip_solved & prp_solved
print "FIP Coverage: %d / %d" % (len(fip_solved), len(nosol_fip_data))
print "PRP Coverage: %d / %d" % (len(prp_solved), len(nosol_prp_data))
print "Combined Coverage: %d" % len(both_solved)
print "FIP - PRP: %s" % str(fip_solved - prp_solved)
all_time_data = []
time_data = []
size_data = []
probs = []
for (dom, prob) in both_solved:
probs.append(prob)
time_data.append((float(fip_mapping[(dom, prob)][2]),
float(prp_mapping[(dom, prob)][17])))
size_data.append((float(fip_mapping[(dom, prob)][3]),
float(prp_mapping[(dom, prob)][3])))
#HACK: We assign 0.001 seconds to a reported time of 0: This is needed for log plots to work
plot([max(0.001, item[0]) for item in time_data],
[max(0.001, item[1]) for item in time_data],
x_label="FIP Time (s)",
y_label="PRP Time (s)",
makesquare=True,
x_log=True,
y_log=True,
col=False)
plot([item[0] for item in size_data], [item[1] for item in size_data],
x_label="FIP Policy Size",
y_label="PRP Policy Size",
makesquare=True,
x_log=True,
y_log=True,
col=False)
x1, y1 = create_time_profile([
max(0.001, float(fip_mapping[(dom, prob)][2]))
for (dom, prob) in fip_solved
])
x2, y2 = create_time_profile([
max(0.001, float(prp_mapping[(dom, prob)][17]))
for (dom, prob) in prp_solved
])
plot([x1, x2], [y1, y2],
x_label="Time",
y_label="Problems Solved",
no_scatter=True,
xyline=False,
names=["FIP", "PRP"],
x_log=True,
col=False)
print
('18000', '0', '1', '1', '0', '1', '0', '0', '0', '0', '0'),
('18000', '0', '0', '1', '0', '1', '0', '0', '0', '0', '0'),
'PRP$_{\\textrm{Full}}$', 'PRP')
if 'ffreplan-vs-prp' in flags:
online_compare(myargs['-domain'])
if 'anova-time' in flags:
do_anova(myargs['-domain'], 'runtime')
if 'anova-size' in flags:
do_anova(myargs['-domain'], 'size')
if 'ablation' in flags:
prp_data = load_CSV("RESULTS/best-prp-%s-results.csv" % myargs['-domain']) + \
load_CSV("RESULTS/no-local-prp-%s-results.csv" % myargs['-domain'])
_prp_compare_two(
myargs['-domain'],
('18000', '0', '0', '1', '1', '1', '1', '1', '1', '1', '1'),
('18000', '0', '0', '1', '0', '1', '0', '1', '1', '1', '1'), 'PRP',
'PRP$_{\\textrm{-local}}$', prp_data)
prp_data = load_CSV("RESULTS/best-prp-%s-results.csv" % myargs['-domain']) + \
load_CSV("RESULTS/no-scd-prp-%s-results.csv" % myargs['-domain'])
_prp_compare_two(
myargs['-domain'],
('18000', '0', '0', '1', '1', '1', '1', '1', '1', '1', '1'),
('18000', '0', '0', '1', '1', '1', '1', '1', '1', '1', '0'), 'PRP',
'PRP$_{\\textrm{-scd}}$', prp_data)
